From e4d0f8609d29d8ce1f0c4f6e18814e1b83a4a745 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 27 Jun 2018 12:36:27 -0700 Subject: [PATCH 01/49] doesn't make messages yet --- NuGet.Jobs.sln | 7 + .../Configuration/JobArgumentNames.cs | 11 + .../Configuration/JobConfigurationManager.cs | 7 + src/StatusAggregator/Component.cs | 46 +++ src/StatusAggregator/ComponentStatus.cs | 9 + src/StatusAggregator/CursorTableEntity.cs | 26 ++ .../EnvironmentIncidentParser.cs | 41 +++ src/StatusAggregator/EventEntity.cs | 38 ++ src/StatusAggregator/IIncidentParser.cs | 10 + src/StatusAggregator/Incident.cs | 30 ++ src/StatusAggregator/IncidentEntity.cs | 46 +++ src/StatusAggregator/IncidentList.cs | 15 + src/StatusAggregator/IncidentParser.cs | 26 ++ src/StatusAggregator/IncidentStatus.cs | 17 + src/StatusAggregator/Job.cs | 327 ++++++++++++++++++ src/StatusAggregator/MessageEntity.cs | 14 + ...atedSearchServiceInstanceIncidentParser.cs | 15 + src/StatusAggregator/ParsedIncident.cs | 25 ++ src/StatusAggregator/Program.cs | 16 + .../Properties/AssemblyInfo.cs | 36 ++ src/StatusAggregator/StatusAggregator.csproj | 128 +++++++ .../ValidationDurationIncidentParser.cs | 18 + src/StatusAggregator/app.config | 19 + src/StatusAggregator/packages.config | 16 + 24 files changed, 943 insertions(+) create mode 100644 src/StatusAggregator/Component.cs create mode 100644 src/StatusAggregator/ComponentStatus.cs create mode 100644 src/StatusAggregator/CursorTableEntity.cs create mode 100644 src/StatusAggregator/EnvironmentIncidentParser.cs create mode 100644 src/StatusAggregator/EventEntity.cs create mode 100644 src/StatusAggregator/IIncidentParser.cs create mode 100644 src/StatusAggregator/Incident.cs create mode 100644 src/StatusAggregator/IncidentEntity.cs create mode 100644 src/StatusAggregator/IncidentList.cs create mode 100644 src/StatusAggregator/IncidentParser.cs create mode 100644 src/StatusAggregator/IncidentStatus.cs create mode 100644 src/StatusAggregator/Job.cs create mode 100644 src/StatusAggregator/MessageEntity.cs create mode 100644 src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs create mode 100644 src/StatusAggregator/ParsedIncident.cs create mode 100644 src/StatusAggregator/Program.cs create mode 100644 src/StatusAggregator/Properties/AssemblyInfo.cs create mode 100644 src/StatusAggregator/StatusAggregator.csproj create mode 100644 src/StatusAggregator/ValidationDurationIncidentParser.cs create mode 100644 src/StatusAggregator/app.config create mode 100644 src/StatusAggregator/packages.config diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln index 66daf9b57..54e8629d5 100644 --- a/NuGet.Jobs.sln +++ b/NuGet.Jobs.sln @@ -113,6 +113,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job", "sr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PackageHash", "src\PackageHash\PackageHash.csproj", "{40843020-6F0A-48F0-AC28-42FFE3A5C21E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusAggregator", "src\StatusAggregator\StatusAggregator.csproj", "{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -289,6 +291,10 @@ Global {40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.Build.0 = Debug|Any CPU {40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.ActiveCfg = Release|Any CPU {40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.Build.0 = Release|Any CPU + {D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -336,6 +342,7 @@ Global {B4B7564A-965B-447B-927F-6749E2C08880} = {6A776396-02B1-475D-A104-26940ADB04AB} {FA87D075-A934-4443-8D0B-5DB32640B6D7} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} {40843020-6F0A-48F0-AC28-42FFE3A5C21E} = {FA5644B5-4F08-43F6-86B3-039374312A47} + {D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB} = {FA5644B5-4F08-43F6-86B3-039374312A47} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B} diff --git a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs index 64a09132c..25642cbc0 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs @@ -117,5 +117,16 @@ public static class JobArgumentNames public const string MailFrom = "MailFrom"; public const string SmtpUri = "SmtpUri"; + // Arguments specific to StatusAggregator + public const string StatusStorageAccount = "StatusStorageAccount"; + public const string StatusContainerName = "StatusContainerName"; + public const string StatusTableName = "StatusTableName"; + public const string StatusEnvironment = "StatusEnvironment"; + public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri"; + public const string StatusIncidentApiRoutingId = "StatusIncidentApiRoutingId"; + public const string StatusIncidentApiCertificateThumbprint = "StatusIncidentApiCertificateThumbprint"; + public const string StatusIncidentApiCertificateStoreName = "StatusIncidentApiCertificateStoreName"; + public const string StatusIncidentApiCertificateStoreLocation = "StatusIncidentApiCertificateStoreLocation"; + public const string StatusIncidentApiEnvironment = "StatusIncidentApiEnvironment"; } } \ No newline at end of file diff --git a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs index 879d50619..e9dca4765 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs @@ -147,6 +147,13 @@ public static bool TryGetBoolArgument(IDictionary jobArgsDiction return null; } + public static T TryGetEnumArgument(IDictionary jobArgsDictionary, string argName, T defaultValue) + where T : struct + { + var argumentString = TryGetArgument(jobArgsDictionary, argName); + return !string.IsNullOrEmpty(argumentString) && Enum.TryParse(argumentString, out T result) ? result : defaultValue; + } + private static Dictionary ReadCommandLineArguments(ILogger logger, string[] commandLineArgs) { var argsDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/StatusAggregator/Component.cs b/src/StatusAggregator/Component.cs new file mode 100644 index 000000000..5ac3b6d31 --- /dev/null +++ b/src/StatusAggregator/Component.cs @@ -0,0 +1,46 @@ +using Newtonsoft.Json; +using System.Collections.Generic; +using System.Linq; + +namespace StatusAggregator +{ + public class Component + { + public string Name { get; set; } + + public string Description { get; set; } + + private ComponentStatus _status; + public ComponentStatus Status + { + get + { + if (!SubComponents.Any()) + { + return _status; + } + + // If all subcomponents are up, we are up. + if (SubComponents.All(c => c.Status == ComponentStatus.Up)) + { + return ComponentStatus.Up; + } + + // If all subcomponents are down, we are down. + if (SubComponents.All(c => c.Status == ComponentStatus.Down)) + { + return ComponentStatus.Down; + } + + // Otherwise, we are degraded, because some subcomponents are degraded or down but not all. + return ComponentStatus.Degraded; + } + set + { + _status = value; + } + } + + public IEnumerable SubComponents { get; set; } + } +} diff --git a/src/StatusAggregator/ComponentStatus.cs b/src/StatusAggregator/ComponentStatus.cs new file mode 100644 index 000000000..686a2ddb4 --- /dev/null +++ b/src/StatusAggregator/ComponentStatus.cs @@ -0,0 +1,9 @@ +namespace StatusAggregator +{ + public enum ComponentStatus + { + Up, + Degraded, + Down + } +} diff --git a/src/StatusAggregator/CursorTableEntity.cs b/src/StatusAggregator/CursorTableEntity.cs new file mode 100644 index 000000000..9029ec9ee --- /dev/null +++ b/src/StatusAggregator/CursorTableEntity.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.WindowsAzure.Storage.Table; + +namespace StatusAggregator +{ + public class CursorEntity : TableEntity + { + public const string DefaultPartitionKey = "cursors"; + public const string DefaultRowKey = "1"; + + public CursorEntity() + { + } + + public CursorEntity(DateTime value) + : base(DefaultPartitionKey, DefaultRowKey) + { + Value = value; + } + + public DateTime Value { get; set; } + } +} diff --git a/src/StatusAggregator/EnvironmentIncidentParser.cs b/src/StatusAggregator/EnvironmentIncidentParser.cs new file mode 100644 index 000000000..a7a5dd756 --- /dev/null +++ b/src/StatusAggregator/EnvironmentIncidentParser.cs @@ -0,0 +1,41 @@ +using System; +using System.Text.RegularExpressions; + +namespace StatusAggregator +{ + public class EnvironmentIncidentParser : IncidentParser + { + private const string EnvironmentGroupName = "Environment"; + + private readonly string _environment; + + public EnvironmentIncidentParser(string subtitleRegEx, string environment) + : base(GetRegEx(subtitleRegEx)) + { + _environment = environment; + } + + protected override bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident) + { + parsedIncident = null; + + if (incident.Severity > 2) + { + return false; + } + + if (!string.Equals(groups[EnvironmentGroupName].Value, _environment, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + parsedIncident = new ParsedIncident(incident, "Publishing", ComponentStatus.Degraded); + return true; + } + + private static string GetRegEx(string subtitleRegEx) + { + return $@"\[(?<{EnvironmentGroupName}>.*)\] {subtitleRegEx}"; + } + } +} diff --git a/src/StatusAggregator/EventEntity.cs b/src/StatusAggregator/EventEntity.cs new file mode 100644 index 000000000..8ec79d77b --- /dev/null +++ b/src/StatusAggregator/EventEntity.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.WindowsAzure.Storage.Table; + +namespace StatusAggregator +{ + public class EventEntity : TableEntity + { + public const string DefaultPartitionKey = "events"; + + public EventEntity() + { + } + + public EventEntity(IncidentEntity incidentEntity) + : base(DefaultPartitionKey, GetRowKey(incidentEntity)) + { + AffectedComponentPath = incidentEntity.AffectedComponentPath; + AffectedComponentStatus = incidentEntity.AffectedComponentStatus; + StartTime = incidentEntity.CreationTime; + incidentEntity.EventRowKey = RowKey; + } + + public string AffectedComponentPath { get; set; } + public ComponentStatus AffectedComponentStatus { get; set; } + public DateTime StartTime { get; set; } + public DateTime? EndTime { get; set; } + public bool IsActive + { + get { return EndTime == null; } + set { } + } + + private static string GetRowKey(IncidentEntity incidentEntity) + { + return $"{incidentEntity.AffectedComponentPath}_{incidentEntity.CreationTime.ToString("o")}"; + } + } +} diff --git a/src/StatusAggregator/IIncidentParser.cs b/src/StatusAggregator/IIncidentParser.cs new file mode 100644 index 000000000..cb43695e8 --- /dev/null +++ b/src/StatusAggregator/IIncidentParser.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace StatusAggregator +{ + public interface IIncidentParser + { + bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident); + } +} diff --git a/src/StatusAggregator/Incident.cs b/src/StatusAggregator/Incident.cs new file mode 100644 index 000000000..f16904ded --- /dev/null +++ b/src/StatusAggregator/Incident.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace StatusAggregator +{ + public class Incident + { + public string Id { get; set; } + public int Severity { get; set; } + public IncidentStatus Status { get; set; } + public DateTime CreateDate { get; set; } + public string Title { get; set; } + public string RoutingId { get; set; } + public IncidentSourceData Source { get; set; } + public IncidentStateChangeEventData MitigationData { get; set; } + public IncidentStateChangeEventData ResolutionData { get; set; } + } + + public class IncidentSourceData + { + public DateTime CreateDate { get; set; } + } + + public class IncidentStateChangeEventData + { + public DateTime Date { get; set; } + } +} diff --git a/src/StatusAggregator/IncidentEntity.cs b/src/StatusAggregator/IncidentEntity.cs new file mode 100644 index 000000000..070155b12 --- /dev/null +++ b/src/StatusAggregator/IncidentEntity.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.WindowsAzure.Storage.Table; + +namespace StatusAggregator +{ + public class IncidentEntity : TableEntity + { + public const string DefaultPartitionKey = "incidents"; + + public IncidentEntity() + { + } + + public IncidentEntity(ParsedIncident parsedIncident) + : base(DefaultPartitionKey, GetRowKey(parsedIncident)) + { + IncidentApiId = parsedIncident.Id; + AffectedComponentPath = parsedIncident.AffectedComponentPath; + AffectedComponentStatus = parsedIncident.AffectedComponentStatus; + CreationTime = parsedIncident.CreationTime; + MitigationTime = parsedIncident.MitigationTime; + } + + public string EventRowKey { get; set; } + public bool IsLinkedToEvent + { + get { return !string.IsNullOrEmpty(EventRowKey); } + set { } + } + public string IncidentApiId { get; set; } + public string AffectedComponentPath { get; set; } + public ComponentStatus AffectedComponentStatus { get; set; } + public DateTime CreationTime { get; set; } + public DateTime? MitigationTime { get; set; } + public bool IsActive + { + get { return MitigationTime == null; } + set { } + } + + private static string GetRowKey(ParsedIncident parsedIncident) + { + return $"{parsedIncident.Id}_{parsedIncident.AffectedComponentPath}_{parsedIncident.AffectedComponentStatus}"; + } + } +} diff --git a/src/StatusAggregator/IncidentList.cs b/src/StatusAggregator/IncidentList.cs new file mode 100644 index 000000000..a60923465 --- /dev/null +++ b/src/StatusAggregator/IncidentList.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace StatusAggregator +{ + public class IncidentList + { + [JsonProperty(PropertyName = "value")] + public IEnumerable Incidents { get; set; } + + [JsonProperty(PropertyName = "odata.nextLink")] + public Uri NextLink { get; set; } + } +} diff --git a/src/StatusAggregator/IncidentParser.cs b/src/StatusAggregator/IncidentParser.cs new file mode 100644 index 000000000..4ec99d2a2 --- /dev/null +++ b/src/StatusAggregator/IncidentParser.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Text.RegularExpressions; + +namespace StatusAggregator +{ + public abstract class IncidentParser : IIncidentParser + { + private readonly string _regExPattern; + + public IncidentParser(string regExPattern) + { + _regExPattern = regExPattern; + } + + public bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident) + { + parsedIncident = null; + var match = Regex.Match(incident.Title, _regExPattern); + return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); + } + + protected abstract bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident); + } +} diff --git a/src/StatusAggregator/IncidentStatus.cs b/src/StatusAggregator/IncidentStatus.cs new file mode 100644 index 000000000..fc03576b5 --- /dev/null +++ b/src/StatusAggregator/IncidentStatus.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace StatusAggregator +{ + public enum IncidentStatus + { + Holding, + Active, + Mitigated, + Resolved, + Suppressed, + New, + Correlating, + Mitigating + } +} diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs new file mode 100644 index 000000000..6d478a4f6 --- /dev/null +++ b/src/StatusAggregator/Job.cs @@ -0,0 +1,327 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Security.Cryptography.X509Certificates; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.Table; +using Newtonsoft.Json; +using NuGet.Jobs; +using NuGet.Services.KeyVault; + +namespace StatusAggregator +{ + public class Job : JobBase + { + private const string StatusBlobName = "status.json"; + + private static TimeSpan EventStartDelay = TimeSpan.FromMinutes(15); + private static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); + + private CloudBlobContainer _container; + private CloudTable _table; + + private Uri _incidentApiBaseUri; + private X509Certificate2 _incidentApiCertificate; + private string _incidentApiRoutingId; + private string _incidentApiEnvironment; + + public override void Init(IDictionary jobArgsDictionary) + { + var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount); + var storageAccount = CloudStorageAccount.Parse(storageConnectionString); + + var blobClient = storageAccount.CreateCloudBlobClient(); + var containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName); + _container = blobClient.GetContainerReference(containerName); + + var tableClient = storageAccount.CreateCloudTableClient(); + var tableName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName); + _table = tableClient.GetTableReference(tableName); + + _incidentApiBaseUri = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)); + _incidentApiRoutingId = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiRoutingId); + _incidentApiEnvironment = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiEnvironment); + var incidentApiCertificateThumbprint = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificateThumbprint); + var incidentApiCertificateStoreName = + JobConfigurationManager.TryGetEnumArgument( + jobArgsDictionary, + JobArgumentNames.StatusIncidentApiCertificateStoreName, + StoreName.My); + var incidentApiCertificateStoreLocation = + JobConfigurationManager.TryGetEnumArgument( + jobArgsDictionary, + JobArgumentNames.StatusIncidentApiCertificateStoreLocation, + StoreLocation.LocalMachine); + _incidentApiCertificate = CertificateUtility.FindCertificateByThumbprint( + incidentApiCertificateStoreName, + incidentApiCertificateStoreLocation, + incidentApiCertificateThumbprint, + true); + } + + public override async Task Run() + { + // await _container.CreateIfNotExistsAsync(); + await _table.CreateIfNotExistsAsync(); + + await AggregateEvents(); + } + + private async Task AggregateEvents() + { + // Check the status of any active incidents. + var activeIncidentEntities = _table + .CreateQuery() + .AsQueryable() + .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); + + foreach (var activeIncidentEntity in activeIncidentEntities) + { + var activeIncident = await GetIncident(activeIncidentEntity.IncidentApiId); + activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; + var incidentOperation = TableOperation.InsertOrReplace(activeIncidentEntity); + await _table.ExecuteAsync(incidentOperation); + } + + // Fetch any new incidents. + var lastCursor = GetCursor(); + var nextCursor = DateTime.MinValue; + Console.WriteLine($"Read cursor at {lastCursor.ToString("o")}"); + var incidentParsers = new IIncidentParser[] { new ValidationDurationIncidentParser("PROD"), new OutdatedSearchServiceInstanceIncidentParser("PROD") }; + var parsedIncidents = new List(); + string query = lastCursor == DateTime.MinValue + ? GetIncidentApiIncidentListAllIncidentsQuery() + : GetIncidentApiIncidentListRecentIncidentsQuery(lastCursor); + var nextLink = GetIncidentApiUri(GetIncidentApiIncidentList(query)); + do + { + var incidents = await GetIncidentApiResponse(nextLink); + foreach (var incident in incidents.Incidents) + { + if (incident.CreateDate <= lastCursor) + { + continue; + } + + if (incident.CreateDate > nextCursor) + { + nextCursor = incident.CreateDate; + } + + foreach (var incidentParser in incidentParsers) + { + if (incidentParser.TryParseIncident(incident, out var parsedIncident)) + { + Console.WriteLine($"Found {parsedIncident.Id} affecting {parsedIncident.AffectedComponentPath} with status {parsedIncident.AffectedComponentStatus} from {parsedIncident.CreationTime} to {parsedIncident.MitigationTime}"); + parsedIncidents.Add(parsedIncident); + } + } + } + + nextLink = incidents.NextLink; + } while (nextLink != null); + + // Close any active events that no longer have any active incidents. + var creationTimesPerPath = parsedIncidents + .GroupBy(i => i.AffectedComponentPath) + .ToDictionary(g => g.Key, g => g.Min(i => i.CreationTime)); + + DateTime minCreationTime = creationTimesPerPath.Any() + ? creationTimesPerPath.Min(t => t.Value) + : new[] { nextCursor, DateTime.UtcNow }.Max(); + + var eventsToCheckClosure = _table + .CreateQuery() + .AsQueryable() + .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && e.IsActive); + + foreach (var eventToCheckClosure in eventsToCheckClosure) + { + var nextCreationTime = creationTimesPerPath.ContainsKey(eventToCheckClosure.AffectedComponentPath) + ? creationTimesPerPath[eventToCheckClosure.AffectedComponentPath] + : minCreationTime; + + await CheckEventForClosure(eventToCheckClosure, nextCreationTime); + } + + // Aggregate the new incidents and create new events if necessary. + if (parsedIncidents.Any()) + { + foreach (var parsedIncident in parsedIncidents.OrderBy(i => i.CreationTime)) + { + Console.WriteLine($"Attempting to save {parsedIncident.Id}"); + var incidentEntity = new IncidentEntity(parsedIncident); + + // Find an event to attach this incident to + var possibleEvents = _table + .CreateQuery() + .AsQueryable() + .Where(e => + e.PartitionKey == EventEntity.DefaultPartitionKey && + // The incident and the event must affect the same component + e.AffectedComponentPath == parsedIncident.AffectedComponentPath && + // The event must begin before or at the same time as the incident + e.StartTime <= parsedIncident.CreationTime && + // The event must be active or the event must end after this incident begins + (e.IsActive || (e.EndTime >= parsedIncident.CreationTime))) + .ToList(); + + Console.WriteLine($"Found {possibleEvents.Count()} possible events to link {parsedIncident.Id} to"); + + if (possibleEvents.Any()) + { + foreach (var possibleEvent in possibleEvents) + { + if (!GetIncidentsLinkedToEvent(possibleEvent).ToList().Any()) + { + Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because it is not linked to any incidents"); + continue; + } + + if (await CheckEventForClosure(possibleEvent, parsedIncident.CreationTime)) + { + Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because its incidents are inactive and too old"); + continue; + } + + Console.WriteLine($"Linking {parsedIncident.Id} to {possibleEvent.RowKey}"); + incidentEntity.EventRowKey = possibleEvents.First().RowKey; + break; + } + } + + if (string.IsNullOrEmpty(incidentEntity.EventRowKey)) + { + var eventEntity = new EventEntity(incidentEntity); + Console.WriteLine($"Could not find existing event to attach {parsedIncident.Id} to, creating new event {eventEntity.RowKey}"); + var eventOperation = TableOperation.InsertOrReplace(eventEntity); + await _table.ExecuteAsync(eventOperation); + } + + var incidentOperation = TableOperation.InsertOrReplace(incidentEntity); + await _table.ExecuteAsync(incidentOperation); + } + } + + // Update the cursor to signify that we've fetched all incidents thus far. + if (nextCursor > lastCursor) + { + var cursorEntity = new CursorEntity(nextCursor); + var operation = TableOperation.InsertOrReplace(cursorEntity); + await _table.ExecuteAsync(operation); + } + } + + private async Task CheckEventForClosure(EventEntity eventEntity, DateTime nextCreationTime) + { + if (!eventEntity.IsActive) + { + return false; + } + + var incidentsLinkedToEventToClose = GetIncidentsLinkedToEvent(eventEntity); + + var shouldClose = !incidentsLinkedToEventToClose + .Where(i => i.IsActive || i.MitigationTime >= nextCreationTime - EventEndDelay) + .ToList() + .Any(); + + if (shouldClose) + { + Console.WriteLine($"Closing {eventEntity.RowKey} because its incidents are inactive and too old"); + eventEntity.EndTime = incidentsLinkedToEventToClose + .ToList() + .Max(i => i.MitigationTime); + var eventOperation = TableOperation.InsertOrReplace(eventEntity); + await _table.ExecuteAsync(eventOperation); + } + + return shouldClose; + } + + private IQueryable GetIncidentsLinkedToEvent(EventEntity eventEntity) + { + return _table + .CreateQuery() + .AsQueryable() + .Where(i => + i.PartitionKey == IncidentEntity.DefaultPartitionKey && + i.IsLinkedToEvent && + i.EventRowKey == eventEntity.RowKey); + } + + private DateTime GetCursor() + { + var query = new TableQuery() + .Where(TableQuery.GenerateFilterCondition( + nameof(ITableEntity.PartitionKey), + QueryComparisons.Equal, + CursorEntity.DefaultPartitionKey)); + + var cursors = _table.ExecuteQuery(query).ToArray(); + return cursors.Any() + ? cursors.Max(c => c.Value) + : DateTime.MinValue; + } + + private const string IncidentApiIncidentsEndpoint = "incidents"; + private static readonly string IncidentApiIndividualIncidentQueryFormatString = $"{IncidentApiIncidentsEndpoint}({{0}})"; + private static readonly string IncidentApiIncidentListQueryFormatString = $"{IncidentApiIncidentsEndpoint}?{{0}}"; + + private static JsonSerializerSettings _incidentApiJsonSerializerSettings = new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc }; + + private string GetIncidentApiIncidentList(string oDataQueryParameters) + { + return string.Format(IncidentApiIncidentListQueryFormatString, oDataQueryParameters); + } + + private string GetIncidentApiGetIncidentQuery(string id) + { + return string.Format(IncidentApiIndividualIncidentQueryFormatString, id); + } + + private string GetIncidentApiIncidentListAllIncidentsQuery() + { + return $"$filter=RoutingId eq '{_incidentApiRoutingId}'"; + } + + private string GetIncidentApiIncidentListRecentIncidentsQuery(DateTime cursor) + { + return $"$filter=RoutingId eq '{_incidentApiRoutingId}' and CreateDate gt datetime'{cursor.ToString("o")}'"; + } + + private Uri GetIncidentApiUri(string query) + { + return new Uri(_incidentApiBaseUri, query); + } + + private Task GetIncident(string id) + { + return GetIncidentApiResponse(GetIncidentApiGetIncidentQuery(id)); + } + + private Task GetIncidentApiResponse(string query) + { + return GetIncidentApiResponse(GetIncidentApiUri(query)); + } + + private async Task GetIncidentApiResponse(Uri uri) + { + var request = WebRequest.CreateHttp(uri); + request.ClientCertificates.Add(_incidentApiCertificate); + var response = await request.GetResponseAsync(); + using (var reader = new StreamReader(response.GetResponseStream())) + { + var content = await reader.ReadToEndAsync(); + return JsonConvert.DeserializeObject(content, _incidentApiJsonSerializerSettings); + } + } + } +} diff --git a/src/StatusAggregator/MessageEntity.cs b/src/StatusAggregator/MessageEntity.cs new file mode 100644 index 000000000..6ea5150a9 --- /dev/null +++ b/src/StatusAggregator/MessageEntity.cs @@ -0,0 +1,14 @@ +using System; +using Microsoft.WindowsAzure.Storage.Table; + +namespace StatusAggregator +{ + public class MessageEntity : TableEntity + { + public const string DefaultPartitionKey = "messages"; + + public string EventRowKey { get; set; } + public DateTime Time { get; set; } + public string Message { get; set; } + } +} diff --git a/src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs new file mode 100644 index 000000000..f7aa76508 --- /dev/null +++ b/src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs @@ -0,0 +1,15 @@ +namespace StatusAggregator +{ + public class OutdatedSearchServiceInstanceIncidentParser : EnvironmentIncidentParser + { + private const string SubtitleRegEx = "A search service instance is using an outdated index!"; + + private readonly string _environment; + + public OutdatedSearchServiceInstanceIncidentParser(string environment) + : base(SubtitleRegEx, environment) + { + _environment = environment; + } + } +} diff --git a/src/StatusAggregator/ParsedIncident.cs b/src/StatusAggregator/ParsedIncident.cs new file mode 100644 index 000000000..7c1bee196 --- /dev/null +++ b/src/StatusAggregator/ParsedIncident.cs @@ -0,0 +1,25 @@ +using System; + +namespace StatusAggregator +{ + public class ParsedIncident + { + public ParsedIncident( + Incident incident, + string affectedComponentPath, + ComponentStatus affectedComponentStatus) + { + Id = incident.Id; + CreationTime = incident.Source.CreateDate; + MitigationTime = incident.MitigationData?.Date; + AffectedComponentPath = affectedComponentPath; + AffectedComponentStatus = affectedComponentStatus; + } + + public string Id { get; set; } + public string AffectedComponentPath { get; set; } + public ComponentStatus AffectedComponentStatus { get; set; } + public DateTime CreationTime { get; set; } + public DateTime? MitigationTime { get; set; } + } +} diff --git a/src/StatusAggregator/Program.cs b/src/StatusAggregator/Program.cs new file mode 100644 index 000000000..ce9117e0d --- /dev/null +++ b/src/StatusAggregator/Program.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Jobs; + +namespace StatusAggregator +{ + public class Program + { + public static void Main(string[] args) + { + var job = new Job(); + JobRunner.Run(job, args).Wait(); + } + } +} diff --git a/src/StatusAggregator/Properties/AssemblyInfo.cs b/src/StatusAggregator/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..3c3117262 --- /dev/null +++ b/src/StatusAggregator/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("StatusAggregator")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("StatusAggregator")] +[assembly: AssemblyCopyright("Copyright © 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("d357fdb5-bf19-41a5-82b0-14c8cec2a5eb")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj new file mode 100644 index 000000000..e35717691 --- /dev/null +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -0,0 +1,128 @@ + + + + + Debug + AnyCPU + {D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB} + Exe + Properties + StatusAggregator + StatusAggregator + v4.5.2 + 512 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + StatusAggregator.Program + + + + ..\..\packages\Hyak.Common.1.0.2\lib\net45\Hyak.Common.dll + + + ..\..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.dll + + + ..\..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.NetFramework.dll + + + ..\..\packages\Microsoft.Azure.KeyVault.1.0.0\lib\net45\Microsoft.Azure.KeyVault.dll + + + ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll + + + ..\..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.13.5\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll + + + ..\..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.13.5\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll + + + ..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll + + + ..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll + + + ..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll + + + ..\..\packages\WindowsAzure.Storage.9.2.0\lib\net45\Microsoft.WindowsAzure.Storage.dll + + + ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + + + ..\..\packages\NuGet.Services.KeyVault.2.13.0\lib\net45\NuGet.Services.KeyVault.dll + + + + + + ..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll + + + ..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} + NuGet.Jobs.Common + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StatusAggregator/ValidationDurationIncidentParser.cs b/src/StatusAggregator/ValidationDurationIncidentParser.cs new file mode 100644 index 000000000..d871bd169 --- /dev/null +++ b/src/StatusAggregator/ValidationDurationIncidentParser.cs @@ -0,0 +1,18 @@ +using System; +using System.Text.RegularExpressions; + +namespace StatusAggregator +{ + public class ValidationDurationIncidentParser : EnvironmentIncidentParser + { + private const string SubtitleRegEx = "Too many packages are stuck in the \"Validating\" state!"; + + private readonly string _environment; + + public ValidationDurationIncidentParser(string environment) + : base(SubtitleRegEx, environment) + { + _environment = environment; + } + } +} diff --git a/src/StatusAggregator/app.config b/src/StatusAggregator/app.config new file mode 100644 index 000000000..6e6c219e3 --- /dev/null +++ b/src/StatusAggregator/app.config @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StatusAggregator/packages.config b/src/StatusAggregator/packages.config new file mode 100644 index 000000000..2924a5a6c --- /dev/null +++ b/src/StatusAggregator/packages.config @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file From 03c14784dc3722a0b09989940107f94f1dbb48d3 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 27 Jun 2018 15:50:09 -0700 Subject: [PATCH 02/49] add messaging --- src/StatusAggregator/Job.cs | 55 ++++++++++++++++-- src/StatusAggregator/MessageEntity.cs | 17 ++++++ src/StatusAggregator/StatusAggregator.csproj | 60 +++----------------- src/StatusAggregator/app.config | 19 +++---- src/StatusAggregator/packages.config | 16 ------ 5 files changed, 83 insertions(+), 84 deletions(-) delete mode 100644 src/StatusAggregator/packages.config diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 86033c03a..1ca0c8cbd 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -149,7 +149,7 @@ private async Task AggregateEvents() ? creationTimesPerPath[eventToCheckClosure.AffectedComponentPath] : minCreationTime; - await CheckEventForClosure(eventToCheckClosure, nextCreationTime); + await UpdateEventAndCheckForClosure(eventToCheckClosure, nextCreationTime); } // Aggregate the new incidents and create new events if necessary. @@ -186,7 +186,7 @@ private async Task AggregateEvents() continue; } - if (await CheckEventForClosure(possibleEvent, parsedIncident.CreationTime)) + if (await UpdateEventAndCheckForClosure(possibleEvent, parsedIncident.CreationTime)) { Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because its incidents are inactive and too old"); continue; @@ -220,33 +220,76 @@ private async Task AggregateEvents() } } - private async Task CheckEventForClosure(EventEntity eventEntity, DateTime nextCreationTime) + private async Task UpdateEventAndCheckForClosure(EventEntity eventEntity, DateTime nextCreationTime) { if (!eventEntity.IsActive) { + // Inactive events have already been closed. return false; } var incidentsLinkedToEventToClose = GetIncidentsLinkedToEvent(eventEntity); + if (!incidentsLinkedToEventToClose.ToList().Any()) + { + // If an event has no linked incidents it must have been created manually and should not be closed automatically. + return false; + } + var shouldClose = !incidentsLinkedToEventToClose - .Where(i => i.IsActive || i.MitigationTime >= nextCreationTime - EventEndDelay) + .Where(i => i.IsActive || i.MitigationTime > nextCreationTime - EventEndDelay) .ToList() .Any(); if (shouldClose) { Console.WriteLine($"Closing {eventEntity.RowKey} because its incidents are inactive and too old"); - eventEntity.EndTime = incidentsLinkedToEventToClose + var mitigationTime = incidentsLinkedToEventToClose .ToList() - .Max(i => i.MitigationTime); + .Max(i => i.MitigationTime ?? DateTime.MinValue); + + await CreateMessageForEventStartIfTimeHasPassed(eventEntity, mitigationTime); + + // Create a message to alert customers that the event is resolved. + // Only create a message if the event already has messages associated with it. + var messagesForEvent = _table + .CreateQuery() + .AsQueryable() + .Where(m => + m.PartitionKey == MessageEntity.DefaultPartitionKey && + m.EventRowKey == eventEntity.RowKey) + .ToList(); + + if (messagesForEvent.Any()) + { + var messageEntity = new MessageEntity(eventEntity, mitigationTime, "NO LONGER IMPACTED"); + var messageOperation = TableOperation.InsertOrReplace(messageEntity); + await _table.ExecuteAsync(messageOperation); + } + + // Update the event + eventEntity.EndTime = mitigationTime; var eventOperation = TableOperation.InsertOrReplace(eventEntity); await _table.ExecuteAsync(eventOperation); } + else + { + await CreateMessageForEventStartIfTimeHasPassed(eventEntity, nextCreationTime); + } return shouldClose; } + private async Task CreateMessageForEventStartIfTimeHasPassed(EventEntity eventEntity, DateTime currentTime) + { + if (currentTime > eventEntity.StartTime + EventStartDelay) + { + var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "WE ARE IMPACTED"); + var messageOperation = TableOperation.InsertOrReplace(messageEntity); + await _table.ExecuteAsync(messageOperation); + } + } + private IQueryable GetIncidentsLinkedToEvent(EventEntity eventEntity) { return _table diff --git a/src/StatusAggregator/MessageEntity.cs b/src/StatusAggregator/MessageEntity.cs index 6ea5150a9..3fd3a9de4 100644 --- a/src/StatusAggregator/MessageEntity.cs +++ b/src/StatusAggregator/MessageEntity.cs @@ -7,8 +7,25 @@ public class MessageEntity : TableEntity { public const string DefaultPartitionKey = "messages"; + public MessageEntity() + { + } + + public MessageEntity(EventEntity eventEntity, DateTime time, string message) + : base(DefaultPartitionKey, GetRowKey(eventEntity, time)) + { + EventRowKey = eventEntity.RowKey; + Time = time; + Message = message; + } + public string EventRowKey { get; set; } public DateTime Time { get; set; } public string Message { get; set; } + + private static string GetRowKey(EventEntity eventEntity, DateTime time) + { + return $"{eventEntity.RowKey}_{time.ToString("o")}"; + } } } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index e35717691..2c07a990e 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -9,8 +9,10 @@ Properties StatusAggregator StatusAggregator - v4.5.2 + v4.6.2 512 + true + true @@ -33,54 +35,9 @@ StatusAggregator.Program - - ..\..\packages\Hyak.Common.1.0.2\lib\net45\Hyak.Common.dll - - - ..\..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.dll - - - ..\..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.NetFramework.dll - - - ..\..\packages\Microsoft.Azure.KeyVault.1.0.0\lib\net45\Microsoft.Azure.KeyVault.dll - - - ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - - ..\..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.13.5\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - - - ..\..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.13.5\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll - - - ..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - - - ..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - - - ..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll - - - ..\..\packages\WindowsAzure.Storage.9.2.0\lib\net45\Microsoft.WindowsAzure.Storage.dll - - - ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll - - - ..\..\packages\NuGet.Services.KeyVault.2.13.0\lib\net45\NuGet.Services.KeyVault.dll - - - ..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - - - ..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll - @@ -117,12 +74,11 @@ - + + + + 9.2.0 + - - - - - \ No newline at end of file diff --git a/src/StatusAggregator/app.config b/src/StatusAggregator/app.config index 6e6c219e3..8881498ea 100644 --- a/src/StatusAggregator/app.config +++ b/src/StatusAggregator/app.config @@ -1,19 +1,18 @@ - + - - + + - - - - - - + + - \ No newline at end of file + + + + diff --git a/src/StatusAggregator/packages.config b/src/StatusAggregator/packages.config deleted file mode 100644 index 2924a5a6c..000000000 --- a/src/StatusAggregator/packages.config +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file From d27b115197e740a819964e73f1f1818a2a01cb9c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 27 Jun 2018 17:59:59 -0700 Subject: [PATCH 03/49] export simple blob --- src/StatusAggregator/Component.cs | 60 ++++++- .../EnvironmentIncidentParser.cs | 2 +- src/StatusAggregator/Event.cs | 30 ++++ src/StatusAggregator/EventEntity.cs | 2 +- src/StatusAggregator/IncidentEntity.cs | 2 +- src/StatusAggregator/Job.cs | 162 +++++++++++++++--- src/StatusAggregator/Message.cs | 20 +++ src/StatusAggregator/MessageEntity.cs | 6 +- src/StatusAggregator/Status.cs | 23 +++ src/StatusAggregator/StatusAggregator.csproj | 3 + 10 files changed, 280 insertions(+), 30 deletions(-) create mode 100644 src/StatusAggregator/Event.cs create mode 100644 src/StatusAggregator/Message.cs create mode 100644 src/StatusAggregator/Status.cs diff --git a/src/StatusAggregator/Component.cs b/src/StatusAggregator/Component.cs index 5ac3b6d31..2195e6c6a 100644 --- a/src/StatusAggregator/Component.cs +++ b/src/StatusAggregator/Component.cs @@ -1,23 +1,70 @@ -using Newtonsoft.Json; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; namespace StatusAggregator { public class Component { + public const char ComponentPathDivider = '/'; + + public Component() + { + } + + public Component( + string name, + string description) + { + Name = name; + Description = description; + SubComponents = Enumerable.Empty(); + } + + public Component( + string name, + string description, + ComponentStatus status) + : this(name, description) + { + Status = status; + } + + public Component( + string name, + string description, + IEnumerable subComponents) + : this(name, description) + { + SubComponents = subComponents; + } + + public Component( + string name, + string description, + ComponentStatus status, + IEnumerable subComponents) + : this(name, description, status) + { + SubComponents = subComponents; + } + public string Name { get; set; } public string Description { get; set; } - private ComponentStatus _status; + private ComponentStatus? _status = null; public ComponentStatus Status { get { + if (_status.HasValue) + { + return _status.Value; + } + if (!SubComponents.Any()) { - return _status; + return ComponentStatus.Up; } // If all subcomponents are up, we are up. @@ -42,5 +89,10 @@ public ComponentStatus Status } public IEnumerable SubComponents { get; set; } + + public static string ToRowKeySafeComponentPath(string componentPath) + { + return componentPath.Replace(ComponentPathDivider, '_'); + } } } diff --git a/src/StatusAggregator/EnvironmentIncidentParser.cs b/src/StatusAggregator/EnvironmentIncidentParser.cs index a7a5dd756..9dd326e5b 100644 --- a/src/StatusAggregator/EnvironmentIncidentParser.cs +++ b/src/StatusAggregator/EnvironmentIncidentParser.cs @@ -29,7 +29,7 @@ protected override bool TryParseIncident(Incident incident, GroupCollection grou return false; } - parsedIncident = new ParsedIncident(incident, "Publishing", ComponentStatus.Degraded); + parsedIncident = new ParsedIncident(incident, $"NuGet{Component.ComponentPathDivider}Package Publishing", ComponentStatus.Degraded); return true; } diff --git a/src/StatusAggregator/Event.cs b/src/StatusAggregator/Event.cs new file mode 100644 index 000000000..40eab956e --- /dev/null +++ b/src/StatusAggregator/Event.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public class Event + { + public Event() + { + } + + public Event(EventEntity eventEntity, IEnumerable messages) + { + AffectedComponentPath = eventEntity.AffectedComponentPath; + AffectedComponentStatus = eventEntity.AffectedComponentStatus; + StartTime = eventEntity.StartTime; + EndTime = eventEntity.EndTime; + Messages = messages; + } + + public string AffectedComponentPath { get; set; } + public ComponentStatus AffectedComponentStatus { get; set; } + public DateTime StartTime { get; set; } + public DateTime? EndTime { get; set; } + public IEnumerable Messages { get; set; } + } +} diff --git a/src/StatusAggregator/EventEntity.cs b/src/StatusAggregator/EventEntity.cs index 8ec79d77b..0fcca5a33 100644 --- a/src/StatusAggregator/EventEntity.cs +++ b/src/StatusAggregator/EventEntity.cs @@ -32,7 +32,7 @@ public bool IsActive private static string GetRowKey(IncidentEntity incidentEntity) { - return $"{incidentEntity.AffectedComponentPath}_{incidentEntity.CreationTime.ToString("o")}"; + return $"{Component.ToRowKeySafeComponentPath(incidentEntity.AffectedComponentPath)}_{incidentEntity.CreationTime.ToString("o")}"; } } } diff --git a/src/StatusAggregator/IncidentEntity.cs b/src/StatusAggregator/IncidentEntity.cs index 070155b12..0d880a471 100644 --- a/src/StatusAggregator/IncidentEntity.cs +++ b/src/StatusAggregator/IncidentEntity.cs @@ -40,7 +40,7 @@ public bool IsActive private static string GetRowKey(ParsedIncident parsedIncident) { - return $"{parsedIncident.Id}_{parsedIncident.AffectedComponentPath}_{parsedIncident.AffectedComponentStatus}"; + return $"{parsedIncident.Id}_{Component.ToRowKeySafeComponentPath(parsedIncident.AffectedComponentPath)}_{parsedIncident.AffectedComponentStatus}"; } } } diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 1ca0c8cbd..1b3ebfd2a 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -13,6 +13,7 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.Table; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; using NuGet.Jobs; using NuGet.Services.KeyVault; @@ -24,6 +25,7 @@ public class Job : JobBase private static TimeSpan EventStartDelay = TimeSpan.FromMinutes(15); private static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); + private static TimeSpan EventVisibilityPeriod = TimeSpan.FromDays(7); private CloudBlobContainer _container; private CloudTable _table; @@ -69,14 +71,14 @@ public override void Init(IServiceContainer serviceContainer, IDictionary() @@ -138,11 +140,7 @@ private async Task AggregateEvents() ? creationTimesPerPath.Min(t => t.Value) : new[] { nextCursor, DateTime.UtcNow }.Max(); - var eventsToCheckClosure = _table - .CreateQuery() - .AsQueryable() - .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && e.IsActive); - + var eventsToCheckClosure = GetActiveEvents(); foreach (var eventToCheckClosure in eventsToCheckClosure) { var nextCreationTime = creationTimesPerPath.ContainsKey(eventToCheckClosure.AffectedComponentPath) @@ -220,6 +218,120 @@ private async Task AggregateEvents() } } + private static readonly JsonSerializerSettings _statusBlobJsonSerializerSettings = new JsonSerializerSettings() + { + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + Converters = new List() { new StringEnumConverter() } + }; + + private async Task ExportData() + { + await _container.CreateIfNotExistsAsync(); + + var rootComponent = SetupRootComponent(); + + var activeEvents = GetActiveEvents(); + foreach (var activeEvent in activeEvents) + { + var componentPathParts = activeEvent.AffectedComponentPath.Split(Component.ComponentPathDivider); + + var currentComponent = new Component("", "", new[] { rootComponent }); + foreach (var componentPathPart in componentPathParts) + { + currentComponent = currentComponent.SubComponents.FirstOrDefault(c => c.Name == componentPathPart); + + if (currentComponent == null) + { + break; + } + } + + if (currentComponent == null) + { + continue; + } + + currentComponent.Status = activeEvent.AffectedComponentStatus; + } + + var recentEvents = _table + .CreateQuery() + .AsQueryable() + .Where(e => + e.PartitionKey == EventEntity.DefaultPartitionKey && + (e.IsActive || (e.EndTime >= DateTime.Now - EventVisibilityPeriod))) + .ToList() + .Select(e => + { + var messages = GetMessagesLinkedToEvent(e) + .ToList() + .Select(m => new Message(m)); + return new Event(e, messages); + }); + + var status = new Status(rootComponent, recentEvents); + var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); + + var blob = _container.GetBlockBlobReference(StatusBlobName); + await blob.UploadTextAsync(statusJson); + } + + private Component SetupRootComponent() + { + return new Component( + "NuGet", + "", + new[] + { + new Component( + "NuGet.org", + "Browsing the Gallery website", + new[] + { + new Component("USNC", "Primary region"), + new Component("USSC", "Backup region") + }), + new Component( + "Restore", + "Downloading and installing packages from NuGet", + new[] + { + new Component( + "V3", + "Restore using the V3 API", + new[] + { + new Component("Global", "V3 restore for users outside of China"), + new Component("China", "V3 restore for users inside China") + }), + new Component("V2", "Restore using the V2 API") + }), + new Component( + "Search", + "Searching for new and existing packages in Visual Studio or the Gallery website", + new[] + { + new Component( + "Global", + "Search for packages outside Asia", + new[] + { + new Component("USNC", "Primary region"), + new Component("USSC", "Backup region") + }), + new Component( + "Asia", + "Search for packages inside Asia", + new[] + { + new Component("EA", "Primary region"), + new Component("SEA", "Backup region") + }) + }), + new Component("Package Publishing", "Uploading new packages to NuGet.org") + }); + } + private async Task UpdateEventAndCheckForClosure(EventEntity eventEntity, DateTime nextCreationTime) { if (!eventEntity.IsActive) @@ -252,15 +364,7 @@ private async Task UpdateEventAndCheckForClosure(EventEntity eventEntity, // Create a message to alert customers that the event is resolved. // Only create a message if the event already has messages associated with it. - var messagesForEvent = _table - .CreateQuery() - .AsQueryable() - .Where(m => - m.PartitionKey == MessageEntity.DefaultPartitionKey && - m.EventRowKey == eventEntity.RowKey) - .ToList(); - - if (messagesForEvent.Any()) + if (GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { var messageEntity = new MessageEntity(eventEntity, mitigationTime, "NO LONGER IMPACTED"); var messageOperation = TableOperation.InsertOrReplace(messageEntity); @@ -290,6 +394,14 @@ private async Task CreateMessageForEventStartIfTimeHasPassed(EventEntity eventEn } } + private IQueryable GetActiveEvents() + { + return _table + .CreateQuery() + .AsQueryable() + .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && e.IsActive); + } + private IQueryable GetIncidentsLinkedToEvent(EventEntity eventEntity) { return _table @@ -301,6 +413,16 @@ private IQueryable GetIncidentsLinkedToEvent(EventEntity eventEn i.EventRowKey == eventEntity.RowKey); } + private IQueryable GetMessagesLinkedToEvent(EventEntity eventEntity) + { + return _table + .CreateQuery() + .AsQueryable() + .Where(m => + m.PartitionKey == MessageEntity.DefaultPartitionKey && + m.EventRowKey == eventEntity.RowKey); + } + private DateTime GetCursor() { var query = new TableQuery() @@ -319,7 +441,7 @@ private DateTime GetCursor() private static readonly string IncidentApiIndividualIncidentQueryFormatString = $"{IncidentApiIncidentsEndpoint}({{0}})"; private static readonly string IncidentApiIncidentListQueryFormatString = $"{IncidentApiIncidentsEndpoint}?{{0}}"; - private static JsonSerializerSettings _incidentApiJsonSerializerSettings = new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc }; + private static readonly JsonSerializerSettings _incidentApiJsonSerializerSettings = new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc }; private string GetIncidentApiIncidentList(string oDataQueryParameters) { diff --git a/src/StatusAggregator/Message.cs b/src/StatusAggregator/Message.cs new file mode 100644 index 000000000..04460920a --- /dev/null +++ b/src/StatusAggregator/Message.cs @@ -0,0 +1,20 @@ +using System; + +namespace StatusAggregator +{ + public class Message + { + public Message() + { + } + + public Message(MessageEntity messageEntity) + { + Time = messageEntity.Time; + Contents = messageEntity.Contents; + } + + public DateTime Time { get; set; } + public string Contents { get; set; } + } +} diff --git a/src/StatusAggregator/MessageEntity.cs b/src/StatusAggregator/MessageEntity.cs index 3fd3a9de4..c616a4b1c 100644 --- a/src/StatusAggregator/MessageEntity.cs +++ b/src/StatusAggregator/MessageEntity.cs @@ -11,17 +11,17 @@ public MessageEntity() { } - public MessageEntity(EventEntity eventEntity, DateTime time, string message) + public MessageEntity(EventEntity eventEntity, DateTime time, string contents) : base(DefaultPartitionKey, GetRowKey(eventEntity, time)) { EventRowKey = eventEntity.RowKey; Time = time; - Message = message; + Contents = contents; } public string EventRowKey { get; set; } public DateTime Time { get; set; } - public string Message { get; set; } + public string Contents { get; set; } private static string GetRowKey(EventEntity eventEntity, DateTime time) { diff --git a/src/StatusAggregator/Status.cs b/src/StatusAggregator/Status.cs new file mode 100644 index 000000000..b7eab10ee --- /dev/null +++ b/src/StatusAggregator/Status.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +namespace StatusAggregator +{ + public class Status + { + public Status() + { + } + + public Status(Component rootComponent, IEnumerable events) + { + LastUpdated = DateTime.Now; + RootComponent = rootComponent; + Events = events; + } + + public DateTime LastUpdated { get; set; } + public Component RootComponent { get; set; } + public IEnumerable Events { get; set; } + } +} diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 2c07a990e..a0d0c4c67 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -51,6 +51,7 @@ + @@ -59,11 +60,13 @@ + + From d82856604f61ad59002579ffc91200ec146eb089 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 29 Jun 2018 17:07:16 -0700 Subject: [PATCH 04/49] reorganized status classes --- .vs/config/applicationhost.config | 10 +- src/StatusAggregator/Component.cs | 61 +---- src/StatusAggregator/Cursor.cs | 36 +++ src/StatusAggregator/Event.cs | 3 +- src/StatusAggregator/IComponent.cs | 12 + src/StatusAggregator/ICursor.cs | 14 ++ .../Incidents/IIncidentCollector.cs | 13 ++ .../{ => Incidents}/Incident.cs | 2 +- .../Incidents/IncidentCollector.cs | 112 +++++++++ .../{ => Incidents}/IncidentList.cs | 2 +- .../{ => Incidents}/IncidentStatus.cs | 2 +- .../Parse}/EnvironmentIncidentParser.cs | 20 +- .../{ => Incidents/Parse}/IIncidentParser.cs | 2 +- .../{ => Incidents/Parse}/IncidentParser.cs | 2 +- ...onalSearchServiceInstanceIncidentParser.cs | 79 +++++++ ...atedSearchServiceInstanceIncidentParser.cs | 29 +++ .../{ => Incidents/Parse}/ParsedIncident.cs | 2 +- .../Parse/ValidationDurationIncidentParser.cs | 30 +++ src/StatusAggregator/Job.cs | 218 ++++++------------ src/StatusAggregator/Message.cs | 3 +- ...atedSearchServiceInstanceIncidentParser.cs | 15 -- .../PrimarySecondaryComponent.cs | 90 ++++++++ src/StatusAggregator/Status.cs | 4 +- src/StatusAggregator/StatusAggregator.csproj | 36 +-- .../CursorEntity.cs} | 2 +- .../{ => Table}/EventEntity.cs | 2 +- src/StatusAggregator/Table/ITableWrapper.cs | 16 ++ .../{ => Table}/IncidentEntity.cs | 3 +- .../{ => Table}/MessageEntity.cs | 2 +- src/StatusAggregator/Table/TableWrapper.cs | 37 +++ src/StatusAggregator/TreeComponent.cs | 82 +++++++ .../ValidationDurationIncidentParser.cs | 18 -- 32 files changed, 685 insertions(+), 274 deletions(-) create mode 100644 src/StatusAggregator/Cursor.cs create mode 100644 src/StatusAggregator/IComponent.cs create mode 100644 src/StatusAggregator/ICursor.cs create mode 100644 src/StatusAggregator/Incidents/IIncidentCollector.cs rename src/StatusAggregator/{ => Incidents}/Incident.cs (96%) create mode 100644 src/StatusAggregator/Incidents/IncidentCollector.cs rename src/StatusAggregator/{ => Incidents}/IncidentList.cs (90%) rename src/StatusAggregator/{ => Incidents}/IncidentStatus.cs (90%) rename src/StatusAggregator/{ => Incidents/Parse}/EnvironmentIncidentParser.cs (54%) rename src/StatusAggregator/{ => Incidents/Parse}/IIncidentParser.cs (87%) rename src/StatusAggregator/{ => Incidents/Parse}/IncidentParser.cs (95%) create mode 100644 src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs create mode 100644 src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs rename src/StatusAggregator/{ => Incidents/Parse}/ParsedIncident.cs (94%) create mode 100644 src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs delete mode 100644 src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs create mode 100644 src/StatusAggregator/PrimarySecondaryComponent.cs rename src/StatusAggregator/{CursorTableEntity.cs => Table/CursorEntity.cs} (95%) rename src/StatusAggregator/{ => Table}/EventEntity.cs (97%) create mode 100644 src/StatusAggregator/Table/ITableWrapper.cs rename src/StatusAggregator/{ => Table}/IncidentEntity.cs (95%) rename src/StatusAggregator/{ => Table}/MessageEntity.cs (96%) create mode 100644 src/StatusAggregator/Table/TableWrapper.cs create mode 100644 src/StatusAggregator/TreeComponent.cs delete mode 100644 src/StatusAggregator/ValidationDurationIncidentParser.cs diff --git a/.vs/config/applicationhost.config b/.vs/config/applicationhost.config index 582b1211a..030cffcb2 100644 --- a/.vs/config/applicationhost.config +++ b/.vs/config/applicationhost.config @@ -163,12 +163,20 @@ - + + + + + + + + + diff --git a/src/StatusAggregator/Component.cs b/src/StatusAggregator/Component.cs index 2195e6c6a..5aed4df72 100644 --- a/src/StatusAggregator/Component.cs +++ b/src/StatusAggregator/Component.cs @@ -3,7 +3,7 @@ namespace StatusAggregator { - public class Component + public abstract class Component : IComponent { public const char ComponentPathDivider = '/'; @@ -23,27 +23,8 @@ public Component( public Component( string name, string description, - ComponentStatus status) + IEnumerable subComponents) : this(name, description) - { - Status = status; - } - - public Component( - string name, - string description, - IEnumerable subComponents) - : this(name, description) - { - SubComponents = subComponents; - } - - public Component( - string name, - string description, - ComponentStatus status, - IEnumerable subComponents) - : this(name, description, status) { SubComponents = subComponents; } @@ -52,43 +33,9 @@ public Component( public string Description { get; set; } - private ComponentStatus? _status = null; - public ComponentStatus Status - { - get - { - if (_status.HasValue) - { - return _status.Value; - } - - if (!SubComponents.Any()) - { - return ComponentStatus.Up; - } - - // If all subcomponents are up, we are up. - if (SubComponents.All(c => c.Status == ComponentStatus.Up)) - { - return ComponentStatus.Up; - } - - // If all subcomponents are down, we are down. - if (SubComponents.All(c => c.Status == ComponentStatus.Down)) - { - return ComponentStatus.Down; - } - - // Otherwise, we are degraded, because some subcomponents are degraded or down but not all. - return ComponentStatus.Degraded; - } - set - { - _status = value; - } - } + public IEnumerable SubComponents { get; set; } - public IEnumerable SubComponents { get; set; } + public abstract ComponentStatus Status { get; set; } public static string ToRowKeySafeComponentPath(string componentPath) { diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs new file mode 100644 index 000000000..e35af4c14 --- /dev/null +++ b/src/StatusAggregator/Cursor.cs @@ -0,0 +1,36 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using StatusAggregator.Table; + + +namespace StatusAggregator +{ + public class Cursor : ICursor + { + public Cursor(ITableWrapper table) + { + _table = table; + } + + private ITableWrapper _table; + + public DateTime Get() + { + var cursors = _table + .CreateQuery() + .Where(e => e.PartitionKey == CursorEntity.DefaultPartitionKey) + .ToList(); + + return cursors.Any() + ? cursors.Max(c => c.Value) + : DateTime.MinValue; + } + + public Task Set(DateTime value) + { + var cursorEntity = new CursorEntity(value); + return _table.InsertOrReplaceAsync(cursorEntity); + } + } +} diff --git a/src/StatusAggregator/Event.cs b/src/StatusAggregator/Event.cs index 40eab956e..1d79d6c9a 100644 --- a/src/StatusAggregator/Event.cs +++ b/src/StatusAggregator/Event.cs @@ -1,4 +1,5 @@ -using System; +using StatusAggregator.Table; +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/src/StatusAggregator/IComponent.cs b/src/StatusAggregator/IComponent.cs new file mode 100644 index 000000000..a592305ee --- /dev/null +++ b/src/StatusAggregator/IComponent.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; + +namespace StatusAggregator +{ + public interface IComponent + { + string Name { get; set; } + string Description { get; set; } + ComponentStatus Status { get; set; } + IEnumerable SubComponents { get; set; } + } +} diff --git a/src/StatusAggregator/ICursor.cs b/src/StatusAggregator/ICursor.cs new file mode 100644 index 000000000..474c7a516 --- /dev/null +++ b/src/StatusAggregator/ICursor.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public interface ICursor + { + DateTime Get(); + Task Set(DateTime value); + } +} diff --git a/src/StatusAggregator/Incidents/IIncidentCollector.cs b/src/StatusAggregator/Incidents/IIncidentCollector.cs new file mode 100644 index 000000000..ca66d0a25 --- /dev/null +++ b/src/StatusAggregator/Incidents/IIncidentCollector.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace StatusAggregator.Incidents +{ + public interface IIncidentCollector + { + Task GetIncident(string incidentApiId); + + Task> GetRecentIncidents(DateTime since); + } +} diff --git a/src/StatusAggregator/Incident.cs b/src/StatusAggregator/Incidents/Incident.cs similarity index 96% rename from src/StatusAggregator/Incident.cs rename to src/StatusAggregator/Incidents/Incident.cs index f16904ded..dd3369707 100644 --- a/src/StatusAggregator/Incident.cs +++ b/src/StatusAggregator/Incidents/Incident.cs @@ -3,7 +3,7 @@ using System; -namespace StatusAggregator +namespace StatusAggregator.Incidents { public class Incident { diff --git a/src/StatusAggregator/Incidents/IncidentCollector.cs b/src/StatusAggregator/Incidents/IncidentCollector.cs new file mode 100644 index 000000000..bf0041b39 --- /dev/null +++ b/src/StatusAggregator/Incidents/IncidentCollector.cs @@ -0,0 +1,112 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Security.Cryptography.X509Certificates; +using System.Threading.Tasks; + +namespace StatusAggregator.Incidents +{ + public class IncidentCollector : IIncidentCollector + { + + private const string IncidentApiIncidentsEndpoint = "incidents"; + private static readonly string IncidentApiIndividualIncidentQueryFormatString = $"{IncidentApiIncidentsEndpoint}({{0}})"; + private static readonly string IncidentApiIncidentListQueryFormatString = $"{IncidentApiIncidentsEndpoint}?{{0}}"; + + private static readonly JsonSerializerSettings _incidentApiJsonSerializerSettings = + new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc }; + + public IncidentCollector( + Uri incidentApiBaseUri, + X509Certificate2 incidentApiCertificate, + string incidentApiRoutingId) + { + _incidentApiBaseUri = incidentApiBaseUri; + _incidentApiCertificate = incidentApiCertificate; + _incidentApiRoutingId = incidentApiRoutingId; + } + + public Task GetIncident(string id) + { + return GetIncidentApiResponse(GetIncidentApiGetIncidentQuery(id)); + } + + public async Task> GetRecentIncidents(DateTime since) + { + var incidents = new List(); + + string query = since == DateTime.MinValue + ? GetIncidentApiIncidentListAllIncidentsQuery() + : GetIncidentApiIncidentListRecentIncidentsQuery(since); + var nextLink = GetIncidentApiUri(GetIncidentApiIncidentList(query)); + do + { + var incidentList = await GetIncidentApiResponse(nextLink); + foreach (var incident in incidentList.Incidents) + { + // The incident API trims the milliseconds from any filter. + // Therefore, a query asking for incidents newer than '2018-06-29T00:00:00.5Z' will return an incident from '2018-06-29T00:00:00.25Z' + // We must perform a check on the CreateDate ourselves to verify that no old incidents are returned. + if (incident.CreateDate <= since) + { + continue; + } + + incidents.Add(incident); + } + + nextLink = incidentList.NextLink; + } while (nextLink != null); + + return incidents; + } + + private readonly Uri _incidentApiBaseUri; + private readonly X509Certificate2 _incidentApiCertificate; + private readonly string _incidentApiRoutingId; + + private string GetIncidentApiIncidentList(string oDataQueryParameters) + { + return string.Format(IncidentApiIncidentListQueryFormatString, oDataQueryParameters); + } + + private string GetIncidentApiGetIncidentQuery(string id) + { + return string.Format(IncidentApiIndividualIncidentQueryFormatString, id); + } + + private string GetIncidentApiIncidentListAllIncidentsQuery() + { + return $"$filter=RoutingId eq '{_incidentApiRoutingId}'"; + } + + private string GetIncidentApiIncidentListRecentIncidentsQuery(DateTime cursor) + { + return $"$filter=RoutingId eq '{_incidentApiRoutingId}' and CreateDate gt datetime'{cursor.ToString("o")}'"; + } + + private Uri GetIncidentApiUri(string query) + { + return new Uri(_incidentApiBaseUri, query); + } + + private Task GetIncidentApiResponse(string query) + { + return GetIncidentApiResponse(GetIncidentApiUri(query)); + } + + private async Task GetIncidentApiResponse(Uri uri) + { + var request = WebRequest.CreateHttp(uri); + request.ClientCertificates.Add(_incidentApiCertificate); + var response = await request.GetResponseAsync(); + using (var reader = new StreamReader(response.GetResponseStream())) + { + var content = await reader.ReadToEndAsync(); + return JsonConvert.DeserializeObject(content, _incidentApiJsonSerializerSettings); + } + } + } +} diff --git a/src/StatusAggregator/IncidentList.cs b/src/StatusAggregator/Incidents/IncidentList.cs similarity index 90% rename from src/StatusAggregator/IncidentList.cs rename to src/StatusAggregator/Incidents/IncidentList.cs index a60923465..64250a511 100644 --- a/src/StatusAggregator/IncidentList.cs +++ b/src/StatusAggregator/Incidents/IncidentList.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using Newtonsoft.Json; -namespace StatusAggregator +namespace StatusAggregator.Incidents { public class IncidentList { diff --git a/src/StatusAggregator/IncidentStatus.cs b/src/StatusAggregator/Incidents/IncidentStatus.cs similarity index 90% rename from src/StatusAggregator/IncidentStatus.cs rename to src/StatusAggregator/Incidents/IncidentStatus.cs index fc03576b5..db7824b0b 100644 --- a/src/StatusAggregator/IncidentStatus.cs +++ b/src/StatusAggregator/Incidents/IncidentStatus.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace StatusAggregator +namespace StatusAggregator.Incidents { public enum IncidentStatus { diff --git a/src/StatusAggregator/EnvironmentIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs similarity index 54% rename from src/StatusAggregator/EnvironmentIncidentParser.cs rename to src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs index 9dd326e5b..b7e4a6f36 100644 --- a/src/StatusAggregator/EnvironmentIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs @@ -1,9 +1,9 @@ using System; using System.Text.RegularExpressions; -namespace StatusAggregator +namespace StatusAggregator.Incidents.Parse { - public class EnvironmentIncidentParser : IncidentParser + public abstract class EnvironmentIncidentParser : IncidentParser { private const string EnvironmentGroupName = "Environment"; @@ -29,10 +29,24 @@ protected override bool TryParseIncident(Incident incident, GroupCollection grou return false; } - parsedIncident = new ParsedIncident(incident, $"NuGet{Component.ComponentPathDivider}Package Publishing", ComponentStatus.Degraded); + if (!TryParseAffectedComponentPath(incident, groups, out var affectedComponentPath)) + { + return false; + } + + if (!TryParseAffectedComponentStatus(incident, groups, out var affectedComponentStatus)) + { + return false; + } + + parsedIncident = new ParsedIncident(incident, affectedComponentPath, affectedComponentStatus); return true; } + protected abstract bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath); + + protected abstract bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus); + private static string GetRegEx(string subtitleRegEx) { return $@"\[(?<{EnvironmentGroupName}>.*)\] {subtitleRegEx}"; diff --git a/src/StatusAggregator/IIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/IIncidentParser.cs similarity index 87% rename from src/StatusAggregator/IIncidentParser.cs rename to src/StatusAggregator/Incidents/Parse/IIncidentParser.cs index cb43695e8..0551030c7 100644 --- a/src/StatusAggregator/IIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/IIncidentParser.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace StatusAggregator +namespace StatusAggregator.Incidents.Parse { public interface IIncidentParser { diff --git a/src/StatusAggregator/IncidentParser.cs b/src/StatusAggregator/Incidents/Parse/IncidentParser.cs similarity index 95% rename from src/StatusAggregator/IncidentParser.cs rename to src/StatusAggregator/Incidents/Parse/IncidentParser.cs index 4ec99d2a2..4c240834e 100644 --- a/src/StatusAggregator/IncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/IncidentParser.cs @@ -3,7 +3,7 @@ using System.Text.RegularExpressions; -namespace StatusAggregator +namespace StatusAggregator.Incidents.Parse { public abstract class IncidentParser : IIncidentParser { diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs new file mode 100644 index 000000000..bb44dc894 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs @@ -0,0 +1,79 @@ +using System; +using System.Text.RegularExpressions; + +namespace StatusAggregator.Incidents.Parse +{ + public class OutdatedRegionalSearchServiceInstanceIncidentParser : EnvironmentIncidentParser + { + private const string ServiceEnvironmentGroupName = "SearchServiceName"; + private const string ServiceRegionGroupName = "SearchServiceName"; + private static string SubtitleRegEx = $@"Search service 'nuget-\[(?<{ServiceEnvironmentGroupName}>.*)\]-\[(?<{ServiceRegionGroupName}>.*)\]-search' is using an outdated index!"; + + private readonly string _environment; + + public OutdatedRegionalSearchServiceInstanceIncidentParser(string environment) + : base(SubtitleRegEx, environment) + { + _environment = environment; + } + + protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) + { + affectedComponentPath = null; + + var searchEnvironment = groups[ServiceEnvironmentGroupName].Value; + var searchRegion = groups[ServiceRegionGroupName].Value; + + if (!string.Equals(searchEnvironment, _environment, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + string region; + string subRegion; + + switch (searchRegion) + { + case "0": + case "usnc": + case "ussc": + region = "Global"; + break; + case "eastasia": + case "southeastasia": + region = "Asia"; + break; + default: + return false; + } + + switch (searchRegion) + { + case "0": + case "usnc": + subRegion = "USNC"; + break; + case "ussc": + subRegion = "USSC"; + break; + case "eastasia": + subRegion = "EA"; + break; + case "southeastasia": + subRegion = "SEA"; + break; + default: + return false; + } + + affectedComponentPath = $"NuGet{Component.ComponentPathDivider}Search{Component.ComponentPathDivider}{region}{Component.ComponentPathDivider}{subRegion}"; + return true; + } + + protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) + { + affectedComponentStatus = ComponentStatus.Degraded; + return true; + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs new file mode 100644 index 000000000..9455e6671 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -0,0 +1,29 @@ +using System.Text.RegularExpressions; + +namespace StatusAggregator.Incidents.Parse +{ + public class OutdatedSearchServiceInstanceIncidentParser : EnvironmentIncidentParser + { + private const string SubtitleRegEx = "A search service instance is using an outdated index!"; + + private readonly string _environment; + + public OutdatedSearchServiceInstanceIncidentParser(string environment) + : base(SubtitleRegEx, environment) + { + _environment = environment; + } + + protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) + { + affectedComponentPath = $"NuGet{Component.ComponentPathDivider}Package Publishing"; + return true; + } + + protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) + { + affectedComponentStatus = ComponentStatus.Degraded; + return true; + } + } +} diff --git a/src/StatusAggregator/ParsedIncident.cs b/src/StatusAggregator/Incidents/Parse/ParsedIncident.cs similarity index 94% rename from src/StatusAggregator/ParsedIncident.cs rename to src/StatusAggregator/Incidents/Parse/ParsedIncident.cs index 7c1bee196..88aee170a 100644 --- a/src/StatusAggregator/ParsedIncident.cs +++ b/src/StatusAggregator/Incidents/Parse/ParsedIncident.cs @@ -1,6 +1,6 @@ using System; -namespace StatusAggregator +namespace StatusAggregator.Incidents.Parse { public class ParsedIncident { diff --git a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs new file mode 100644 index 000000000..60f6a19f2 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs @@ -0,0 +1,30 @@ +using System; +using System.Text.RegularExpressions; + +namespace StatusAggregator.Incidents.Parse +{ + public class ValidationDurationIncidentParser : EnvironmentIncidentParser + { + private const string SubtitleRegEx = "Too many packages are stuck in the \"Validating\" state!"; + + private readonly string _environment; + + public ValidationDurationIncidentParser(string environment) + : base(SubtitleRegEx, environment) + { + _environment = environment; + } + + protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) + { + affectedComponentPath = $"NuGet{Component.ComponentPathDivider}Package Publishing"; + return true; + } + + protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) + { + affectedComponentStatus = ComponentStatus.Degraded; + return true; + } + } +} diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 1b3ebfd2a..9f89ff3c6 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -4,18 +4,18 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.IO; using System.Linq; -using System.Net; using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.Table; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using NuGet.Jobs; using NuGet.Services.KeyVault; +using StatusAggregator.Incidents; +using StatusAggregator.Incidents.Parse; +using StatusAggregator.Table; namespace StatusAggregator { @@ -28,12 +28,12 @@ public class Job : JobBase private static TimeSpan EventVisibilityPeriod = TimeSpan.FromDays(7); private CloudBlobContainer _container; - private CloudTable _table; - - private Uri _incidentApiBaseUri; - private X509Certificate2 _incidentApiCertificate; - private string _incidentApiRoutingId; - private string _incidentApiEnvironment; + private ITableWrapper _table; + + private ICursor _cursor; + + private IIncidentCollector _incidentCollector; + private IEnumerable _incidentParsers; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { @@ -44,13 +44,14 @@ public override void Init(IServiceContainer serviceContainer, IDictionary() - .AsQueryable() .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); foreach (var activeIncidentEntity in activeIncidentEntities) { - var activeIncident = await GetIncident(activeIncidentEntity.IncidentApiId); + var activeIncident = await _incidentCollector.GetIncident(activeIncidentEntity.IncidentApiId); activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; - var incidentOperation = TableOperation.InsertOrReplace(activeIncidentEntity); - await _table.ExecuteAsync(incidentOperation); + await _table.InsertOrReplaceAsync(activeIncidentEntity); } // Fetch any new incidents. - var lastCursor = GetCursor(); - var nextCursor = DateTime.MinValue; + var lastCursor = _cursor.Get(); Console.WriteLine($"Read cursor at {lastCursor.ToString("o")}"); - var incidentParsers = new IIncidentParser[] { new ValidationDurationIncidentParser("PROD"), new OutdatedSearchServiceInstanceIncidentParser("PROD") }; + + var incidents = await _incidentCollector.GetRecentIncidents(lastCursor); + var nextCursor = incidents.Any() ? incidents.Max(i => i.CreateDate) : lastCursor; var parsedIncidents = new List(); - string query = lastCursor == DateTime.MinValue - ? GetIncidentApiIncidentListAllIncidentsQuery() - : GetIncidentApiIncidentListRecentIncidentsQuery(lastCursor); - var nextLink = GetIncidentApiUri(GetIncidentApiIncidentList(query)); - do + foreach (var incident in incidents) { - var incidents = await GetIncidentApiResponse(nextLink); - foreach (var incident in incidents.Incidents) + foreach (var incidentParser in _incidentParsers) { - if (incident.CreateDate <= lastCursor) - { - continue; - } - - if (incident.CreateDate > nextCursor) - { - nextCursor = incident.CreateDate; - } - - foreach (var incidentParser in incidentParsers) + if (incidentParser.TryParseIncident(incident, out var parsedIncident)) { - if (incidentParser.TryParseIncident(incident, out var parsedIncident)) - { - Console.WriteLine($"Found {parsedIncident.Id} affecting {parsedIncident.AffectedComponentPath} with status {parsedIncident.AffectedComponentStatus} from {parsedIncident.CreationTime} to {parsedIncident.MitigationTime}"); - parsedIncidents.Add(parsedIncident); - } + parsedIncidents.Add(parsedIncident); } } - - nextLink = incidents.NextLink; - } while (nextLink != null); + } // Close any active events that no longer have any active incidents. var creationTimesPerPath = parsedIncidents @@ -161,7 +142,6 @@ private async Task AggregateData() // Find an event to attach this incident to var possibleEvents = _table .CreateQuery() - .AsQueryable() .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && // The incident and the event must affect the same component @@ -200,24 +180,30 @@ private async Task AggregateData() { var eventEntity = new EventEntity(incidentEntity); Console.WriteLine($"Could not find existing event to attach {parsedIncident.Id} to, creating new event {eventEntity.RowKey}"); - var eventOperation = TableOperation.InsertOrReplace(eventEntity); - await _table.ExecuteAsync(eventOperation); + await _table.InsertOrReplaceAsync(eventEntity); } - var incidentOperation = TableOperation.InsertOrReplace(incidentEntity); - await _table.ExecuteAsync(incidentOperation); + await _table.InsertOrReplaceAsync(incidentEntity); } } // Update the cursor to signify that we've fetched all incidents thus far. if (nextCursor > lastCursor) { - var cursorEntity = new CursorEntity(nextCursor); - var operation = TableOperation.InsertOrReplace(cursorEntity); - await _table.ExecuteAsync(operation); + await _cursor.Set(nextCursor); } } + private IEnumerable GetIncidentParsers() + { + return new IIncidentParser[] + { + new ValidationDurationIncidentParser("PROD"), + new OutdatedRegionalSearchServiceInstanceIncidentParser("PROD"), + new OutdatedSearchServiceInstanceIncidentParser("PROD") + }; + } + private static readonly JsonSerializerSettings _statusBlobJsonSerializerSettings = new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc, @@ -235,7 +221,7 @@ private async Task ExportData() { var componentPathParts = activeEvent.AffectedComponentPath.Split(Component.ComponentPathDivider); - var currentComponent = new Component("", "", new[] { rootComponent }); + IComponent currentComponent = new TreeComponent("", "", new[] { rootComponent }); foreach (var componentPathPart in componentPathParts) { currentComponent = currentComponent.SubComponents.FirstOrDefault(c => c.Name == componentPathPart); @@ -256,7 +242,6 @@ private async Task ExportData() var recentEvents = _table .CreateQuery() - .AsQueryable() .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && (e.IsActive || (e.EndTime >= DateTime.Now - EventVisibilityPeriod))) @@ -276,59 +261,59 @@ private async Task ExportData() await blob.UploadTextAsync(statusJson); } - private Component SetupRootComponent() + private IComponent SetupRootComponent() { - return new Component( + return new TreeComponent( "NuGet", "", - new[] + new IComponent[] { - new Component( + new PrimarySecondaryComponent( "NuGet.org", "Browsing the Gallery website", new[] { - new Component("USNC", "Primary region"), - new Component("USSC", "Backup region") + new TreeComponent("USNC", "Primary region"), + new TreeComponent("USSC", "Backup region") }), - new Component( + new TreeComponent( "Restore", "Downloading and installing packages from NuGet", new[] { - new Component( + new TreeComponent( "V3", "Restore using the V3 API", new[] { - new Component("Global", "V3 restore for users outside of China"), - new Component("China", "V3 restore for users inside China") + new TreeComponent("Global", "V3 restore for users outside of China"), + new TreeComponent("China", "V3 restore for users inside China") }), - new Component("V2", "Restore using the V2 API") + new TreeComponent("V2", "Restore using the V2 API") }), - new Component( + new TreeComponent( "Search", "Searching for new and existing packages in Visual Studio or the Gallery website", new[] { - new Component( + new PrimarySecondaryComponent( "Global", "Search for packages outside Asia", new[] { - new Component("USNC", "Primary region"), - new Component("USSC", "Backup region") + new TreeComponent("USNC", "Primary region"), + new TreeComponent("USSC", "Backup region") }), - new Component( + new PrimarySecondaryComponent( "Asia", "Search for packages inside Asia", new[] { - new Component("EA", "Primary region"), - new Component("SEA", "Backup region") + new TreeComponent("EA", "Primary region"), + new TreeComponent("SEA", "Backup region") }) }), - new Component("Package Publishing", "Uploading new packages to NuGet.org") + new TreeComponent("Package Publishing", "Uploading new packages to NuGet.org") }); } @@ -367,14 +352,12 @@ private async Task UpdateEventAndCheckForClosure(EventEntity eventEntity, if (GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { var messageEntity = new MessageEntity(eventEntity, mitigationTime, "NO LONGER IMPACTED"); - var messageOperation = TableOperation.InsertOrReplace(messageEntity); - await _table.ExecuteAsync(messageOperation); + await _table.InsertOrReplaceAsync(messageEntity); } // Update the event eventEntity.EndTime = mitigationTime; - var eventOperation = TableOperation.InsertOrReplace(eventEntity); - await _table.ExecuteAsync(eventOperation); + await _table.InsertOrReplaceAsync(eventEntity); } else { @@ -389,8 +372,7 @@ private async Task CreateMessageForEventStartIfTimeHasPassed(EventEntity eventEn if (currentTime > eventEntity.StartTime + EventStartDelay) { var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "WE ARE IMPACTED"); - var messageOperation = TableOperation.InsertOrReplace(messageEntity); - await _table.ExecuteAsync(messageOperation); + await _table.InsertOrReplaceAsync(messageEntity); } } @@ -398,7 +380,6 @@ private IQueryable GetActiveEvents() { return _table .CreateQuery() - .AsQueryable() .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && e.IsActive); } @@ -406,7 +387,6 @@ private IQueryable GetIncidentsLinkedToEvent(EventEntity eventEn { return _table .CreateQuery() - .AsQueryable() .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsLinkedToEvent && @@ -417,77 +397,9 @@ private IQueryable GetMessagesLinkedToEvent(EventEntity eventEnti { return _table .CreateQuery() - .AsQueryable() .Where(m => m.PartitionKey == MessageEntity.DefaultPartitionKey && m.EventRowKey == eventEntity.RowKey); } - - private DateTime GetCursor() - { - var query = new TableQuery() - .Where(TableQuery.GenerateFilterCondition( - nameof(ITableEntity.PartitionKey), - QueryComparisons.Equal, - CursorEntity.DefaultPartitionKey)); - - var cursors = _table.ExecuteQuery(query).ToArray(); - return cursors.Any() - ? cursors.Max(c => c.Value) - : DateTime.MinValue; - } - - private const string IncidentApiIncidentsEndpoint = "incidents"; - private static readonly string IncidentApiIndividualIncidentQueryFormatString = $"{IncidentApiIncidentsEndpoint}({{0}})"; - private static readonly string IncidentApiIncidentListQueryFormatString = $"{IncidentApiIncidentsEndpoint}?{{0}}"; - - private static readonly JsonSerializerSettings _incidentApiJsonSerializerSettings = new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc }; - - private string GetIncidentApiIncidentList(string oDataQueryParameters) - { - return string.Format(IncidentApiIncidentListQueryFormatString, oDataQueryParameters); - } - - private string GetIncidentApiGetIncidentQuery(string id) - { - return string.Format(IncidentApiIndividualIncidentQueryFormatString, id); - } - - private string GetIncidentApiIncidentListAllIncidentsQuery() - { - return $"$filter=RoutingId eq '{_incidentApiRoutingId}'"; - } - - private string GetIncidentApiIncidentListRecentIncidentsQuery(DateTime cursor) - { - return $"$filter=RoutingId eq '{_incidentApiRoutingId}' and CreateDate gt datetime'{cursor.ToString("o")}'"; - } - - private Uri GetIncidentApiUri(string query) - { - return new Uri(_incidentApiBaseUri, query); - } - - private Task GetIncident(string id) - { - return GetIncidentApiResponse(GetIncidentApiGetIncidentQuery(id)); - } - - private Task GetIncidentApiResponse(string query) - { - return GetIncidentApiResponse(GetIncidentApiUri(query)); - } - - private async Task GetIncidentApiResponse(Uri uri) - { - var request = WebRequest.CreateHttp(uri); - request.ClientCertificates.Add(_incidentApiCertificate); - var response = await request.GetResponseAsync(); - using (var reader = new StreamReader(response.GetResponseStream())) - { - var content = await reader.ReadToEndAsync(); - return JsonConvert.DeserializeObject(content, _incidentApiJsonSerializerSettings); - } - } } } diff --git a/src/StatusAggregator/Message.cs b/src/StatusAggregator/Message.cs index 04460920a..f69c6f130 100644 --- a/src/StatusAggregator/Message.cs +++ b/src/StatusAggregator/Message.cs @@ -1,4 +1,5 @@ -using System; +using StatusAggregator.Table; +using System; namespace StatusAggregator { diff --git a/src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs deleted file mode 100644 index f7aa76508..000000000 --- a/src/StatusAggregator/OutdatedSearchServiceInstanceIncidentParser.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace StatusAggregator -{ - public class OutdatedSearchServiceInstanceIncidentParser : EnvironmentIncidentParser - { - private const string SubtitleRegEx = "A search service instance is using an outdated index!"; - - private readonly string _environment; - - public OutdatedSearchServiceInstanceIncidentParser(string environment) - : base(SubtitleRegEx, environment) - { - _environment = environment; - } - } -} diff --git a/src/StatusAggregator/PrimarySecondaryComponent.cs b/src/StatusAggregator/PrimarySecondaryComponent.cs new file mode 100644 index 000000000..cff495fd6 --- /dev/null +++ b/src/StatusAggregator/PrimarySecondaryComponent.cs @@ -0,0 +1,90 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StatusAggregator +{ + public class PrimarySecondaryComponent : Component + { + public PrimarySecondaryComponent() + { + } + + public PrimarySecondaryComponent( + string name, + string description) + : base(name, description) + { + } + + public PrimarySecondaryComponent( + string name, + string description, + IEnumerable subComponents) + : base(name, description, subComponents) + { + } + + public PrimarySecondaryComponent( + string name, + string description, + ComponentStatus status) + : this(name, description) + { + _status = status; + } + + public PrimarySecondaryComponent( + string name, + string description, + ComponentStatus status, + IEnumerable subComponents) + : this(name, description, subComponents) + { + _status = status; + } + + private ComponentStatus? _status = null; + public override ComponentStatus Status + { + get + { + if (_status.HasValue) + { + return _status.Value; + } + + if (!SubComponents.Any()) + { + return ComponentStatus.Up; + } + + // Iterate through the list of subcomponents in order. + var isFirst = true; + foreach (var subComponent in SubComponents) + { + if (subComponent.Status == ComponentStatus.Up) + { + // If the first component is up, the status is up. + // If any child component is up, the status is degraded. + return isFirst ? ComponentStatus.Up : ComponentStatus.Degraded; + } + + // If any component is degraded, the status is degraded. + if (subComponent.Status == ComponentStatus.Degraded) + { + return ComponentStatus.Degraded; + } + + isFirst = false; + } + + // If all components are down, the status is down. + return ComponentStatus.Down; + } + set + { + _status = value; + } + } + } +} diff --git a/src/StatusAggregator/Status.cs b/src/StatusAggregator/Status.cs index b7eab10ee..588312b04 100644 --- a/src/StatusAggregator/Status.cs +++ b/src/StatusAggregator/Status.cs @@ -9,7 +9,7 @@ public Status() { } - public Status(Component rootComponent, IEnumerable events) + public Status(IComponent rootComponent, IEnumerable events) { LastUpdated = DateTime.Now; RootComponent = rootComponent; @@ -17,7 +17,7 @@ public Status(Component rootComponent, IEnumerable events) } public DateTime LastUpdated { get; set; } - public Component RootComponent { get; set; } + public IComponent RootComponent { get; set; } public IEnumerable Events { get; set; } } } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index a0d0c4c67..ec3baee79 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -49,25 +49,35 @@ - - + + + + + + + + - - - - - - - + + + + + + + - - - + + + + + - + + + diff --git a/src/StatusAggregator/CursorTableEntity.cs b/src/StatusAggregator/Table/CursorEntity.cs similarity index 95% rename from src/StatusAggregator/CursorTableEntity.cs rename to src/StatusAggregator/Table/CursorEntity.cs index 9029ec9ee..d5b229f46 100644 --- a/src/StatusAggregator/CursorTableEntity.cs +++ b/src/StatusAggregator/Table/CursorEntity.cs @@ -4,7 +4,7 @@ using System; using Microsoft.WindowsAzure.Storage.Table; -namespace StatusAggregator +namespace StatusAggregator.Table { public class CursorEntity : TableEntity { diff --git a/src/StatusAggregator/EventEntity.cs b/src/StatusAggregator/Table/EventEntity.cs similarity index 97% rename from src/StatusAggregator/EventEntity.cs rename to src/StatusAggregator/Table/EventEntity.cs index 0fcca5a33..8160d1e01 100644 --- a/src/StatusAggregator/EventEntity.cs +++ b/src/StatusAggregator/Table/EventEntity.cs @@ -1,7 +1,7 @@ using System; using Microsoft.WindowsAzure.Storage.Table; -namespace StatusAggregator +namespace StatusAggregator.Table { public class EventEntity : TableEntity { diff --git a/src/StatusAggregator/Table/ITableWrapper.cs b/src/StatusAggregator/Table/ITableWrapper.cs new file mode 100644 index 000000000..7ffe369c2 --- /dev/null +++ b/src/StatusAggregator/Table/ITableWrapper.cs @@ -0,0 +1,16 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage.Table; + + +namespace StatusAggregator.Table +{ + public interface ITableWrapper + { + Task CreateIfNotExistsAsync(); + + Task InsertOrReplaceAsync(ITableEntity tableEntity); + + IQueryable CreateQuery() where T : ITableEntity, new(); + } +} diff --git a/src/StatusAggregator/IncidentEntity.cs b/src/StatusAggregator/Table/IncidentEntity.cs similarity index 95% rename from src/StatusAggregator/IncidentEntity.cs rename to src/StatusAggregator/Table/IncidentEntity.cs index 0d880a471..999a6c3d7 100644 --- a/src/StatusAggregator/IncidentEntity.cs +++ b/src/StatusAggregator/Table/IncidentEntity.cs @@ -1,7 +1,8 @@ using System; using Microsoft.WindowsAzure.Storage.Table; +using StatusAggregator.Incidents.Parse; -namespace StatusAggregator +namespace StatusAggregator.Table { public class IncidentEntity : TableEntity { diff --git a/src/StatusAggregator/MessageEntity.cs b/src/StatusAggregator/Table/MessageEntity.cs similarity index 96% rename from src/StatusAggregator/MessageEntity.cs rename to src/StatusAggregator/Table/MessageEntity.cs index c616a4b1c..158974cbf 100644 --- a/src/StatusAggregator/MessageEntity.cs +++ b/src/StatusAggregator/Table/MessageEntity.cs @@ -1,7 +1,7 @@ using System; using Microsoft.WindowsAzure.Storage.Table; -namespace StatusAggregator +namespace StatusAggregator.Table { public class MessageEntity : TableEntity { diff --git a/src/StatusAggregator/Table/TableWrapper.cs b/src/StatusAggregator/Table/TableWrapper.cs new file mode 100644 index 000000000..951b37254 --- /dev/null +++ b/src/StatusAggregator/Table/TableWrapper.cs @@ -0,0 +1,37 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Table; + +namespace StatusAggregator.Table +{ + public class TableWrapper : ITableWrapper + { + public TableWrapper(CloudStorageAccount storageAccount, string tableName) + { + var tableClient = storageAccount.CreateCloudTableClient(); + _table = tableClient.GetTableReference(tableName); + } + + private readonly CloudTable _table; + + public Task CreateIfNotExistsAsync() + { + return _table.CreateIfNotExistsAsync(); + } + + public Task InsertOrReplaceAsync(ITableEntity tableEntity) + { + var operation = TableOperation.InsertOrReplace(tableEntity); + return _table.ExecuteAsync(operation); + } + + public IQueryable CreateQuery() where T : ITableEntity, new() + { + return _table + .CreateQuery() + .AsQueryable(); + } + } +} diff --git a/src/StatusAggregator/TreeComponent.cs b/src/StatusAggregator/TreeComponent.cs new file mode 100644 index 000000000..5c41eb456 --- /dev/null +++ b/src/StatusAggregator/TreeComponent.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StatusAggregator +{ + public class TreeComponent : Component + { + public TreeComponent() + { + } + + public TreeComponent( + string name, + string description) + : base(name, description) + { + } + + public TreeComponent( + string name, + string description, + IEnumerable subComponents) + : base(name, description, subComponents) + { + } + + public TreeComponent( + string name, + string description, + ComponentStatus status) + : this(name, description) + { + _status = status; + } + + public TreeComponent( + string name, + string description, + ComponentStatus status, + IEnumerable subComponents) + : this(name, description, subComponents) + { + _status = status; + } + + private ComponentStatus? _status = null; + public override ComponentStatus Status + { + get + { + if (_status.HasValue) + { + return _status.Value; + } + + if (!SubComponents.Any()) + { + return ComponentStatus.Up; + } + + // If all subcomponents are up, we are up. + if (SubComponents.All(c => c.Status == ComponentStatus.Up)) + { + return ComponentStatus.Up; + } + + // If all subcomponents are down, we are down. + if (SubComponents.All(c => c.Status == ComponentStatus.Down)) + { + return ComponentStatus.Down; + } + + // Otherwise, we are degraded, because some subcomponents are degraded or down but not all. + return ComponentStatus.Degraded; + } + set + { + _status = value; + } + } + } +} diff --git a/src/StatusAggregator/ValidationDurationIncidentParser.cs b/src/StatusAggregator/ValidationDurationIncidentParser.cs deleted file mode 100644 index d871bd169..000000000 --- a/src/StatusAggregator/ValidationDurationIncidentParser.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Text.RegularExpressions; - -namespace StatusAggregator -{ - public class ValidationDurationIncidentParser : EnvironmentIncidentParser - { - private const string SubtitleRegEx = "Too many packages are stuck in the \"Validating\" state!"; - - private readonly string _environment; - - public ValidationDurationIncidentParser(string environment) - : base(SubtitleRegEx, environment) - { - _environment = environment; - } - } -} From 822f756969cf64c59d41406af2e02185a89adc7c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Tue, 3 Jul 2018 11:27:06 -0700 Subject: [PATCH 05/49] refactor classes add test project --- NuGet.Jobs.sln | 7 + src/StatusAggregator/Component.cs | 17 +- src/StatusAggregator/Components.cs | 83 ++++ src/StatusAggregator/EventUpdater.cs | 64 ++++ src/StatusAggregator/IComponent.cs | 6 +- src/StatusAggregator/IEventUpdater.cs | 11 + src/StatusAggregator/IIncidentFactory.cs | 11 + src/StatusAggregator/IIncidentUpdater.cs | 12 + src/StatusAggregator/IMessageUpdater.cs | 13 + src/StatusAggregator/IStatusExporter.cs | 9 + src/StatusAggregator/IStatusUpdater.cs | 9 + src/StatusAggregator/ISubComponent.cs | 11 + src/StatusAggregator/IncidentFactory.cs | 71 ++++ src/StatusAggregator/IncidentUpdater.cs | 70 ++++ .../Parse/AggregateIncidentParser.cs | 29 ++ .../Parse/IAggregateIncidentParser.cs | 9 + ...onalSearchServiceInstanceIncidentParser.cs | 2 +- ...atedSearchServiceInstanceIncidentParser.cs | 2 +- .../Parse/ValidationDurationIncidentParser.cs | 2 +- src/StatusAggregator/Job.cs | 355 ++---------------- src/StatusAggregator/MessageUpdater.cs | 44 +++ src/StatusAggregator/StatusAggregator.csproj | 19 + .../StatusContractResolver.cs | 64 ++++ src/StatusAggregator/StatusExporter.cs | 70 ++++ src/StatusAggregator/StatusUpdater.cs | 38 ++ src/StatusAggregator/SubComponent.cs | 32 ++ .../Table/TableWrapperExtensions.cs | 34 ++ .../Properties/AssemblyInfo.cs | 36 ++ .../StatusAggregator.Tests.csproj | 53 +++ 29 files changed, 832 insertions(+), 351 deletions(-) create mode 100644 src/StatusAggregator/Components.cs create mode 100644 src/StatusAggregator/EventUpdater.cs create mode 100644 src/StatusAggregator/IEventUpdater.cs create mode 100644 src/StatusAggregator/IIncidentFactory.cs create mode 100644 src/StatusAggregator/IIncidentUpdater.cs create mode 100644 src/StatusAggregator/IMessageUpdater.cs create mode 100644 src/StatusAggregator/IStatusExporter.cs create mode 100644 src/StatusAggregator/IStatusUpdater.cs create mode 100644 src/StatusAggregator/ISubComponent.cs create mode 100644 src/StatusAggregator/IncidentFactory.cs create mode 100644 src/StatusAggregator/IncidentUpdater.cs create mode 100644 src/StatusAggregator/Incidents/Parse/AggregateIncidentParser.cs create mode 100644 src/StatusAggregator/Incidents/Parse/IAggregateIncidentParser.cs create mode 100644 src/StatusAggregator/MessageUpdater.cs create mode 100644 src/StatusAggregator/StatusContractResolver.cs create mode 100644 src/StatusAggregator/StatusExporter.cs create mode 100644 src/StatusAggregator/StatusUpdater.cs create mode 100644 src/StatusAggregator/SubComponent.cs create mode 100644 src/StatusAggregator/Table/TableWrapperExtensions.cs create mode 100644 tests/StatusAggregator.Tests/Properties/AssemblyInfo.cs create mode 100644 tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln index d0d1573c5..aba2dacb4 100644 --- a/NuGet.Jobs.sln +++ b/NuGet.Jobs.sln @@ -137,6 +137,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate.Tests", "tests\NuGet.Services.Revalidate.Tests\NuGet.Services.Revalidate.Tests.csproj", "{19780DCB-B307-4254-B10C-4335FC784DEA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusAggregator.Tests", "tests\StatusAggregator.Tests\StatusAggregator.Tests.csproj", "{784F938D-4142-4C1C-B654-0978FEAD1731}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -357,6 +359,10 @@ Global {19780DCB-B307-4254-B10C-4335FC784DEA}.Debug|Any CPU.Build.0 = Debug|Any CPU {19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.ActiveCfg = Release|Any CPU {19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.Build.0 = Release|Any CPU + {784F938D-4142-4C1C-B654-0978FEAD1731}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {784F938D-4142-4C1C-B654-0978FEAD1731}.Debug|Any CPU.Build.0 = Debug|Any CPU + {784F938D-4142-4C1C-B654-0978FEAD1731}.Release|Any CPU.ActiveCfg = Release|Any CPU + {784F938D-4142-4C1C-B654-0978FEAD1731}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -415,6 +421,7 @@ Global {60152AB1-2EB4-4D44-B6D6-EEE24209A1F7} = {6A776396-02B1-475D-A104-26940ADB04AB} {1963909D-8BE3-4CB8-B57E-AB6A8CB22FED} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} {19780DCB-B307-4254-B10C-4335FC784DEA} = {6A776396-02B1-475D-A104-26940ADB04AB} + {784F938D-4142-4C1C-B654-0978FEAD1731} = {6A776396-02B1-475D-A104-26940ADB04AB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B} diff --git a/src/StatusAggregator/Component.cs b/src/StatusAggregator/Component.cs index 5aed4df72..acebad492 100644 --- a/src/StatusAggregator/Component.cs +++ b/src/StatusAggregator/Component.cs @@ -5,8 +5,6 @@ namespace StatusAggregator { public abstract class Component : IComponent { - public const char ComponentPathDivider = '/'; - public Component() { } @@ -17,7 +15,7 @@ public Component( { Name = name; Description = description; - SubComponents = Enumerable.Empty(); + SubComponents = Enumerable.Empty(); } public Component( @@ -26,20 +24,17 @@ public Component( IEnumerable subComponents) : this(name, description) { - SubComponents = subComponents; + SubComponents = subComponents.Select(s => new SubComponent(s, this)); } - public string Name { get; set; } - - public string Description { get; set; } - - public IEnumerable SubComponents { get; set; } - + public string Name { get; } + public string Description { get; } public abstract ComponentStatus Status { get; set; } + public IEnumerable SubComponents { get; } public static string ToRowKeySafeComponentPath(string componentPath) { - return componentPath.Replace(ComponentPathDivider, '_'); + return componentPath.Replace(SubComponent.ComponentPathDivider, '_'); } } } diff --git a/src/StatusAggregator/Components.cs b/src/StatusAggregator/Components.cs new file mode 100644 index 000000000..a444ee8ce --- /dev/null +++ b/src/StatusAggregator/Components.cs @@ -0,0 +1,83 @@ +using System.Linq; + +namespace StatusAggregator +{ + public static class Components + { + public static IComponent Root = new TreeComponent( + "NuGet", + "", + new IComponent[] + { + new PrimarySecondaryComponent( + "NuGet.org", + "Browsing the Gallery website", + new[] + { + new TreeComponent("USNC", "Primary region"), + new TreeComponent("USSC", "Backup region") + }), + new TreeComponent( + "Restore", + "Downloading and installing packages from NuGet", + new[] + { + new TreeComponent( + "V3", + "Restore using the V3 API", + new[] + { + new TreeComponent("Global", "V3 restore for users outside of China"), + new TreeComponent("China", "V3 restore for users inside China") + }), + new TreeComponent("V2", "Restore using the V2 API") + }), + new TreeComponent( + "Search", + "Searching for new and existing packages in Visual Studio or the Gallery website", + new[] + { + new PrimarySecondaryComponent( + "Global", + "Search for packages outside Asia", + new[] + { + new TreeComponent("USNC", "Primary region"), + new TreeComponent("USSC", "Backup region") + }), + new PrimarySecondaryComponent( + "Asia", + "Search for packages inside Asia", + new[] + { + new TreeComponent("EA", "Primary region"), + new TreeComponent("SEA", "Backup region") + }) + }), + new TreeComponent("Package Publishing", "Uploading new packages to NuGet.org") + }); + + public static ISubComponent Get(string path) + { + var componentPathParts = path.Split(SubComponent.ComponentPathDivider); + + if (componentPathParts.First() != Root.Name) + { + return null; + } + + ISubComponent currentComponent = new SubComponent(Root); + foreach (var componentPathPart in componentPathParts.Skip(1)) + { + currentComponent = currentComponent.SubComponents.FirstOrDefault(c => c.Name == componentPathPart); + + if (currentComponent == null) + { + break; + } + } + + return currentComponent; + } + } +} diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs new file mode 100644 index 000000000..8caaddff5 --- /dev/null +++ b/src/StatusAggregator/EventUpdater.cs @@ -0,0 +1,64 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public class EventUpdater : IEventUpdater + { + private static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); + + private readonly ITableWrapper _table; + private readonly IMessageUpdater _messageUpdater; + + public EventUpdater(ITableWrapper table, IMessageUpdater messageUpdater) + { + _table = table; + _messageUpdater = messageUpdater; + } + + public async Task UpdateEvent(EventEntity eventEntity, DateTime nextCreationTime) + { + if (!eventEntity.IsActive) + { + // Inactive events cannot be updated. + return false; + } + + var incidentsLinkedToEventToClose = _table.GetIncidentsLinkedToEvent(eventEntity); + + if (!incidentsLinkedToEventToClose.ToList().Any()) + { + // If an event has no linked incidents it must have been created manually and should not be closed automatically. + return false; + } + + var shouldClose = !incidentsLinkedToEventToClose + .Where(i => i.IsActive || i.MitigationTime > nextCreationTime - EventEndDelay) + .ToList() + .Any(); + + if (shouldClose) + { + Console.WriteLine($"Closing {eventEntity.RowKey} because its incidents are inactive and too old"); + var mitigationTime = incidentsLinkedToEventToClose + .ToList() + .Max(i => i.MitigationTime ?? DateTime.MinValue); + eventEntity.EndTime = mitigationTime; + + await _messageUpdater.CreateMessageForEventStart(eventEntity, mitigationTime); + await _messageUpdater.CreateMessageForEventEnd(eventEntity); + + // Update the event + await _table.InsertOrReplaceAsync(eventEntity); + } + else + { + await _messageUpdater.CreateMessageForEventStart(eventEntity, nextCreationTime); + } + + return shouldClose; + } + } +} diff --git a/src/StatusAggregator/IComponent.cs b/src/StatusAggregator/IComponent.cs index a592305ee..0991b33fc 100644 --- a/src/StatusAggregator/IComponent.cs +++ b/src/StatusAggregator/IComponent.cs @@ -4,9 +4,9 @@ namespace StatusAggregator { public interface IComponent { - string Name { get; set; } - string Description { get; set; } + string Name { get; } + string Description { get; } ComponentStatus Status { get; set; } - IEnumerable SubComponents { get; set; } + IEnumerable SubComponents { get; } } } diff --git a/src/StatusAggregator/IEventUpdater.cs b/src/StatusAggregator/IEventUpdater.cs new file mode 100644 index 000000000..b4920121f --- /dev/null +++ b/src/StatusAggregator/IEventUpdater.cs @@ -0,0 +1,11 @@ +using System; +using System.Threading.Tasks; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public interface IEventUpdater + { + Task UpdateEvent(EventEntity eventEntity, DateTime nextCreationTime); + } +} diff --git a/src/StatusAggregator/IIncidentFactory.cs b/src/StatusAggregator/IIncidentFactory.cs new file mode 100644 index 000000000..77d8b0159 --- /dev/null +++ b/src/StatusAggregator/IIncidentFactory.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; +using StatusAggregator.Incidents.Parse; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public interface IIncidentFactory + { + Task CreateIncident(ParsedIncident parsedIncident); + } +} diff --git a/src/StatusAggregator/IIncidentUpdater.cs b/src/StatusAggregator/IIncidentUpdater.cs new file mode 100644 index 000000000..b405ffaf2 --- /dev/null +++ b/src/StatusAggregator/IIncidentUpdater.cs @@ -0,0 +1,12 @@ +using System; +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public interface IIncidentUpdater + { + Task RefreshExistingIncidents(); + Task FetchNewIncidents(DateTime cursor); + Task UpdateActiveEvents(DateTime cursor); + } +} diff --git a/src/StatusAggregator/IMessageUpdater.cs b/src/StatusAggregator/IMessageUpdater.cs new file mode 100644 index 000000000..2db6d4c5f --- /dev/null +++ b/src/StatusAggregator/IMessageUpdater.cs @@ -0,0 +1,13 @@ +using System; +using System.Threading.Tasks; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public interface IMessageUpdater + { + Task CreateMessageForEventStart(EventEntity eventEntity, DateTime nextCreationTime); + + Task CreateMessageForEventEnd(EventEntity eventEntity); + } +} diff --git a/src/StatusAggregator/IStatusExporter.cs b/src/StatusAggregator/IStatusExporter.cs new file mode 100644 index 000000000..24fb36f67 --- /dev/null +++ b/src/StatusAggregator/IStatusExporter.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public interface IStatusExporter + { + Task Export(); + } +} diff --git a/src/StatusAggregator/IStatusUpdater.cs b/src/StatusAggregator/IStatusUpdater.cs new file mode 100644 index 000000000..0a4c227c8 --- /dev/null +++ b/src/StatusAggregator/IStatusUpdater.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public interface IStatusUpdater + { + Task Update(); + } +} diff --git a/src/StatusAggregator/ISubComponent.cs b/src/StatusAggregator/ISubComponent.cs new file mode 100644 index 000000000..817b85d9e --- /dev/null +++ b/src/StatusAggregator/ISubComponent.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; + +namespace StatusAggregator +{ + public interface ISubComponent : IComponent + { + [JsonIgnore] + IComponent Parent { get; } + string Path { get; } + } +} diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs new file mode 100644 index 000000000..3213e04d1 --- /dev/null +++ b/src/StatusAggregator/IncidentFactory.cs @@ -0,0 +1,71 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using StatusAggregator.Incidents.Parse; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public class IncidentFactory : IIncidentFactory + { + private readonly ITableWrapper _table; + + private readonly IEventUpdater _eventUpdater; + + public IncidentFactory(ITableWrapper table, IEventUpdater eventUpdater) + { + _table = table; + _eventUpdater = eventUpdater; + } + + public async Task CreateIncident(ParsedIncident parsedIncident) + { + Console.WriteLine($"Attempting to save {parsedIncident.Id}"); + var incidentEntity = new IncidentEntity(parsedIncident); + + // Find an event to attach this incident to + var possibleEvents = _table + .CreateQuery() + .Where(e => + e.PartitionKey == EventEntity.DefaultPartitionKey && + // The incident and the event must affect the same component + e.AffectedComponentPath == parsedIncident.AffectedComponentPath && + // The event must begin before or at the same time as the incident + e.StartTime <= parsedIncident.CreationTime && + // The event must be active or the event must end after this incident begins + (e.IsActive || (e.EndTime >= parsedIncident.CreationTime))) + .ToList(); + + Console.WriteLine($"Found {possibleEvents.Count()} possible events to link {parsedIncident.Id} to"); + + foreach (var possibleEvent in possibleEvents) + { + if (!_table.GetIncidentsLinkedToEvent(possibleEvent).ToList().Any()) + { + Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because it is not linked to any incidents"); + continue; + } + + if (await _eventUpdater.UpdateEvent(possibleEvent, parsedIncident.CreationTime)) + { + Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because its incidents are inactive and too old"); + continue; + } + + Console.WriteLine($"Linking {parsedIncident.Id} to {possibleEvent.RowKey}"); + incidentEntity.EventRowKey = possibleEvents.First().RowKey; + break; + } + + if (string.IsNullOrEmpty(incidentEntity.EventRowKey)) + { + var eventEntity = new EventEntity(incidentEntity); + Console.WriteLine($"Could not find existing event to attach {parsedIncident.Id} to, creating new event {eventEntity.RowKey}"); + await _table.InsertOrReplaceAsync(eventEntity); + } + + await _table.InsertOrReplaceAsync(incidentEntity); + return incidentEntity; + } + } +} diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs new file mode 100644 index 000000000..383579ac1 --- /dev/null +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -0,0 +1,70 @@ +using StatusAggregator.Incidents; +using StatusAggregator.Incidents.Parse; +using StatusAggregator.Table; +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public class IncidentUpdater : IIncidentUpdater + { + private readonly ITableWrapper _table; + + private readonly IEventUpdater _eventUpdater; + + private readonly IAggregateIncidentParser _aggregateIncidentParser; + private readonly IIncidentCollector _incidentCollector; + private readonly IIncidentFactory _incidentFactory; + + public IncidentUpdater( + ITableWrapper table, + IEventUpdater eventUpdater, + IIncidentCollector incidentCollector, + IAggregateIncidentParser aggregateIncidentParser, + IIncidentFactory incidentFactory) + { + _table = table; + _eventUpdater = eventUpdater; + _incidentCollector = incidentCollector; + _aggregateIncidentParser = aggregateIncidentParser; + _incidentFactory = incidentFactory; + } + + public async Task RefreshExistingIncidents() + { + var activeIncidentEntities = _table + .CreateQuery() + .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); + + foreach (var activeIncidentEntity in activeIncidentEntities) + { + var activeIncident = await _incidentCollector.GetIncident(activeIncidentEntity.IncidentApiId); + activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; + await _table.InsertOrReplaceAsync(activeIncidentEntity); + } + } + + public async Task FetchNewIncidents(DateTime cursor) + { + var incidents = await _incidentCollector.GetRecentIncidents(cursor); + + var parsedIncidents = incidents.SelectMany(i => _aggregateIncidentParser.ParseIncident(i)); + foreach (var parsedIncident in parsedIncidents.OrderBy(i => i.CreationTime)) + { + await _incidentFactory.CreateIncident(parsedIncident); + } + + return incidents.Any() ? incidents.Max(i => i.CreateDate) : (DateTime?)null; + } + + public async Task UpdateActiveEvents(DateTime cursor) + { + var eventsToCheckClosure = _table.GetActiveEvents(); + foreach (var eventToCheckClosure in eventsToCheckClosure) + { + await _eventUpdater.UpdateEvent(eventToCheckClosure, cursor); + } + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/AggregateIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/AggregateIncidentParser.cs new file mode 100644 index 000000000..11ea84175 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/AggregateIncidentParser.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; + +namespace StatusAggregator.Incidents.Parse +{ + public class AggregateIncidentParser : IAggregateIncidentParser + { + private readonly IEnumerable _incidentParsers; + + public AggregateIncidentParser(IEnumerable incidentParsers) + { + _incidentParsers = incidentParsers; + } + + public IEnumerable ParseIncident(Incident incident) + { + var parsedIncidents = new List(); + foreach (var incidentParser in _incidentParsers) + { + if (incidentParser.TryParseIncident(incident, out var parsedIncident)) + { + parsedIncidents.Add(parsedIncident); + } + } + + return parsedIncidents; + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/IAggregateIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/IAggregateIncidentParser.cs new file mode 100644 index 000000000..0ef50ddaf --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/IAggregateIncidentParser.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace StatusAggregator.Incidents.Parse +{ + public interface IAggregateIncidentParser + { + IEnumerable ParseIncident(Incident incident); + } +} diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs index bb44dc894..b1d551621 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs @@ -66,7 +66,7 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo return false; } - affectedComponentPath = $"NuGet{Component.ComponentPathDivider}Search{Component.ComponentPathDivider}{region}{Component.ComponentPathDivider}{subRegion}"; + affectedComponentPath = $"NuGet{SubComponent.ComponentPathDivider}Search{SubComponent.ComponentPathDivider}{region}{SubComponent.ComponentPathDivider}{subRegion}"; return true; } diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index 9455e6671..c672c94a9 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -16,7 +16,7 @@ public OutdatedSearchServiceInstanceIncidentParser(string environment) protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = $"NuGet{Component.ComponentPathDivider}Package Publishing"; + affectedComponentPath = $"NuGet{SubComponent.ComponentPathDivider}Package Publishing"; return true; } diff --git a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs index 60f6a19f2..bb579d406 100644 --- a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs @@ -17,7 +17,7 @@ public ValidationDurationIncidentParser(string environment) protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = $"NuGet{Component.ComponentPathDivider}Package Publishing"; + affectedComponentPath = $"NuGet{SubComponent.ComponentPathDivider}Package Publishing"; return true; } diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 9f89ff3c6..3f95a3bd5 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -4,13 +4,10 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Linq; using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; using NuGet.Jobs; using NuGet.Services.KeyVault; using StatusAggregator.Incidents; @@ -21,33 +18,25 @@ namespace StatusAggregator { public class Job : JobBase { - private const string StatusBlobName = "status.json"; - - private static TimeSpan EventStartDelay = TimeSpan.FromMinutes(15); - private static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); - private static TimeSpan EventVisibilityPeriod = TimeSpan.FromDays(7); - private CloudBlobContainer _container; private ITableWrapper _table; - private ICursor _cursor; - - private IIncidentCollector _incidentCollector; - private IEnumerable _incidentParsers; + private IStatusUpdater _statusUpdater; + private IStatusExporter _statusExporter; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount); var storageAccount = CloudStorageAccount.Parse(storageConnectionString); - var blobClient = storageAccount.CreateCloudBlobClient(); - var containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName); - _container = blobClient.GetContainerReference(containerName); - var tableName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName); _table = new TableWrapper(storageAccount, tableName); - _cursor = new Cursor(_table); + var cursor = new Cursor(_table); + + var messageUpdater = new MessageUpdater(_table); + var eventUpdater = new EventUpdater(_table, messageUpdater); + var incidentFactory = new IncidentFactory(_table, eventUpdater); var incidentApiBaseUri = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)); var incidentApiRoutingId = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiRoutingId); @@ -68,130 +57,26 @@ public override void Init(IServiceContainer serviceContainer, IDictionary() - .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); - - foreach (var activeIncidentEntity in activeIncidentEntities) - { - var activeIncident = await _incidentCollector.GetIncident(activeIncidentEntity.IncidentApiId); - activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; - await _table.InsertOrReplaceAsync(activeIncidentEntity); - } - - // Fetch any new incidents. - var lastCursor = _cursor.Get(); - Console.WriteLine($"Read cursor at {lastCursor.ToString("o")}"); - - var incidents = await _incidentCollector.GetRecentIncidents(lastCursor); - var nextCursor = incidents.Any() ? incidents.Max(i => i.CreateDate) : lastCursor; - var parsedIncidents = new List(); - foreach (var incident in incidents) - { - foreach (var incidentParser in _incidentParsers) - { - if (incidentParser.TryParseIncident(incident, out var parsedIncident)) - { - parsedIncidents.Add(parsedIncident); - } - } - } - - // Close any active events that no longer have any active incidents. - var creationTimesPerPath = parsedIncidents - .GroupBy(i => i.AffectedComponentPath) - .ToDictionary(g => g.Key, g => g.Min(i => i.CreationTime)); - - DateTime minCreationTime = creationTimesPerPath.Any() - ? creationTimesPerPath.Min(t => t.Value) - : new[] { nextCursor, DateTime.UtcNow }.Max(); - - var eventsToCheckClosure = GetActiveEvents(); - foreach (var eventToCheckClosure in eventsToCheckClosure) - { - var nextCreationTime = creationTimesPerPath.ContainsKey(eventToCheckClosure.AffectedComponentPath) - ? creationTimesPerPath[eventToCheckClosure.AffectedComponentPath] - : minCreationTime; - - await UpdateEventAndCheckForClosure(eventToCheckClosure, nextCreationTime); - } - - // Aggregate the new incidents and create new events if necessary. - if (parsedIncidents.Any()) - { - foreach (var parsedIncident in parsedIncidents.OrderBy(i => i.CreationTime)) - { - Console.WriteLine($"Attempting to save {parsedIncident.Id}"); - var incidentEntity = new IncidentEntity(parsedIncident); - - // Find an event to attach this incident to - var possibleEvents = _table - .CreateQuery() - .Where(e => - e.PartitionKey == EventEntity.DefaultPartitionKey && - // The incident and the event must affect the same component - e.AffectedComponentPath == parsedIncident.AffectedComponentPath && - // The event must begin before or at the same time as the incident - e.StartTime <= parsedIncident.CreationTime && - // The event must be active or the event must end after this incident begins - (e.IsActive || (e.EndTime >= parsedIncident.CreationTime))) - .ToList(); - - Console.WriteLine($"Found {possibleEvents.Count()} possible events to link {parsedIncident.Id} to"); - - if (possibleEvents.Any()) - { - foreach (var possibleEvent in possibleEvents) - { - if (!GetIncidentsLinkedToEvent(possibleEvent).ToList().Any()) - { - Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because it is not linked to any incidents"); - continue; - } - - if (await UpdateEventAndCheckForClosure(possibleEvent, parsedIncident.CreationTime)) - { - Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because its incidents are inactive and too old"); - continue; - } - - Console.WriteLine($"Linking {parsedIncident.Id} to {possibleEvent.RowKey}"); - incidentEntity.EventRowKey = possibleEvents.First().RowKey; - break; - } - } - - if (string.IsNullOrEmpty(incidentEntity.EventRowKey)) - { - var eventEntity = new EventEntity(incidentEntity); - Console.WriteLine($"Could not find existing event to attach {parsedIncident.Id} to, creating new event {eventEntity.RowKey}"); - await _table.InsertOrReplaceAsync(eventEntity); - } - - await _table.InsertOrReplaceAsync(incidentEntity); - } - } - - // Update the cursor to signify that we've fetched all incidents thus far. - if (nextCursor > lastCursor) - { - await _cursor.Set(nextCursor); - } + await _statusUpdater.Update(); + await _statusExporter.Export(); } private IEnumerable GetIncidentParsers() @@ -203,203 +88,5 @@ private IEnumerable GetIncidentParsers() new OutdatedSearchServiceInstanceIncidentParser("PROD") }; } - - private static readonly JsonSerializerSettings _statusBlobJsonSerializerSettings = new JsonSerializerSettings() - { - DateTimeZoneHandling = DateTimeZoneHandling.Utc, - Converters = new List() { new StringEnumConverter() } - }; - - private async Task ExportData() - { - await _container.CreateIfNotExistsAsync(); - - var rootComponent = SetupRootComponent(); - - var activeEvents = GetActiveEvents(); - foreach (var activeEvent in activeEvents) - { - var componentPathParts = activeEvent.AffectedComponentPath.Split(Component.ComponentPathDivider); - - IComponent currentComponent = new TreeComponent("", "", new[] { rootComponent }); - foreach (var componentPathPart in componentPathParts) - { - currentComponent = currentComponent.SubComponents.FirstOrDefault(c => c.Name == componentPathPart); - - if (currentComponent == null) - { - break; - } - } - - if (currentComponent == null) - { - continue; - } - - currentComponent.Status = activeEvent.AffectedComponentStatus; - } - - var recentEvents = _table - .CreateQuery() - .Where(e => - e.PartitionKey == EventEntity.DefaultPartitionKey && - (e.IsActive || (e.EndTime >= DateTime.Now - EventVisibilityPeriod))) - .ToList() - .Select(e => - { - var messages = GetMessagesLinkedToEvent(e) - .ToList() - .Select(m => new Message(m)); - return new Event(e, messages); - }); - - var status = new Status(rootComponent, recentEvents); - var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); - - var blob = _container.GetBlockBlobReference(StatusBlobName); - await blob.UploadTextAsync(statusJson); - } - - private IComponent SetupRootComponent() - { - return new TreeComponent( - "NuGet", - "", - new IComponent[] - { - new PrimarySecondaryComponent( - "NuGet.org", - "Browsing the Gallery website", - new[] - { - new TreeComponent("USNC", "Primary region"), - new TreeComponent("USSC", "Backup region") - }), - new TreeComponent( - "Restore", - "Downloading and installing packages from NuGet", - new[] - { - new TreeComponent( - "V3", - "Restore using the V3 API", - new[] - { - new TreeComponent("Global", "V3 restore for users outside of China"), - new TreeComponent("China", "V3 restore for users inside China") - }), - new TreeComponent("V2", "Restore using the V2 API") - }), - new TreeComponent( - "Search", - "Searching for new and existing packages in Visual Studio or the Gallery website", - new[] - { - new PrimarySecondaryComponent( - "Global", - "Search for packages outside Asia", - new[] - { - new TreeComponent("USNC", "Primary region"), - new TreeComponent("USSC", "Backup region") - }), - new PrimarySecondaryComponent( - "Asia", - "Search for packages inside Asia", - new[] - { - new TreeComponent("EA", "Primary region"), - new TreeComponent("SEA", "Backup region") - }) - }), - new TreeComponent("Package Publishing", "Uploading new packages to NuGet.org") - }); - } - - private async Task UpdateEventAndCheckForClosure(EventEntity eventEntity, DateTime nextCreationTime) - { - if (!eventEntity.IsActive) - { - // Inactive events have already been closed. - return false; - } - - var incidentsLinkedToEventToClose = GetIncidentsLinkedToEvent(eventEntity); - - if (!incidentsLinkedToEventToClose.ToList().Any()) - { - // If an event has no linked incidents it must have been created manually and should not be closed automatically. - return false; - } - - var shouldClose = !incidentsLinkedToEventToClose - .Where(i => i.IsActive || i.MitigationTime > nextCreationTime - EventEndDelay) - .ToList() - .Any(); - - if (shouldClose) - { - Console.WriteLine($"Closing {eventEntity.RowKey} because its incidents are inactive and too old"); - var mitigationTime = incidentsLinkedToEventToClose - .ToList() - .Max(i => i.MitigationTime ?? DateTime.MinValue); - - await CreateMessageForEventStartIfTimeHasPassed(eventEntity, mitigationTime); - - // Create a message to alert customers that the event is resolved. - // Only create a message if the event already has messages associated with it. - if (GetMessagesLinkedToEvent(eventEntity).ToList().Any()) - { - var messageEntity = new MessageEntity(eventEntity, mitigationTime, "NO LONGER IMPACTED"); - await _table.InsertOrReplaceAsync(messageEntity); - } - - // Update the event - eventEntity.EndTime = mitigationTime; - await _table.InsertOrReplaceAsync(eventEntity); - } - else - { - await CreateMessageForEventStartIfTimeHasPassed(eventEntity, nextCreationTime); - } - - return shouldClose; - } - - private async Task CreateMessageForEventStartIfTimeHasPassed(EventEntity eventEntity, DateTime currentTime) - { - if (currentTime > eventEntity.StartTime + EventStartDelay) - { - var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "WE ARE IMPACTED"); - await _table.InsertOrReplaceAsync(messageEntity); - } - } - - private IQueryable GetActiveEvents() - { - return _table - .CreateQuery() - .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && e.IsActive); - } - - private IQueryable GetIncidentsLinkedToEvent(EventEntity eventEntity) - { - return _table - .CreateQuery() - .Where(i => - i.PartitionKey == IncidentEntity.DefaultPartitionKey && - i.IsLinkedToEvent && - i.EventRowKey == eventEntity.RowKey); - } - - private IQueryable GetMessagesLinkedToEvent(EventEntity eventEntity) - { - return _table - .CreateQuery() - .Where(m => - m.PartitionKey == MessageEntity.DefaultPartitionKey && - m.EventRowKey == eventEntity.RowKey); - } } } diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs new file mode 100644 index 000000000..143e03e06 --- /dev/null +++ b/src/StatusAggregator/MessageUpdater.cs @@ -0,0 +1,44 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public class MessageUpdater : IMessageUpdater + { + private static TimeSpan EventStartDelay = TimeSpan.FromMinutes(15); + + private ITableWrapper _table; + + public MessageUpdater(ITableWrapper table) + { + _table = table; + } + + public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime nextCreationTime) + { + // Enough time must have passed before we create a start message for an event. + if (nextCreationTime > eventEntity.StartTime + EventStartDelay) + { + var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "WE ARE IMPACTED"); + await _table.InsertOrReplaceAsync(messageEntity); + } + } + + public async Task CreateMessageForEventEnd(EventEntity eventEntity) + { + if (!eventEntity.EndTime.HasValue) + { + throw new ArgumentException("Must pass in an event with an end time!", nameof(eventEntity)); + } + + // Only create a message if the event already has messages associated with it. + if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) + { + var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, "NO LONGER IMPACTED"); + await _table.InsertOrReplaceAsync(messageEntity); + } + } + } +} diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index ec3baee79..b7e62f794 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -48,13 +48,31 @@ + + + + + + + + + + + + + + + + + + @@ -77,6 +95,7 @@ + diff --git a/src/StatusAggregator/StatusContractResolver.cs b/src/StatusAggregator/StatusContractResolver.cs new file mode 100644 index 000000000..124ef791a --- /dev/null +++ b/src/StatusAggregator/StatusContractResolver.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Reflection; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace StatusAggregator +{ + public class StatusContractResolver : DefaultContractResolver + { + protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) + { + JsonProperty property = base.CreateProperty(member, memberSerialization); + + var propertyType = property.PropertyType; + + if (propertyType == typeof(string)) + { + property.ShouldSerialize = instance => !string.IsNullOrEmpty((string)instance); + } + + if (typeof(IEnumerable).IsAssignableFrom(propertyType)) + { + SetShouldSerializeForIEnumerable(property, member); + } + + return property; + } + + private void SetShouldSerializeForIEnumerable(JsonProperty property, MemberInfo member) + { + Func getValue; + + switch (member.MemberType) + { + case MemberTypes.Field: + getValue = instance => ((FieldInfo)member).GetValue(instance); + break; + case MemberTypes.Property: + getValue = instance => ((PropertyInfo)member).GetValue(instance); + break; + default: + return; + } + + property.ShouldSerialize = instance => + { + var value = (IEnumerable)getValue(instance); + + if (value == null) + { + return false; + } + + foreach (var obj in value) + { + return true; + } + + return false; + }; + } + } +} diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs new file mode 100644 index 000000000..0796a61db --- /dev/null +++ b/src/StatusAggregator/StatusExporter.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public class StatusExporter : IStatusExporter + { + private const string StatusBlobName = "status.json"; + private static TimeSpan EventVisibilityPeriod = TimeSpan.FromDays(7); + + private readonly CloudBlobContainer _container; + private readonly ITableWrapper _table; + + private static readonly JsonSerializerSettings _statusBlobJsonSerializerSettings = new JsonSerializerSettings() + { + ContractResolver = new StatusContractResolver(), + Converters = new List() { new StringEnumConverter() }, + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + NullValueHandling = NullValueHandling.Ignore + }; + + public StatusExporter(CloudBlobContainer container, ITableWrapper table) + { + _container = container; + _table = table; + } + + public async Task Export() + { + var activeEvents = _table.GetActiveEvents(); + foreach (var activeEvent in activeEvents) + { + var currentComponent = Components.Get(activeEvent.AffectedComponentPath); + + if (currentComponent == null) + { + continue; + } + + currentComponent.Status = activeEvent.AffectedComponentStatus; + } + + var recentEvents = _table + .CreateQuery() + .Where(e => + e.PartitionKey == EventEntity.DefaultPartitionKey && + (e.IsActive || (e.EndTime >= DateTime.Now - EventVisibilityPeriod))) + .ToList() + .Select(e => + { + var messages = _table.GetMessagesLinkedToEvent(e) + .ToList() + .Select(m => new Message(m)); + return new Event(e, messages); + }); + + var status = new Status(Components.Root, recentEvents); + var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); + + var blob = _container.GetBlockBlobReference(StatusBlobName); + await blob.UploadTextAsync(statusJson); + } + } +} diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs new file mode 100644 index 000000000..62b77b122 --- /dev/null +++ b/src/StatusAggregator/StatusUpdater.cs @@ -0,0 +1,38 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using StatusAggregator.Incidents; +using StatusAggregator.Incidents.Parse; +using StatusAggregator.Table; + +namespace StatusAggregator +{ + public class StatusUpdater : IStatusUpdater + { + private readonly IIncidentUpdater _incidentUpdater; + + private ICursor _cursor; + + public StatusUpdater( + ICursor cursor, + IIncidentUpdater incidentUpdater) + { + _cursor = cursor; + _incidentUpdater = incidentUpdater; + } + + public async Task Update() + { + var lastCursor = _cursor.Get(); + + await _incidentUpdater.RefreshExistingIncidents(); + var nextCursor = await _incidentUpdater.FetchNewIncidents(lastCursor); + await _incidentUpdater.UpdateActiveEvents(nextCursor ?? DateTime.UtcNow); + + if (nextCursor.HasValue) + { + await _cursor.Set(nextCursor.Value); + } + } + } +} diff --git a/src/StatusAggregator/SubComponent.cs b/src/StatusAggregator/SubComponent.cs new file mode 100644 index 000000000..42aa942d1 --- /dev/null +++ b/src/StatusAggregator/SubComponent.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; + +namespace StatusAggregator +{ + public class SubComponent : ISubComponent + { + public const char ComponentPathDivider = '/'; + + private readonly IComponent _component; + + public string Name => _component.Name; + public string Description => _component.Description; + public ComponentStatus Status { get => _component.Status; set => _component.Status = value; } + public IEnumerable SubComponents => _component.SubComponents; + public IComponent Parent { get; } + public string Path { get; } + + public SubComponent(IComponent component) + { + _component = component; + Parent = null; + Path = (_component as SubComponent)?.Path ?? _component.Name; + } + + public SubComponent(IComponent component, IComponent parent) + : this(component) + { + Parent = parent; + Path = parent.Name + ComponentPathDivider + Path; + } + } +} diff --git a/src/StatusAggregator/Table/TableWrapperExtensions.cs b/src/StatusAggregator/Table/TableWrapperExtensions.cs new file mode 100644 index 000000000..887f5ebd5 --- /dev/null +++ b/src/StatusAggregator/Table/TableWrapperExtensions.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StatusAggregator.Table +{ + public static class TableWrapperExtensions + { + public static IQueryable GetActiveEvents(this ITableWrapper table) + { + return table + .CreateQuery() + .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && e.IsActive); + } + + public static IQueryable GetIncidentsLinkedToEvent(this ITableWrapper table, EventEntity eventEntity) + { + return table + .CreateQuery() + .Where(i => + i.PartitionKey == IncidentEntity.DefaultPartitionKey && + i.IsLinkedToEvent && + i.EventRowKey == eventEntity.RowKey); + } + + public static IQueryable GetMessagesLinkedToEvent(this ITableWrapper table, EventEntity eventEntity) + { + return table + .CreateQuery() + .Where(m => + m.PartitionKey == MessageEntity.DefaultPartitionKey && + m.EventRowKey == eventEntity.RowKey); + } + } +} diff --git a/tests/StatusAggregator.Tests/Properties/AssemblyInfo.cs b/tests/StatusAggregator.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..429b1f7c4 --- /dev/null +++ b/tests/StatusAggregator.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("StatusAggregator.Tests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("StatusAggregator.Tests")] +[assembly: AssemblyCopyright("Copyright © 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("784f938d-4142-4c1c-b654-0978fead1731")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj new file mode 100644 index 000000000..713e30e03 --- /dev/null +++ b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj @@ -0,0 +1,53 @@ + + + + + Debug + AnyCPU + {784F938D-4142-4C1C-B654-0978FEAD1731} + Library + Properties + StatusAggregator.Tests + StatusAggregator.Tests + v4.5.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + {d357fdb5-bf19-41a5-82b0-14c8cec2a5eb} + StatusAggregator + + + + \ No newline at end of file From 7a96990efead44835e4438f66900648bdba26f47 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Thu, 12 Jul 2018 17:03:09 -0700 Subject: [PATCH 06/49] remove servercommon classes, add some tests --- src/StatusAggregator/Component.cs | 40 ----- src/StatusAggregator/ComponentStatus.cs | 9 -- src/StatusAggregator/Components.cs | 47 ++---- src/StatusAggregator/Event.cs | 31 ---- src/StatusAggregator/EventUpdater.cs | 12 +- src/StatusAggregator/IComponent.cs | 12 -- src/StatusAggregator/ISubComponent.cs | 11 -- .../Parse/EnvironmentIncidentParser.cs | 3 +- ...onalSearchServiceInstanceIncidentParser.cs | 3 +- ...atedSearchServiceInstanceIncidentParser.cs | 5 +- .../Incidents/Parse/ParsedIncident.cs | 3 +- .../Parse/ValidationDurationIncidentParser.cs | 6 +- src/StatusAggregator/Message.cs | 21 --- .../PrimarySecondaryComponent.cs | 90 ----------- src/StatusAggregator/Status.cs | 23 --- src/StatusAggregator/StatusAggregator.csproj | 14 +- src/StatusAggregator/StatusExporter.cs | 11 +- src/StatusAggregator/SubComponent.cs | 32 ---- src/StatusAggregator/Table/EventEntity.cs | 9 +- src/StatusAggregator/Table/IncidentEntity.cs | 3 +- src/StatusAggregator/Table/MessageEntity.cs | 6 + src/StatusAggregator/Table/TableUtility.cs | 12 ++ src/StatusAggregator/TreeComponent.cs | 82 ---------- .../EventUpdaterTests.cs | 140 ++++++++++++++++++ .../IncidentFactoryTests.cs | 78 ++++++++++ .../StatusAggregator.Tests.csproj | 31 +++- 26 files changed, 318 insertions(+), 416 deletions(-) delete mode 100644 src/StatusAggregator/Component.cs delete mode 100644 src/StatusAggregator/ComponentStatus.cs delete mode 100644 src/StatusAggregator/Event.cs delete mode 100644 src/StatusAggregator/IComponent.cs delete mode 100644 src/StatusAggregator/ISubComponent.cs delete mode 100644 src/StatusAggregator/Message.cs delete mode 100644 src/StatusAggregator/PrimarySecondaryComponent.cs delete mode 100644 src/StatusAggregator/Status.cs delete mode 100644 src/StatusAggregator/SubComponent.cs create mode 100644 src/StatusAggregator/Table/TableUtility.cs delete mode 100644 src/StatusAggregator/TreeComponent.cs create mode 100644 tests/StatusAggregator.Tests/EventUpdaterTests.cs create mode 100644 tests/StatusAggregator.Tests/IncidentFactoryTests.cs diff --git a/src/StatusAggregator/Component.cs b/src/StatusAggregator/Component.cs deleted file mode 100644 index acebad492..000000000 --- a/src/StatusAggregator/Component.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace StatusAggregator -{ - public abstract class Component : IComponent - { - public Component() - { - } - - public Component( - string name, - string description) - { - Name = name; - Description = description; - SubComponents = Enumerable.Empty(); - } - - public Component( - string name, - string description, - IEnumerable subComponents) - : this(name, description) - { - SubComponents = subComponents.Select(s => new SubComponent(s, this)); - } - - public string Name { get; } - public string Description { get; } - public abstract ComponentStatus Status { get; set; } - public IEnumerable SubComponents { get; } - - public static string ToRowKeySafeComponentPath(string componentPath) - { - return componentPath.Replace(SubComponent.ComponentPathDivider, '_'); - } - } -} diff --git a/src/StatusAggregator/ComponentStatus.cs b/src/StatusAggregator/ComponentStatus.cs deleted file mode 100644 index 686a2ddb4..000000000 --- a/src/StatusAggregator/ComponentStatus.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace StatusAggregator -{ - public enum ComponentStatus - { - Up, - Degraded, - Down - } -} diff --git a/src/StatusAggregator/Components.cs b/src/StatusAggregator/Components.cs index a444ee8ce..c39567096 100644 --- a/src/StatusAggregator/Components.cs +++ b/src/StatusAggregator/Components.cs @@ -1,4 +1,4 @@ -using System.Linq; +using NuGet.Services.Status; namespace StatusAggregator { @@ -14,23 +14,23 @@ public static class Components "Browsing the Gallery website", new[] { - new TreeComponent("USNC", "Primary region"), - new TreeComponent("USSC", "Backup region") + new LeafComponent("USNC", "Primary region"), + new LeafComponent("USSC", "Backup region") }), new TreeComponent( "Restore", "Downloading and installing packages from NuGet", - new[] + new IComponent[] { new TreeComponent( "V3", "Restore using the V3 API", new[] { - new TreeComponent("Global", "V3 restore for users outside of China"), - new TreeComponent("China", "V3 restore for users inside China") + new LeafComponent("Global", "V3 restore for users outside of China"), + new LeafComponent("China", "V3 restore for users inside China") }), - new TreeComponent("V2", "Restore using the V2 API") + new LeafComponent("V2", "Restore using the V2 API") }), new TreeComponent( "Search", @@ -42,42 +42,19 @@ public static class Components "Search for packages outside Asia", new[] { - new TreeComponent("USNC", "Primary region"), - new TreeComponent("USSC", "Backup region") + new LeafComponent("USNC", "Primary region"), + new LeafComponent("USSC", "Backup region") }), new PrimarySecondaryComponent( "Asia", "Search for packages inside Asia", new[] { - new TreeComponent("EA", "Primary region"), - new TreeComponent("SEA", "Backup region") + new LeafComponent("EA", "Primary region"), + new LeafComponent("SEA", "Backup region") }) }), - new TreeComponent("Package Publishing", "Uploading new packages to NuGet.org") + new LeafComponent("Package Publishing", "Uploading new packages to NuGet.org") }); - - public static ISubComponent Get(string path) - { - var componentPathParts = path.Split(SubComponent.ComponentPathDivider); - - if (componentPathParts.First() != Root.Name) - { - return null; - } - - ISubComponent currentComponent = new SubComponent(Root); - foreach (var componentPathPart in componentPathParts.Skip(1)) - { - currentComponent = currentComponent.SubComponents.FirstOrDefault(c => c.Name == componentPathPart); - - if (currentComponent == null) - { - break; - } - } - - return currentComponent; - } } } diff --git a/src/StatusAggregator/Event.cs b/src/StatusAggregator/Event.cs deleted file mode 100644 index 1d79d6c9a..000000000 --- a/src/StatusAggregator/Event.cs +++ /dev/null @@ -1,31 +0,0 @@ -using StatusAggregator.Table; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace StatusAggregator -{ - public class Event - { - public Event() - { - } - - public Event(EventEntity eventEntity, IEnumerable messages) - { - AffectedComponentPath = eventEntity.AffectedComponentPath; - AffectedComponentStatus = eventEntity.AffectedComponentStatus; - StartTime = eventEntity.StartTime; - EndTime = eventEntity.EndTime; - Messages = messages; - } - - public string AffectedComponentPath { get; set; } - public ComponentStatus AffectedComponentStatus { get; set; } - public DateTime StartTime { get; set; } - public DateTime? EndTime { get; set; } - public IEnumerable Messages { get; set; } - } -} diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index 8caaddff5..00ad64671 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -7,7 +7,7 @@ namespace StatusAggregator { public class EventUpdater : IEventUpdater { - private static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); + public static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); private readonly ITableWrapper _table; private readonly IMessageUpdater _messageUpdater; @@ -20,21 +20,24 @@ public EventUpdater(ITableWrapper table, IMessageUpdater messageUpdater) public async Task UpdateEvent(EventEntity eventEntity, DateTime nextCreationTime) { + eventEntity = eventEntity ?? throw new ArgumentNullException(nameof(eventEntity)); + if (!eventEntity.IsActive) { // Inactive events cannot be updated. return false; } - var incidentsLinkedToEventToClose = _table.GetIncidentsLinkedToEvent(eventEntity); + var incidentsLinkedToEventToCloseQuery = _table.GetIncidentsLinkedToEvent(eventEntity); - if (!incidentsLinkedToEventToClose.ToList().Any()) + var incidentsLinkedToEventToClose = incidentsLinkedToEventToCloseQuery.ToList(); + if (!incidentsLinkedToEventToClose.Any()) { // If an event has no linked incidents it must have been created manually and should not be closed automatically. return false; } - var shouldClose = !incidentsLinkedToEventToClose + var shouldClose = !incidentsLinkedToEventToCloseQuery .Where(i => i.IsActive || i.MitigationTime > nextCreationTime - EventEndDelay) .ToList() .Any(); @@ -43,7 +46,6 @@ public async Task UpdateEvent(EventEntity eventEntity, DateTime nextCreati { Console.WriteLine($"Closing {eventEntity.RowKey} because its incidents are inactive and too old"); var mitigationTime = incidentsLinkedToEventToClose - .ToList() .Max(i => i.MitigationTime ?? DateTime.MinValue); eventEntity.EndTime = mitigationTime; diff --git a/src/StatusAggregator/IComponent.cs b/src/StatusAggregator/IComponent.cs deleted file mode 100644 index 0991b33fc..000000000 --- a/src/StatusAggregator/IComponent.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Collections.Generic; - -namespace StatusAggregator -{ - public interface IComponent - { - string Name { get; } - string Description { get; } - ComponentStatus Status { get; set; } - IEnumerable SubComponents { get; } - } -} diff --git a/src/StatusAggregator/ISubComponent.cs b/src/StatusAggregator/ISubComponent.cs deleted file mode 100644 index 817b85d9e..000000000 --- a/src/StatusAggregator/ISubComponent.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Newtonsoft.Json; - -namespace StatusAggregator -{ - public interface ISubComponent : IComponent - { - [JsonIgnore] - IComponent Parent { get; } - string Path { get; } - } -} diff --git a/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs index b7e4a6f36..1cc13f21d 100644 --- a/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs @@ -1,4 +1,5 @@ -using System; +using NuGet.Services.Status; +using System; using System.Text.RegularExpressions; namespace StatusAggregator.Incidents.Parse diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs index b1d551621..e8776a18c 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs @@ -1,5 +1,6 @@ using System; using System.Text.RegularExpressions; +using NuGet.Services.Status; namespace StatusAggregator.Incidents.Parse { @@ -66,7 +67,7 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo return false; } - affectedComponentPath = $"NuGet{SubComponent.ComponentPathDivider}Search{SubComponent.ComponentPathDivider}{region}{SubComponent.ComponentPathDivider}{subRegion}"; + affectedComponentPath = ComponentUtility.GetPath("NuGet", "Search", region, subRegion); return true; } diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index c672c94a9..313e4f8b9 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -1,4 +1,5 @@ -using System.Text.RegularExpressions; +using NuGet.Services.Status; +using System.Text.RegularExpressions; namespace StatusAggregator.Incidents.Parse { @@ -16,7 +17,7 @@ public OutdatedSearchServiceInstanceIncidentParser(string environment) protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = $"NuGet{SubComponent.ComponentPathDivider}Package Publishing"; + affectedComponentPath = ComponentUtility.GetPath("NuGet", "Package Publishing"); return true; } diff --git a/src/StatusAggregator/Incidents/Parse/ParsedIncident.cs b/src/StatusAggregator/Incidents/Parse/ParsedIncident.cs index 88aee170a..ad879b954 100644 --- a/src/StatusAggregator/Incidents/Parse/ParsedIncident.cs +++ b/src/StatusAggregator/Incidents/Parse/ParsedIncident.cs @@ -1,4 +1,5 @@ -using System; +using NuGet.Services.Status; +using System; namespace StatusAggregator.Incidents.Parse { diff --git a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs index bb579d406..66a680dfa 100644 --- a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs @@ -1,5 +1,5 @@ -using System; -using System.Text.RegularExpressions; +using System.Text.RegularExpressions; +using NuGet.Services.Status; namespace StatusAggregator.Incidents.Parse { @@ -17,7 +17,7 @@ public ValidationDurationIncidentParser(string environment) protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = $"NuGet{SubComponent.ComponentPathDivider}Package Publishing"; + affectedComponentPath = ComponentUtility.GetPath("NuGet", "Package Publishing"); return true; } diff --git a/src/StatusAggregator/Message.cs b/src/StatusAggregator/Message.cs deleted file mode 100644 index f69c6f130..000000000 --- a/src/StatusAggregator/Message.cs +++ /dev/null @@ -1,21 +0,0 @@ -using StatusAggregator.Table; -using System; - -namespace StatusAggregator -{ - public class Message - { - public Message() - { - } - - public Message(MessageEntity messageEntity) - { - Time = messageEntity.Time; - Contents = messageEntity.Contents; - } - - public DateTime Time { get; set; } - public string Contents { get; set; } - } -} diff --git a/src/StatusAggregator/PrimarySecondaryComponent.cs b/src/StatusAggregator/PrimarySecondaryComponent.cs deleted file mode 100644 index cff495fd6..000000000 --- a/src/StatusAggregator/PrimarySecondaryComponent.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace StatusAggregator -{ - public class PrimarySecondaryComponent : Component - { - public PrimarySecondaryComponent() - { - } - - public PrimarySecondaryComponent( - string name, - string description) - : base(name, description) - { - } - - public PrimarySecondaryComponent( - string name, - string description, - IEnumerable subComponents) - : base(name, description, subComponents) - { - } - - public PrimarySecondaryComponent( - string name, - string description, - ComponentStatus status) - : this(name, description) - { - _status = status; - } - - public PrimarySecondaryComponent( - string name, - string description, - ComponentStatus status, - IEnumerable subComponents) - : this(name, description, subComponents) - { - _status = status; - } - - private ComponentStatus? _status = null; - public override ComponentStatus Status - { - get - { - if (_status.HasValue) - { - return _status.Value; - } - - if (!SubComponents.Any()) - { - return ComponentStatus.Up; - } - - // Iterate through the list of subcomponents in order. - var isFirst = true; - foreach (var subComponent in SubComponents) - { - if (subComponent.Status == ComponentStatus.Up) - { - // If the first component is up, the status is up. - // If any child component is up, the status is degraded. - return isFirst ? ComponentStatus.Up : ComponentStatus.Degraded; - } - - // If any component is degraded, the status is degraded. - if (subComponent.Status == ComponentStatus.Degraded) - { - return ComponentStatus.Degraded; - } - - isFirst = false; - } - - // If all components are down, the status is down. - return ComponentStatus.Down; - } - set - { - _status = value; - } - } - } -} diff --git a/src/StatusAggregator/Status.cs b/src/StatusAggregator/Status.cs deleted file mode 100644 index 588312b04..000000000 --- a/src/StatusAggregator/Status.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace StatusAggregator -{ - public class Status - { - public Status() - { - } - - public Status(IComponent rootComponent, IEnumerable events) - { - LastUpdated = DateTime.Now; - RootComponent = rootComponent; - Events = events; - } - - public DateTime LastUpdated { get; set; } - public IComponent RootComponent { get; set; } - public IEnumerable Events { get; set; } - } -} diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index b7e62f794..f254a6eb2 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -47,12 +47,9 @@ - - - @@ -68,14 +65,10 @@ - - - - @@ -84,7 +77,6 @@ - @@ -92,11 +84,10 @@ - + - @@ -108,6 +99,9 @@ + + 2.26.0-sb-status-34842 + 9.2.0 diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 0796a61db..368008f17 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -5,6 +5,7 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using NuGet.Services.Status; using StatusAggregator.Table; namespace StatusAggregator @@ -33,10 +34,12 @@ public StatusExporter(CloudBlobContainer container, ITableWrapper table) public async Task Export() { + var rootComponent = Components.Root; + var activeEvents = _table.GetActiveEvents(); foreach (var activeEvent in activeEvents) { - var currentComponent = Components.Get(activeEvent.AffectedComponentPath); + var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); if (currentComponent == null) { @@ -56,11 +59,11 @@ public async Task Export() { var messages = _table.GetMessagesLinkedToEvent(e) .ToList() - .Select(m => new Message(m)); - return new Event(e, messages); + .Select(m => m.AsMessage()); + return e.AsEvent(messages); }); - var status = new Status(Components.Root, recentEvents); + var status = new ServiceStatus(Components.Root, recentEvents); var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); var blob = _container.GetBlockBlobReference(StatusBlobName); diff --git a/src/StatusAggregator/SubComponent.cs b/src/StatusAggregator/SubComponent.cs deleted file mode 100644 index 42aa942d1..000000000 --- a/src/StatusAggregator/SubComponent.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System.Collections.Generic; - -namespace StatusAggregator -{ - public class SubComponent : ISubComponent - { - public const char ComponentPathDivider = '/'; - - private readonly IComponent _component; - - public string Name => _component.Name; - public string Description => _component.Description; - public ComponentStatus Status { get => _component.Status; set => _component.Status = value; } - public IEnumerable SubComponents => _component.SubComponents; - public IComponent Parent { get; } - public string Path { get; } - - public SubComponent(IComponent component) - { - _component = component; - Parent = null; - Path = (_component as SubComponent)?.Path ?? _component.Name; - } - - public SubComponent(IComponent component, IComponent parent) - : this(component) - { - Parent = parent; - Path = parent.Name + ComponentPathDivider + Path; - } - } -} diff --git a/src/StatusAggregator/Table/EventEntity.cs b/src/StatusAggregator/Table/EventEntity.cs index 8160d1e01..f51d64b24 100644 --- a/src/StatusAggregator/Table/EventEntity.cs +++ b/src/StatusAggregator/Table/EventEntity.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using Microsoft.WindowsAzure.Storage.Table; +using NuGet.Services.Status; namespace StatusAggregator.Table { @@ -30,9 +32,14 @@ public bool IsActive set { } } + public Event AsEvent(IEnumerable messages) + { + return new Event(AffectedComponentPath, AffectedComponentStatus, StartTime, EndTime, messages); + } + private static string GetRowKey(IncidentEntity incidentEntity) { - return $"{Component.ToRowKeySafeComponentPath(incidentEntity.AffectedComponentPath)}_{incidentEntity.CreationTime.ToString("o")}"; + return $"{TableUtility.ToRowKeySafeComponentPath(incidentEntity.AffectedComponentPath)}_{incidentEntity.CreationTime.ToString("o")}"; } } } diff --git a/src/StatusAggregator/Table/IncidentEntity.cs b/src/StatusAggregator/Table/IncidentEntity.cs index 999a6c3d7..c8ead2217 100644 --- a/src/StatusAggregator/Table/IncidentEntity.cs +++ b/src/StatusAggregator/Table/IncidentEntity.cs @@ -1,5 +1,6 @@ using System; using Microsoft.WindowsAzure.Storage.Table; +using NuGet.Services.Status; using StatusAggregator.Incidents.Parse; namespace StatusAggregator.Table @@ -41,7 +42,7 @@ public bool IsActive private static string GetRowKey(ParsedIncident parsedIncident) { - return $"{parsedIncident.Id}_{Component.ToRowKeySafeComponentPath(parsedIncident.AffectedComponentPath)}_{parsedIncident.AffectedComponentStatus}"; + return $"{parsedIncident.Id}_{TableUtility.ToRowKeySafeComponentPath(parsedIncident.AffectedComponentPath)}_{parsedIncident.AffectedComponentStatus}"; } } } diff --git a/src/StatusAggregator/Table/MessageEntity.cs b/src/StatusAggregator/Table/MessageEntity.cs index 158974cbf..8913d9c7d 100644 --- a/src/StatusAggregator/Table/MessageEntity.cs +++ b/src/StatusAggregator/Table/MessageEntity.cs @@ -1,5 +1,6 @@ using System; using Microsoft.WindowsAzure.Storage.Table; +using NuGet.Services.Status; namespace StatusAggregator.Table { @@ -23,6 +24,11 @@ public MessageEntity(EventEntity eventEntity, DateTime time, string contents) public DateTime Time { get; set; } public string Contents { get; set; } + public Message AsMessage() + { + return new Message(Time, Contents); + } + private static string GetRowKey(EventEntity eventEntity, DateTime time) { return $"{eventEntity.RowKey}_{time.ToString("o")}"; diff --git a/src/StatusAggregator/Table/TableUtility.cs b/src/StatusAggregator/Table/TableUtility.cs new file mode 100644 index 000000000..a50bd3b72 --- /dev/null +++ b/src/StatusAggregator/Table/TableUtility.cs @@ -0,0 +1,12 @@ +using NuGet.Services.Status; + +namespace StatusAggregator.Table +{ + public static class TableUtility + { + public static string ToRowKeySafeComponentPath(string componentPath) + { + return componentPath.Replace(Constants.ComponentPathDivider, '_'); + } + } +} diff --git a/src/StatusAggregator/TreeComponent.cs b/src/StatusAggregator/TreeComponent.cs deleted file mode 100644 index 5c41eb456..000000000 --- a/src/StatusAggregator/TreeComponent.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace StatusAggregator -{ - public class TreeComponent : Component - { - public TreeComponent() - { - } - - public TreeComponent( - string name, - string description) - : base(name, description) - { - } - - public TreeComponent( - string name, - string description, - IEnumerable subComponents) - : base(name, description, subComponents) - { - } - - public TreeComponent( - string name, - string description, - ComponentStatus status) - : this(name, description) - { - _status = status; - } - - public TreeComponent( - string name, - string description, - ComponentStatus status, - IEnumerable subComponents) - : this(name, description, subComponents) - { - _status = status; - } - - private ComponentStatus? _status = null; - public override ComponentStatus Status - { - get - { - if (_status.HasValue) - { - return _status.Value; - } - - if (!SubComponents.Any()) - { - return ComponentStatus.Up; - } - - // If all subcomponents are up, we are up. - if (SubComponents.All(c => c.Status == ComponentStatus.Up)) - { - return ComponentStatus.Up; - } - - // If all subcomponents are down, we are down. - if (SubComponents.All(c => c.Status == ComponentStatus.Down)) - { - return ComponentStatus.Down; - } - - // Otherwise, we are degraded, because some subcomponents are degraded or down but not all. - return ComponentStatus.Degraded; - } - set - { - _status = value; - } - } - } -} diff --git a/tests/StatusAggregator.Tests/EventUpdaterTests.cs b/tests/StatusAggregator.Tests/EventUpdaterTests.cs new file mode 100644 index 000000000..7ba29bccb --- /dev/null +++ b/tests/StatusAggregator.Tests/EventUpdaterTests.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Moq; +using StatusAggregator.Table; +using Xunit; + +namespace StatusAggregator.Tests +{ + public class EventUpdaterTests + { + private const string RowKey = "rowkey"; + private static readonly DateTime NextCreationTime = new DateTime(2018, 7, 10); + + private static IEnumerable ClosableIncidents => new[] + { + CreateIncidentEntity(new DateTime(2018, 7, 9)), // Recently closed incident + CreateIncidentEntity(DateTime.MinValue), // Old incident + }; + + private static IEnumerable UnclosableIncidents => new[] + { + CreateIncidentEntity(NextCreationTime + EventUpdater.EventEndDelay), // Incident closed too recently + CreateIncidentEntity() // Active incident + }; + + private Mock _tableWrapperMock { get; } + private Mock _messageUpdaterMock { get; } + private EventUpdater _eventUpdater { get; } + private EventEntity _eventEntity { get; } + + public EventUpdaterTests() + { + _tableWrapperMock = new Mock(); + _messageUpdaterMock = new Mock(); + _eventUpdater = new EventUpdater(_tableWrapperMock.Object, _messageUpdaterMock.Object); + + _eventEntity = new EventEntity() + { + RowKey = RowKey, + StartTime = DateTime.MinValue, + EndTime = null + }; + } + + private static IncidentEntity CreateIncidentEntity(DateTime? mitigationTime = null) + { + return new IncidentEntity() + { + PartitionKey = IncidentEntity.DefaultPartitionKey, + EventRowKey = RowKey, + CreationTime = DateTime.MinValue, + MitigationTime = mitigationTime + }; + } + + [Fact] + public async Task ThrowsIfEventNull() + { + await Assert.ThrowsAsync(() => _eventUpdater.UpdateEvent(null, DateTime.MinValue)); + } + + [Fact] + public async Task ReturnsFalseIfNotActive() + { + _eventEntity.EndTime = DateTime.MinValue; + + var result = await _eventUpdater.UpdateEvent(_eventEntity, NextCreationTime); + + Assert.False(result); + } + + [Fact] + public async Task ReturnsFalseIfNoLinkedIncidents() + { + _tableWrapperMock + .Setup(x => x.CreateQuery()) + .Returns(new IncidentEntity[0].AsQueryable()); + + var result = await _eventUpdater.UpdateEvent(_eventEntity, NextCreationTime); + + Assert.False(result); + } + + public static IEnumerable DoesNotCloseEventIfUnclosableIncident_Data + { + get + { + foreach (var unclosableIncident in UnclosableIncidents) + { + yield return new object[] { unclosableIncident }; + } + } + } + + [Theory] + [MemberData(nameof(DoesNotCloseEventIfUnclosableIncident_Data))] + public async Task DoesNotCloseEventIfUnclosableIncident(IncidentEntity unclosableIncident) + { + _tableWrapperMock + .Setup(x => x.CreateQuery()) + .Returns(ClosableIncidents.Concat(new[] { unclosableIncident }).AsQueryable()); + + var result = await _eventUpdater.UpdateEvent(_eventEntity, NextCreationTime); + + Assert.False(result); + Assert.Null(_eventEntity.EndTime); + _messageUpdaterMock.Verify( + x => x.CreateMessageForEventStart(_eventEntity, NextCreationTime), + Times.Once()); + _messageUpdaterMock.Verify( + x => x.CreateMessageForEventEnd(It.IsAny()), + Times.Never()); + } + + [Fact] + public async Task ClosesEventIfClosableIncidents() + { + _tableWrapperMock + .Setup(x => x.CreateQuery()) + .Returns(ClosableIncidents.AsQueryable()); + + var result = await _eventUpdater.UpdateEvent(_eventEntity, NextCreationTime); + + var expectedEndTime = ClosableIncidents.Max(i => i.MitigationTime ?? DateTime.MinValue); + Assert.True(result); + Assert.Equal(expectedEndTime, _eventEntity.EndTime); + _tableWrapperMock.Verify( + x => x.InsertOrReplaceAsync(_eventEntity), + Times.Once()); + _messageUpdaterMock.Verify( + x => x.CreateMessageForEventStart(_eventEntity, expectedEndTime), + Times.Once()); + _messageUpdaterMock.Verify( + x => x.CreateMessageForEventEnd(_eventEntity), + Times.Once()); + } + } +} diff --git a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs new file mode 100644 index 000000000..e6dd48eea --- /dev/null +++ b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs @@ -0,0 +1,78 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage.Table; +using Moq; +using NuGet.Services.Status; +using StatusAggregator.Incidents; +using StatusAggregator.Incidents.Parse; +using StatusAggregator.Table; +using Xunit; + +namespace StatusAggregator.Tests +{ + public class IncidentFactoryTests + { + private const string Id = "id"; + private const string AffectedComponentPath = "path"; + private const ComponentStatus AffectedComponentStatus = ComponentStatus.Degraded; + private static DateTime CreationTime = new DateTime(2017, 7, 10); + + private Mock _tableWrapperMock { get; } + private Mock _eventUpdater { get; } + private IncidentFactory _incidentFactory { get; } + private ParsedIncident _parsedIncident { get; } + + public IncidentFactoryTests() + { + _tableWrapperMock = new Mock(); + _eventUpdater = new Mock(); + _incidentFactory = new IncidentFactory(_tableWrapperMock.Object, _eventUpdater.Object); + + var incident = new Incident() { Id = Id, Source = new IncidentSourceData() { CreateDate = CreationTime } }; + _parsedIncident = new ParsedIncident(incident, AffectedComponentPath, AffectedComponentStatus); + } + + [Fact] + public async Task CreatesNewEventIfNoPossibleEvents() + { + _tableWrapperMock + .Setup(x => x.CreateQuery()) + .Returns(new EventEntity[0].AsQueryable()); + + _tableWrapperMock + .Setup(x => x.InsertOrReplaceAsync(It.IsAny())) + .Returns(Task.CompletedTask); + + EventEntity eventEntity = null; + _tableWrapperMock + .Setup(x => x.InsertOrReplaceAsync(It.Is(e => e is EventEntity))) + .Callback(e => { eventEntity = e as EventEntity; }) + .Returns(Task.CompletedTask); + + var incidentEntity = await _incidentFactory.CreateIncident(_parsedIncident); + + Assert.Equal(Id, incidentEntity.IncidentApiId); + Assert.Equal(CreationTime, incidentEntity.CreationTime); + Assert.Equal(AffectedComponentPath, incidentEntity.AffectedComponentPath); + Assert.Equal(AffectedComponentStatus, incidentEntity.AffectedComponentStatus); + Assert.NotNull(eventEntity); + Assert.Equal(eventEntity.RowKey, incidentEntity.EventRowKey); + Assert.Equal(IncidentEntity.DefaultPartitionKey, incidentEntity.PartitionKey); + Assert.True(incidentEntity.IsLinkedToEvent); + Assert.True(incidentEntity.IsActive); + + _tableWrapperMock.Verify( + x => x.InsertOrReplaceAsync(incidentEntity), + Times.Once()); + + _tableWrapperMock.Verify( + x => x.InsertOrReplaceAsync(eventEntity), + Times.Once()); + + _tableWrapperMock.Verify( + x => x.InsertOrReplaceAsync(It.IsAny()), + Times.Exactly(2)); + } + } +} diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj index 713e30e03..efb72f93d 100644 --- a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj +++ b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj @@ -9,9 +9,12 @@ Properties StatusAggregator.Tests StatusAggregator.Tests - v4.5.2 + v4.6.2 512 true + + + true @@ -32,6 +35,7 @@ + @@ -41,6 +45,8 @@ + + @@ -49,5 +55,28 @@ StatusAggregator + + + 4.3.0 + + + 4.7.145 + + + 4.3.0 + + + 4.4.0 + + + 9.2.0 + + + 2.3.1 + + + 2.3.1 + + \ No newline at end of file From fd8d8ce8081b556effb16aab94ff4df29d7ad67c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 13 Jul 2018 16:00:56 -0700 Subject: [PATCH 07/49] fixed bugs and configured deployment --- .../Configuration/JobArgumentNames.cs | 2 +- src/StatusAggregator/Components.cs | 5 ++- ...dentParser.cs => DefaultIncidentParser.cs} | 18 +++++++---- ...onalSearchServiceInstanceIncidentParser.cs | 14 ++++---- ...atedSearchServiceInstanceIncidentParser.cs | 10 +++--- .../Parse/ValidationDurationIncidentParser.cs | 12 +++---- src/StatusAggregator/Job.cs | 14 ++++---- src/StatusAggregator/Scripts/Functions.ps1 | 30 ++++++++++++++++++ src/StatusAggregator/Scripts/PostDeploy.ps1 | 18 +++++++++++ src/StatusAggregator/Scripts/PreDeploy.ps1 | 11 +++++++ .../Scripts/StatusAggregator.cmd | 28 ++++++++++++++++ src/StatusAggregator/Scripts/nssm.exe | Bin 0 -> 331264 bytes src/StatusAggregator/StatusAggregator.csproj | 6 +++- src/StatusAggregator/StatusAggregator.nuspec | 22 +++++++++++++ src/StatusAggregator/StatusExporter.cs | 6 ++-- src/StatusAggregator/Table/EventEntity.cs | 6 ++-- src/StatusAggregator/Table/IncidentEntity.cs | 4 +-- .../IncidentFactoryTests.cs | 2 +- 18 files changed, 164 insertions(+), 44 deletions(-) rename src/StatusAggregator/Incidents/Parse/{EnvironmentIncidentParser.cs => DefaultIncidentParser.cs} (70%) create mode 100644 src/StatusAggregator/Scripts/Functions.ps1 create mode 100644 src/StatusAggregator/Scripts/PostDeploy.ps1 create mode 100644 src/StatusAggregator/Scripts/PreDeploy.ps1 create mode 100644 src/StatusAggregator/Scripts/StatusAggregator.cmd create mode 100644 src/StatusAggregator/Scripts/nssm.exe create mode 100644 src/StatusAggregator/StatusAggregator.nuspec diff --git a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs index a0ed43b28..d679da57a 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs @@ -123,12 +123,12 @@ public static class JobArgumentNames public const string StatusContainerName = "StatusContainerName"; public const string StatusTableName = "StatusTableName"; public const string StatusEnvironment = "StatusEnvironment"; + public const string StatusMaximumSeverity = "StatusMaximumSeverity"; public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri"; public const string StatusIncidentApiRoutingId = "StatusIncidentApiRoutingId"; public const string StatusIncidentApiCertificateThumbprint = "StatusIncidentApiCertificateThumbprint"; public const string StatusIncidentApiCertificateStoreName = "StatusIncidentApiCertificateStoreName"; public const string StatusIncidentApiCertificateStoreLocation = "StatusIncidentApiCertificateStoreLocation"; - public const string StatusIncidentApiEnvironment = "StatusIncidentApiEnvironment"; // Arguments specific to Stats.AggregateCdnDownloadsInGallery public static string BatchSleepSeconds = "BatchSleepSeconds"; diff --git a/src/StatusAggregator/Components.cs b/src/StatusAggregator/Components.cs index c39567096..766724bd7 100644 --- a/src/StatusAggregator/Components.cs +++ b/src/StatusAggregator/Components.cs @@ -4,7 +4,9 @@ namespace StatusAggregator { public static class Components { - public static IComponent Root = new TreeComponent( + public static IComponent CreateNuGetServiceRootComponent() + { + return new TreeComponent( "NuGet", "", new IComponent[] @@ -56,5 +58,6 @@ public static class Components }), new LeafComponent("Package Publishing", "Uploading new packages to NuGet.org") }); + } } } diff --git a/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs similarity index 70% rename from src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs rename to src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs index 1cc13f21d..daf37e596 100644 --- a/src/StatusAggregator/Incidents/Parse/EnvironmentIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs @@ -1,31 +1,35 @@ using NuGet.Services.Status; using System; +using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; namespace StatusAggregator.Incidents.Parse { - public abstract class EnvironmentIncidentParser : IncidentParser + public abstract class DefaultIncidentParser : IncidentParser { private const string EnvironmentGroupName = "Environment"; - private readonly string _environment; + private readonly IEnumerable _environments; + private readonly int _maximumSeverity; - public EnvironmentIncidentParser(string subtitleRegEx, string environment) + public DefaultIncidentParser(string subtitleRegEx, IEnumerable environments, int maximumSeverity) : base(GetRegEx(subtitleRegEx)) { - _environment = environment; + _environments = environments; + _maximumSeverity = maximumSeverity; } protected override bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident) { parsedIncident = null; - if (incident.Severity > 2) + if (incident.Severity > _maximumSeverity) { return false; } - - if (!string.Equals(groups[EnvironmentGroupName].Value, _environment, StringComparison.OrdinalIgnoreCase)) + + if (!_environments.Any(e => string.Equals(groups[EnvironmentGroupName].Value, e, StringComparison.OrdinalIgnoreCase))) { return false; } diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs index e8776a18c..075a4536b 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs @@ -1,21 +1,23 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; using NuGet.Services.Status; namespace StatusAggregator.Incidents.Parse { - public class OutdatedRegionalSearchServiceInstanceIncidentParser : EnvironmentIncidentParser + public class OutdatedRegionalSearchServiceInstanceIncidentParser : DefaultIncidentParser { private const string ServiceEnvironmentGroupName = "SearchServiceName"; private const string ServiceRegionGroupName = "SearchServiceName"; private static string SubtitleRegEx = $@"Search service 'nuget-\[(?<{ServiceEnvironmentGroupName}>.*)\]-\[(?<{ServiceRegionGroupName}>.*)\]-search' is using an outdated index!"; - private readonly string _environment; + private readonly IEnumerable _environments; - public OutdatedRegionalSearchServiceInstanceIncidentParser(string environment) - : base(SubtitleRegEx, environment) + public OutdatedRegionalSearchServiceInstanceIncidentParser(IEnumerable environments, int maximumSeverity) + : base(SubtitleRegEx, environments, maximumSeverity) { - _environment = environment; + _environments = environments; } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) @@ -25,7 +27,7 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo var searchEnvironment = groups[ServiceEnvironmentGroupName].Value; var searchRegion = groups[ServiceRegionGroupName].Value; - if (!string.Equals(searchEnvironment, _environment, StringComparison.OrdinalIgnoreCase)) + if (!_environments.Any(e => string.Equals(searchEnvironment, e, StringComparison.OrdinalIgnoreCase))) { return false; } diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index 313e4f8b9..fe2ce935d 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -1,18 +1,16 @@ using NuGet.Services.Status; +using System.Collections.Generic; using System.Text.RegularExpressions; namespace StatusAggregator.Incidents.Parse { - public class OutdatedSearchServiceInstanceIncidentParser : EnvironmentIncidentParser + public class OutdatedSearchServiceInstanceIncidentParser : DefaultIncidentParser { private const string SubtitleRegEx = "A search service instance is using an outdated index!"; - private readonly string _environment; - - public OutdatedSearchServiceInstanceIncidentParser(string environment) - : base(SubtitleRegEx, environment) + public OutdatedSearchServiceInstanceIncidentParser(IEnumerable environments, int maximumSeverity) + : base(SubtitleRegEx, environments, maximumSeverity) { - _environment = environment; } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) diff --git a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs index 66a680dfa..0ae17dc2d 100644 --- a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs @@ -1,18 +1,16 @@ -using System.Text.RegularExpressions; +using System.Collections.Generic; +using System.Text.RegularExpressions; using NuGet.Services.Status; namespace StatusAggregator.Incidents.Parse { - public class ValidationDurationIncidentParser : EnvironmentIncidentParser + public class ValidationDurationIncidentParser : DefaultIncidentParser { private const string SubtitleRegEx = "Too many packages are stuck in the \"Validating\" state!"; - private readonly string _environment; - - public ValidationDurationIncidentParser(string environment) - : base(SubtitleRegEx, environment) + public ValidationDurationIncidentParser(IEnumerable environments, int maximumSeverity) + : base(SubtitleRegEx, environments, maximumSeverity) { - _environment = environment; } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 3f95a3bd5..f548194f7 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -40,7 +40,6 @@ public override void Init(IServiceContainer serviceContainer, IDictionary GetIncidentParsers() + private IEnumerable GetIncidentParsers(IEnumerable environments, int maximumSeverity) { return new IIncidentParser[] { - new ValidationDurationIncidentParser("PROD"), - new OutdatedRegionalSearchServiceInstanceIncidentParser("PROD"), - new OutdatedSearchServiceInstanceIncidentParser("PROD") + new ValidationDurationIncidentParser(environments, maximumSeverity), + new OutdatedRegionalSearchServiceInstanceIncidentParser(environments, maximumSeverity), + new OutdatedSearchServiceInstanceIncidentParser(environments, maximumSeverity) }; } } diff --git a/src/StatusAggregator/Scripts/Functions.ps1 b/src/StatusAggregator/Scripts/Functions.ps1 new file mode 100644 index 000000000..a8bff40fc --- /dev/null +++ b/src/StatusAggregator/Scripts/Functions.ps1 @@ -0,0 +1,30 @@ +Function Uninstall-NuGetService() { + Param ([string]$ServiceName) + + if (Get-Service $ServiceName -ErrorAction SilentlyContinue) + { + Write-Host Removing service $ServiceName... + Stop-Service $ServiceName -Force + sc.exe delete $ServiceName + Write-Host Removed service $ServiceName. + } else { + Write-Host Skipping removal of service $ServiceName - no such service exists. + } +} + +Function Install-NuGetService() { + Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) + + Write-Host Installing service $ServiceName... + + $installService = "nssm install $ServiceName $ScriptToRun" + cmd /C $installService + + Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic + sc.exe failure $ServiceName reset= 30 actions= restart/5000 + + # Run service + net start $ServiceName + + Write-Host Installed service $ServiceName. +} \ No newline at end of file diff --git a/src/StatusAggregator/Scripts/PostDeploy.ps1 b/src/StatusAggregator/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..7d5183d5b --- /dev/null +++ b/src/StatusAggregator/Scripts/PostDeploy.ps1 @@ -0,0 +1,18 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Installing services... + +$currentDirectory = [string](Get-Location) + +$jobsToInstall.Split("{;}") | %{ + $serviceName = $_ + $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] + $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] + $scriptToRun = "$currentDirectory\$scriptToRun" + + Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun +} + +Write-Host Installed services. \ No newline at end of file diff --git a/src/StatusAggregator/Scripts/PreDeploy.ps1 b/src/StatusAggregator/Scripts/PreDeploy.ps1 new file mode 100644 index 000000000..ef711a912 --- /dev/null +++ b/src/StatusAggregator/Scripts/PreDeploy.ps1 @@ -0,0 +1,11 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Removing services... + +$jobsToInstall.Split("{;}") | %{ + Uninstall-NuGetService -ServiceName $_ +} + +Write-Host Removed services. \ No newline at end of file diff --git a/src/StatusAggregator/Scripts/StatusAggregator.cmd b/src/StatusAggregator/Scripts/StatusAggregator.cmd new file mode 100644 index 000000000..fe84a08c7 --- /dev/null +++ b/src/StatusAggregator/Scripts/StatusAggregator.cmd @@ -0,0 +1,28 @@ +@echo OFF + +cd bin + +:Top +echo "Starting job - #{Jobs.statusaggregator.Title}" + +title #{Jobs.archivepackages.Title} + +start /w statusaggregator.exe ^ + -StatusIncidentApiBaseUri "#{Jobs.statusaggregator.IncidentApiBaseUri}" ^ + -StatusIncidentApiRoutingId "#{Jobs.statusaggregator.IncidentApiRoutingId}" ^ + -StatusIncidentApiCertificateThumbprint "#{Jobs.statusaggregator.IncidentApiCertificateThumbprint}" ^ + -StatusStorageAccount "#{Jobs.statusaggregator.StorageAccount}" ^ + -StatusContainerName "#{Jobs.statusaggregator.TableName}" ^ + -StatusTableName "#{Jobs.statusaggregator.TableName}" ^ + -StatusEnvironment "#{Jobs.statusaggregator.Environment}" ^ + -StatusMaximumSeverity "#{Jobs.statusaggregator.MaximumSeverity}" + -VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^ + -ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^ + -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^ + -Sleep "#{Jobs.statusaggregator.Sleep}" ^ + -InstrumentationKey "#{Jobs.statusaggregator.ApplicationInsightsInstrumentationKey}" + +echo "Finished #{Jobs.statusaggregator.Title}" + +goto Top + \ No newline at end of file diff --git a/src/StatusAggregator/Scripts/nssm.exe b/src/StatusAggregator/Scripts/nssm.exe new file mode 100644 index 0000000000000000000000000000000000000000..6ccfe3cfb85f0f7126648dceb7ac93d58b025a0f GIT binary patch literal 331264 zcmeFad3;nw7B<}3AS|&FxyUGLt5KsUMxrB#Xt#8rBaKF7bAz}cIxeUkS5&YQG;?Wn z#@%tjaUErJW^^P32tguAKt#X=#09tU#<(CV22t+!Jg4g3+qV;f-~0ai^81mlTXpKG zQ>RXyIdQ_l!pe&vN14K66?S*W3R#KyGM z9ePl$^Z%)vo~iv1&rjX-uiBUK`{;L*Yv)UPVeLf{|3vK_^84@Fh4TA~{GNWv1=EO; z$#HpUAaLRAyg*erXPlF^IdEX0S9T9nIy(^P#@Yu0b8E5!fj$xpBc#Ge&I{lV{yD#a z@;njNW1r_yZZn-sIxi>{i1Wj|I)=f(9oJ?D<}k21J1}qxu;0oKd_I5?|Cj$l*@4<_ zJI5HDxaj%>l8?wnnPiA{XQQY63k3odgRj2uyu^8dz-yNy0hoa+@SBg{w0~i!$6!Sk zcP{o@r$~p{|g5K4TGmELa8I_8rTaI_jaHQ55D@Et1m#3$ON(pRN()M9qH2X z|6ln3UIKCRgIMzGNhJ+&v!yI`MsuXr4h91yEoC($oALiG{9lj%oABSl|IOA_XJ-Yf z8xq6K=2*=+h2_@Scs84@aZ>_;n7Q7XfycPHES8E@)OE`agsp{8rkDxJT6U{gvT@R~ zk_P#Cga5*3p!iVC%)952Y|w3p0eeDSe~{X5Ay^Pg}7R(62e?*8yA6E zC;#0`WXw&cKd(vU_Na*#7Mtz1(S46>2(PJRZOm-8jVwIH%*~X@ zrBM?k1cj)#os$)4ttWD7TwyVJs%SzgJGE`R-8VasUI6o@B$E<(jiJ_s^vqV^J-OHb5D|HSkXtrDb#bexD&6Z)8x?q>P-^=)ja-*6WG^A2tfm_@s`5I;?ms3va(Emes~&&Qa;vF_sk{fNfb7W9 z_FK*c=fsJbs@S-xc{a78n@Z7R=A6Qrq$_#D+NT2evl`jCB9kNMp1myi>dmS?;T=u3 zQQsR3nY+czx1zyU@%$>D!Q8pFF%Yb>4nGF7A;50BVRzt3F}`_%!}$LYLIbWli=4Gv z+wqvzau~Y8*0E$A<9(ntY}`aK9=%Q?|Q>gRlNKJMngHG7E6 z89NBEW`0etHmEza(v0QS^aAgsy7o1RDzTG8^D2vgWDVYK+m!C!Rsw8LbJ*yVq4hE} zYe`R(sQ52-`9GNaH?=+kgCeRBGgnxB-PC-9tV8g}r$ti%R@)t5t=A!&W6StkTGG&3 z1TLJo@EPN2-)ULV!-`|(S5}W-#B}wpl@11jjAmiagDDwTI288N`qH7GDWk0hg;bTi z7spd`3h!V7IwtC-3?%`=s>@0bKYu)GwH~HxEY*SJ)(Av7x?vSFeWkSzf10iSXHqWf ztrPI*j2=a7xtq^MMd_irTxi{b*_a-ynNe7dE;JyPIzAjTs}xt}^!(ubNdDBE+#OSs z4Y|Sj>uuvj^a$db&AKN%Egfb(+v-%8!AHiWF?bgqtQYYI{b30^QYvo~e!TwhsVE%T zFu_J@WDm_^G&nzZ2U>CmdUmrFM~@25@4X|IG9G32oRw;Tuu7!o-CH;lm17QMI|(^X z?}K(4kZtvoWJl2F1}yuGH0HL*gHtx`y|fV;U5}?JW0`c74c1Nb0vsuOIjxNWFqRxq zcpsX!GGB5s#%cM*zfm_TzvPHLG*#Z8B8JkeEWw!ZJ>0eh)EO9|(rq%%)}woI6d{p} z&`$Xr$XhBodJiWbk_HhB#)H-R0wF3tQ^OaOASQ(-JZ`9E)~7$SLx|3z1;?SJ7+O%3 z;%GPOI|1rl9y7mZc?zL?GU^jAoMwf9kY05pP)(wnJ!1hwKH-naGVMjTc0RtN4H8hqQv1&INhVfLyk*xxQ8jFUmyo1pg z!*vA=k?oCYA!k%OK;a(=e3W)3H0YPp&_&vl?b(SjV8k{~#$cp~+L^N2+AyA-r`^gz z9&PzB6xQ`%{~0lfY2-ZYD?5t+T+9@W#r9BBCv*vW{Ox zgKokrRO}A}F-GLEs6v0%q}dAUnjDKqdM6xvda@eVPJ|iY(A##b$Y7KPnQmF-xZH`_RugvrJt*NLPI^QkJK2@@mZfL3gNzu+gdZK?T5Bsjl4IRAayutj+-@q?x! z!=$NDjI+Seq*AjnIR9PLDQd11N5?kaCo#lVQ5Qmm5f2&==+s7mf;-Iyod#BlBj-$+ z?6QOYS35l$ovaGZSZvTHIMyf61fKF}!$?Uk2dvF4jl z`Jn-%Sqnd9Qah#n$5`r&Y<7>KGI}6`ZN$-TFb;j$gTp0jBeek-DytjZsF=ChHjY8^ z)Z~o9aG>(QSZXA47d(%K5^8#wbu2?fWj#z?gdD@5xmIZA6ovuDRJf#hrz1|XaM?p; zxmdC&GWY}NXwsCnaRP!R-_S2_C*;vn(cs1WGX#xT98Hy?IaYg3xF@?P=^{&{4Urx( z-(xylN)C^<@hdb~gMWgc223bGrx*gW&@|{p)O*apZS}{6n5Ai1i3Y7Sl$FI2QCI zb75uyz{&HK(1M&!6X#`?@Cp*OgdSp$T&n^&M;ubShsY?sd86i1AN-|c%zRIkV6I#! z0%ilj1CXwInpeah5u^bhqU*4amX}|FeDJ-H#o@k#e73@nYVJ28h(?4bl_np^BF8U& zANEVg<6!4xO6Fk?`&I;j>a^@(j!dMDf#@&7vty^G;dkKuI1hh6h0j(!!iVh#7J0Ht z$`abfcUfKV|2`76Y?dIX_=je+*B$VB7jF z%eBBH^*KuGkfWT{&(Bm8oZ65 z!e2t2|Ksn)e+v0TLW=(-KK}3W@Dm75rO8Qa0@QbjzX5#0FLR)!jZ5Ks{g%BA$BdF|!s&Se3kW%BjQ#8FH8Uk*Yl3ZKn zWBtx_@a-AkH+bNaGr?)=cLjb_SKtFZ@T^R5ZUOEJ{OxUNV#(xu2@ftOqva{u;cm-lrDDv6U2!fmgmnco%sQ^@9uCR6TrdGHAZ4>!A`zDR*mkgrM0 zx`IC1gZ_hprmb~*yrVTcf&O-@=Gv9{b`dzrw~Y-*)D2pXV1`g=8FwX=yBv5aK11d= zdhk;atRKqu^Q$`Tq;d$*i#iIX*nv-t|B@yA>NC1~dHD8E9oVNvy4sQ|zggPOC-$|- zq`R@8Z}*`uLaG+d6FzkG_V!wz@182ZsUvyg-C$(CkO|31$KEq+w%~Ep1)tDI{vS)MR4Q zsP6*%+^(==9`+HL*tF>#vB?^z9HP%zU15KX^-9h68U&SywM-7|6x%10r@7n-6+9Yk zN;Z5lx!c38RM=d->(ED08ngOv$2z!?#Uaf_#C*qxKEy-cQ=wzD*PIvcn)9|Tnsb(& ziTy4TwNw@&7+HKA05dwN_>UrG6w2P}Rl+t>4ti&Dx!yxRm*|VQJ&D{a*oCO^Qc}kH zPe(IwsK1OAKI!!H%GmaOmZ*qEP~fdr(IZrmym0x za(UQ8zm@1T=AIy&3ZS@_jsltB2X65-X~Y9Qn7|bH5Qptu+x1_*>w@Vokf*@%AKTA&jfZ^&f{`U$H38qQQAaggcoA!cQ4jmk{YHI|hu#N4B^7N3 zeuq*290xsi+LWu2sHO4>f;(x-&g@spft9(7cmUE;u2D{S&}R}_1g+UVNy*2PxiILn zk4HH8)T0as;}0J6j&HI8b-k2D5h;(;>+RZ8NA4A(g+4`l3%R67CTD;zLYfxI6A1d1 z@b@^3*Q9|rXMj)fz@rF8mgv!~h59IKZDM5n=Bi^a>Vv#ky5X=y@%ZZY>(?&-pCDMz zi3W+>5~REO>|w_OUP8_^rSX^WuvfyZ2m;)(eCg9_)}{-~JJw%62e41}&+xuBqlqVr9Az%zIz?1ml|Bc@{y@{NhdwO`2HJmrieT zu-&~$cpd)byb7;`QkFnftfviwa75i0Bk*p$($;48atffvWkTn9&|kH(XAP%OAe@UW zT}#ie3E@+j#mJPV@qWoXheR!z6oU0!ApwH8Gh|=c&0X}Vv;8Iby_r#734;(Md#+;A z=t#rvN^!PXUDV-oB-D|QR*KgCxJlw%9Zkpi{*|6gZXA~>F42lqC*haP=+uwD+`NHLSErP12?qr6> z*{*|b8!v!d3e+p`r7JTA^S^q~*AZG!HQW25Ny(s}bszp12cL8q(8qev12drq?HYRX z=bH6Spg%yGZp5Vsik2@90~5VeRRNoF^eoi+%%}}Mrtf!X~4S`o6AICNX_z0t>akso`b z4|@*}yZuw)8^ZPQ9GQ29`xfY>_>{=QkGl|wS{`#1E|(pYJZNay@?FRy;m7vNV~U3z zRoI;`m~_&H1O2#u{pjxDe*KA(vupA?m9t+z8kk2Gc{NY+THxverbQd$>scA1f{EYrXV&7`M{@mFM&Kw zARqWxe%OP(m9S#>)JmqK;He3=Uq$4oVOnoLC9ME}t7I|jwsD+?zn{W~Vn^yJc1NjE zrsF71z=22{V@$)|^0CK!CiX{2(~Y?d!N{WBlqK=|i8wkWej?Q{!%w^~4V;NS$;|Z7 zFG5gpsk@FcC8bS%x%e0#mWHi{I7lW{ieu0iR#{IF~tGDVjJ=Y2u$b(~T^yXSeZqX;WXtkr|&ihOj@ zgnRk1{i>JeVSn`j6kFyf0PGKL#Y#aYe(a1js71(?#(96Apr1ve7RVd~9nRCNREI{S zJd|kg-jE;LFO@MKc5x>5@k%OPb(-9+u)o{r^8Ge~J8ei)L;e%wA`ZPbU)q%6 z?cD1@-#}>q-vAwl;B)TEY&4&6E0P8|9B(!k%y0H5W7PwfECvw^z;AKC?Yp$Far!FrkVNe^=R zqqMH_E=!xVz|v&Caf74a#5{8mIgZ2^ShjtHG|x- zfc-Pi7rl%`Es_Tv*wpwEvd!5rMd&vuK0|nU8gx@8^dt{@1nB^|A0P!z_>nq@Ovab6 zARlz!uAqN=&*l141O-TQefX}qo{ub!Kss>!cMtk@LhJKY`mDaACJ~uT;k21P=#8B> z-A-qD;G>skilo#7KWyjVge745kdxCD?6>c_;#h~E%Vd2}W5$M>y^bGyn&$B?9sBI6hZ}d zWlyYke7yTtGu9zd>vt`JK5!m3-W9mn75KFt_}KzZ4}u+&l5k|h*6b*e5&!A;U{qXJB~iw;1s|tna~${&}D>{ zSpx`10=uN4OEa~fS&^9S>N6f5+_UC)@G|3OX?$d}&y2~Re zlj|3ess(Znf(ke_0k7<;6TgDjg&CaMC3?AseoQ7h59RI(J-RvM7Yo@oJKrsLQ_3!@p7C zi(5AeFTC!`oSp7~W7#F5piwi8JHo>~fVf&M0iG^(?PHAEqD11l7(c)1w&=KPMX z(68~(Cm|SFbYe%=Wzxv9RMq{C-C+)TYQnEfhYEUnVtDAkyn#CE4b;fu-Q7}DeKdL} zeRKu$iDxJHKlt$fi$pD(hY?h*sl@yH&dpxtNB46-(L+Bz9UVinGL5@3UAy_w{k(5m z>GJ*&f-diZ{(B7LzmRLElKGp5J~JI1iud~tSDo%h_e*Akhkighy6}EH+IClx`Ef-T zynl>D-KvcU`X%G8m{VmsJq7ikgY8U4M{DN2$>_~q3FoU4a4ZLoZWz+pg`0Pri>{}( z%%+smHV*P|d%C!(2?Hr~H60!OeHTOBM=e?^!2CXQ?$v@sEtMA$RQ(2j!FjqJKxxSR zqtCv~PQ%3r>VHG$Di3#p!Udw{nFoK!OBXyJmWJJdXT!t(r5QZy(K&)wJWHqK>)5;MHj9zVVH<;xzagJPqLxU?MHfVh zgmc~wug_7U+Ly!nrvu&z=W!4FC>>)A!HTS zeX{v#xy$`(1Q+w{a7F=a2Q^TP{$AHBZ3*BBb^7$nR1f8JOmzmsE8>?%VIFG$q}&~4?84BlK?dEP zpwFCpEkvpo%UlwIWg06nRE|38^@h~=KEj;O09=!fzAO{{To3&u1S3mmP#}6Brl~s| zQ?_TP_vbikzTQsVKC6v*C*OZW3@Zp)Tp*2P&1YN$o?DH3(V zHX#U9wXKyQ8Zy>V-ycmw?jV&rJmkxXtj+rG0*-bCe542555dSHz3LC-G;Pt_wVgBm z`BE(sQJ2!qK9Q_Oq87<32s)!^N5;2wl!)mE_S=?gJ>atmED8<=WenZ51vt`=?YAy_ zdDwOXgtC}A0@wTqh(xyfz`kEuGPe5GAeZR5h-B0eK9MX!q87;$2rACeU8sy|(PGp` zeso_Ss!YQcy#qLS`v^oB2R-ak1S5;-jB9LH(KB8P(V5AkvV41_mB2(F_dAxj+<%6k zG-y4cU3GD$yJ7)^FAusFS<(dKhknw7zC%O9;1c2Y_58ED0w3jp4?-}auNT4891WnC z(VQxv&`qL_zMB79&(nvSg?&=^62C;EZqzprtXJOWsE^6J2wqG93Q`U)$J34q|1ff< zN#|o<3DsT+mm;W)iz6NuA$U|AuLz&w9PdZ>%l;4#y|+Tg2my4r0=q*02tMUbd2d0Y zmd=X^iuuyjcf^ySd z5lBP9nafMTM6D(?v1x|ukycsH3f6(ltSIhxp*FI#byhVZQ1+qlphY;9cvcrXr6yn! z_6Fh!G#Swdn(yKPvDPWXnUF1)K1Izpk;K@ciV7TRvO`FA#^*=|5h?mh#J3CiQW^A0 z=L;});xTyA`M&!%BaV}NtC{a3M1ai?5o4}XExr;_OY~d>*(4I8d4*ZW_fS}h@(GD5 zNRPTxqHySgB`T_$Bx3UGPQ~?9F%xLKC@?0MV}d!kS9l#S^Sm{OP?Cv<-=D+EQa0GwmS(BwL(Q9t5bTSH~4cn*$RIClpRZ5 zlO3!c&AUQz-(yT_PGKi-0c5f<6gd~SKl)VYKUuA<=~n`mtZm(p{w()ZQonEtZYVea zujm|BDXw6phOa_N)C4=xOI*QyK%xXL7rZJp%9s4;!`6$!*h&E8YJeFMgGpfys?`|s zGY9>~{gj&nxLNn}xhPxir+oeh8Vb2_eG^jTrs8`GH=zY^7yYf!;uf6O`)^hNV#A$~ z)1#@ArdeC_@Z1nfDLQ(b<@_^=e=6uel$VyYpabza#^>oJEeu0#(MhIH12`XtQX0jG zDdLrwh=5S`Kv8I6%+GPxID1D=mfg_S-!|?*2zR{Jv9l%HCkCS{c$M57@CvqJ>WaY? zOdadz8U=M9ua(D(p{0u@uol{E{TFRw8+!uK>xTQ|i86EZ(yqAe6*jk-1#ceDD>uWh zP)yQak4?qGb-ch5_iTExxIHs3DQ1rsg(e%rQ>@#OChc<-a!+ z(0;hg@nT=^iPd)hiTuLm2HQ9v$Swd_5jfpC11DG(&en)S^k&k-tf$)NmG7cV$na+|OQCGp?{t97F@+ zxQ;M%#zcH+U_AsLoWBj%As+%ns`rU6ISrb)797TLr#5s5!fR-cm}|<+SU!UJ99niy z{t~>ockF`K?AKy7@y7DDPl+2w1hgH&#&Xeqm@ComC=xMI!`;17d=$#sE~UR`8*5)= zq1eA(!@#UYvNgr0du)Ub{Nujd7*uhEb-tKGYGL9JyuYt``8?Hr)?i{egS4Bq#$v!sm6 zVPz@7|DuW1jrx#Xv)n(YE(*tm$es?ehg%1CqdMPkh}gGp{v`w5D24VCtX*UQdjv7h z17=%7nH{_!cT%}^YZgmmrzr=GZ9I$I9I%H%6~y+0o$JskeHfm+Z`jKaF0bBLxjQmW znu(0m#)^v=fI%y^jQ|P)HzuyRJ{O4W3}l|Nc*ELA<1(v}lA1CZs)eHDzTrMBcz}2c zW#+ard?&*E3bqsH@>wNwSsg}dFw2f&8{bmFAU#}lybQ@B_Jkvw$c0rKL3KkHYKf#y z%d?H;iZ1a5+L9uu*U#_7%{MW$Nvya^xcB!zWLkU*8JN_F1V_W|x)K0>YYeu|iG>me ztE#{nzxx4Y9mjK4;QDPS3p8mO^pa|W;JHi;O9z~<634(KA0Nd{)I)I-e|*r=sJWrk zToD~~4&JXG1W7(L2n&TPX$OEGSkoS54~?1)ppHcby@up9?KFVbE5$57`-f$oeZ@74 z4N&ZuS@7>;abfZMU_64M3M=(*vAsfs#QqktK9^OQ|MR;2AFs{rt|cH2ma3s^uZzuJ z%r!u}0BA1Okd0|r8_*Ik(g6SlUtMOt7c9kSQcwc=;HyxDzf%?Hb}A{ejF^4Uy@+*W z$zQj;P|3!smf)+b!1_OC2kHtiq6P0=20zTfv5kocBE6RBL24cQn7F$n<~WIDPdNu*?q$X^CICQP`F9N%X_8hTM-s=PRk2_^5Q4wqq? zcL+vz_M)|8a~jZ*t%(>9lvv=j;p+7n;;x1w=PZLf#`DH{WV2!>UT?y^^=m78rB2G? za4}iFv#`P%{UGvJf0O7Je6=)iM^4PFZl5A^wbymuz^jfd;2l$ATrwN8bq<3PC&`;m5}vhQI*aB{!e7d z6IIEIZ@8HvmK(Q$Z!i{UeE{z#BKH%>UDrcc9o(^;UTNiAr^?BOFeQU(0o-FdPim;j z8KTSS%W`OhTaQ%b3}ZPxrJUu+kWrr}5#`h;sLKJZWN_+pdwMybk@`H}?v&Gn7v52y zHQ=DG7xKorz;7F8x`jc(>qbLoaF4pgeN&SRRMD5UE4xQ)USJscA3}aGSJxX5$^;T9 zWE(}u1Y1>gvZ!6bRho{L#zz+rxa^`JC>kC#RZg0Zkh(nn=9@T&KIgzv$*1zzCWYrQ4l>=g_f{V~0 z(JyHS;ZGKZT0e(wM@>@-GYjr4=HgF-<}F_;TRGA;CJU}0>FMr;h^eFJ6seU@-oRh* zHKv7y8O`Y4WYjdthPCQ%w49ru_`#~h$PJgSr?gMA_1b7U)rmp2vH4kPc4p9}ZDouP zzcrWcsC-hY%=}e-Su0kf>XACW2i_AG5VkLy;%PjkiDHn|_$~ATa4ssZN6$|ok&KpHLLqVZMA2GT4@U;o zP8qcKfL7G+o5bLld7o;uwTS~9$vrtr#M3uE>;*zq_CV(fR_*weHeZ2bKv24ZMiiZ# z>DJ>xT>BCm-k#V;HIa6rxSSsA$u)XV?yo2+zk8`VpFr>r!z^OV^0gnVu@h zf&J7?B^rh>#Rpl}z)ZAVhpsOr@T+9qeFbC@phop}6mlv0|I=i^Sz-AePqwj6;*?UW zw%vdj6#QqvcV2KiG)z~V%YW>y7yaNg@y&3J(E)Vxq%dl*11N{6JG2HWS?r}>F6pmp zpXf_)Ol$BrYO;gz8!gunKF*INhsxyn>|1wLmo)oUz!;x>vyD%lq7>3a{Uu^_g7Pd8ei&MSFU@MSWRqq~E(kg2Rd;oUAB^c8)RALOWx=hAQsm&DO0Mh~KOPAoO zv)T+(e5TdrM8;&)W)DS`*17+ns2HR9R|f7g*w}c=A8N(RC)`;EBRuV7y0kwO#p#)I>OgXY0OSwHf;h zqw`uH1K{xXn@3bP+%gQGAT%xT1(DU=t^nZwA~*ezzHnr*JOM>sw<*>m$tuLSRQXL- zC5kDiI!rkUMrJK?kY85FP&fE)Ti*)SHzz;IsxkT58M>o?hC111lY*;#3qau4co$NOu>zhvYiXyuOWL%R6PoMq0qgpIMTupKwXdRb} zVMudSI=n6UUk0sPr+@+ecFTb{fZIkG?=)8w2g`g+K-LZ0Z?lzfDw$kdd4P=}DO5AwN9 zuBMWgBZn)+xgZ5u>mg}mYEn5Wo$M(^wgO~z*W^o&J|QeU;lp}?SeT(+FW-~G{ z8|DZ(5@`7Ds>-V^|CN$=rSX;rQED zfX8Ba9E!jE5G`gcQ4qUcsz`I>a5ODg?BNKrxyG(hLzf&z2A z`;`Je#m#}9$@x2~s!1_97Sb0q>47dO#O^1>ijt7d)THgKC715hrTyusLpAAmO}fe@ z48teyk(Vkr4 zj9FhHmu}Qi@l-flO}b?T1fsHys~=?ts@%<8`)FMQR>`CuQjXJn&T6&oWP7(tEXb_1 z#@&N?uZ78M3y39fw+bXK;3r)NJC!^@XO1E>^DPo?WZ4tUp}OhP!w!oGdQVM!mycL| zZ6}?6xTgOST~4j?aZlCxq@dTE3ICS za=`(D-$;44=LD-*@Rzy`k3{^cS%ts1+l|sD%Ow3AcEi(bfPZ^);(1if(-JnDOqp=8qzxaGgY+; za5W9XC)ugvYO7-C3!uZJo5qFFwu343}W^)|*!1|=Vl0rr z?)RnlLz}skqSs)q#xqkZBsH#6D|G5Orp{_?N3s0yGj>|kzwpG`FqFpnd?wP>{(nKX zv;SX=lv!W(fxKX8f>i_2rINp7CA#CA#(N=L8l3-DBz2pCHI8I^QSkO(q2?I9hh2rL zL&GK^ckIF7t5wm}SgH^12Vq@J#h`I;8L%BF|PE_9j|aK%_WAxcUo(>kBG?!H z1)aC`OvZ55#Tbh*qZz{q7GvIJ%rT7N)I}=(@_o*PMe2&$Nm?sc1?xC@5vmlZ(q*9& zL>8AIO}8IgNgTj5L41m}jr6RX>-MdHcp8V;BE*(B&0?#ze6(;gN>f!d~nTSb#m0gA%E_`aX>SL^QC%@Va{+0wz z2v!}5z^t!c16i=DnZ&(ZB}h2sYG zK`>}BR}8I9(xw;*41fGfh_8beP8SaQ?MO_vPpce~Dt}HQzWXas(9zAetlsiT=*mF^ zZ)UUyeyIYtZootv4re||p7xWBW;x_Ijxmx0h;3{XYzz#4MWC)#Vc13^p5n548m#Jp zNS@wUBv!h#1lBBC7>2b8qlaK1AMxb?h*id5{7F0MG_+fy--459k>2@$b(2x8yshto z2;-Kf#i;?4^UE+sI&_V?6L2yZpQ6SmsIl%p2{s@5GT(>LaerrX|3}L znKKU&lH(TSSTK@I_a@UA-Gx=BMmQO|<)vsUoYfiEmm)6;c@o=gWcX4RG8=_B48OJ~ z86GG+75op%0wyFeH(Ec7E(DG5{FTMzkzd7Bp6Gku0yIMMMN8rDu!b- zR^k?%LV^h%0Zca`hW^{{oiI^*C2K!5{7b8rngX#ddTASxW(2`XyZF<$rLS!38Ah(`4snzWgCrb&D6L;TKa zoQp@EfmtvLyv5MNXg1<-J!u`RwHp+#5%q~&-5Er6M486oLW-Z)D4x@levo4LLiGDc z>Xa)WHY=9K?;{2!_R*apvAgh~SABDlZNWJ50qvQHqBFH;9kMJKPazItYHHMPqCt*o zyuxI-2w`h4mAu23d>@l1GI{%Y$*uIy$$c%8Co%a0B%@}&%#)aUCR6KG<}6?GFeHN? zaNOc)T_GalJfT;|6MJm^*FEQztKGmDvFRBz&8lmyi( zb2HBIr09&>)mv|gPMzwW=cd04xAq3wXIP`_;c4c|!N9h6(3Xs$AZIEVe7^=DcGK`3 zGn$e+LctWxK8Mk1D8rd+;QTU(qF5_RpiHn(u7Ch8(jXu(m!x4() z8IZ79xy&A^{Q&mbHon4M1L|ioHy7|b$!#l=X*##~fjaj*l{A*Qxz^uF?xl+4Fr8a` zMxFZ#l@w%dZVPmhI|-5nF^X^LJ6(vcTW8|i4|yTJNf_WMzAGrc;pm=@_#WS#;#>1) z5x`&ZK-q9Nz>&?HD(N~T>1}`y5VN2h?9fge363)D#0w-ijr{J8ROzVxQRdGiIGqHW z-*ySoN101Va0Urhdjw$09c5x5&{}_rXgyZVjz%0>aaA9w1esFX4!sFhMzGre*7@k; zi0(jf9WX9;7x&?l8j;>!vVX8{!DW=*Z%|=JdY7Oq+qef*3r=PM`|q>>K<4`kkpgC_ z0v5=Q-MR&rvw%m^3g{yR3;`8tmZNGW=goYb?|ha1Sdl&h--H*a`s{Qx6s zWY!IRHsGK^uo{cK%9&L8xYlrkZsJ~a?C8Io20(gH7vw#=TQ)ZR5+}t{$IL^J4bpqL zZRC^Sg4K06j{(Bocqd^iwgF}HPTb;<008MdMAwWs4_VU~IvWhx#ylipdVUHX=nvng z9#6!hp6`#JAZPHV$1lL)yDf2Z1tigg4u(aO;QTkN!`VsIDFU4IICwTDmd#f8HB9Hm zWOJkz+kIi{b;z9GrB60akDN34TpWXaLi1G?@aM8l1G1gOOJfX8WIqi!ZhWc*$Q!dbTQAEfYzotl=QsG~?~uCg|;7c18o zu@@i~+iF*2$5W%S6KCsnZzNobgh=w&iz`RheTUvIW6u==!A00$)$Dv{6syDc$B~YP zJOSxiO|m5pv!TNf*Lo(10X8jIeJVp3VxN;mT$J;0HR{JXjg-(E)Y74CqX&Ydd`J?0 z0GX8WJOa_7gA;qB6Q0nUTWzxsuY$I!qlB*@we1f`I#4C?xS>vZNDvBossx@9)CUM> zs)X-*37C7z&o(Yr37b>`YoyQdouU$&bOJmYeemuWmGB}G^fX{NVxs2k09b@%Py_La z@O}j1-SyB=|5WL_FRg~ZEwiI{mOPhZ4S zbCk!9BlmM5AZz*wvR%19{JTn+4&U|z+_ej_aQWM6EOeB_u{g;bDEw#^uFmQ`l;nAy zoZBc*CtVJhOJ2E<^i^=!$R&73@f5xW2#Jb}zOf{P~9J}rb;&bVPO^5_B< zAXIs8V1+{aToELLgF3`8n>vp{pI44M!gAvLo+5d`%hVe&UNd#WlWoj|yUKSqq)K=v z={YG13|xr>F1JEpa()7@3mvG*hA&O*p#^jn3>^FYCvWoY*bl892T_c-92MLz%8Jfr z*7yZ0e&lgv&IcID+Ex>(idcfe`6WB3&P|=z|!ozXb@Q*XXkrw`vS^8lz>FQ$n_s zaGp*TXED8m^&mtE5tcw7(KcR0j4t5?1W|%Mrm>I4P-GXoCFDv8c6U`-aVXPExK@?$ zG16u25$Dvr63SEw`h>?qjiJtfQr$$(@W^5K%Da*noOM#X+ZNM?g6L8=j!xJnm810|A}@Nw5-o zIVJih15$}=naP~34}NUUQ);4$KR^LKN5Fv~(#JklY9htMZ@0ko%%;T5J9(gG_|nRO z=+Qm7UzwL|&#v4JCTXAKw%o*kMPO_fw1H;!q3$8J1c5OKz+`;86#QoS=vK8+ZZnQtVHK zT_=$qsD2;B+r<6_?_4x-`h-d?xsf_acd7D7+xW*#bUtgkTX!tSP65_b_RS5? zvd7|O6mD4w%F#eEcS~-&HF2=@-0Q60vbz1!>qkm5$vTd1LLH${IIEvf#c88_ixelP zA8-sHw%fUl7E2Y3;uunscY8Kw4mEjlH9$>Xje7dC`Z3kd9QBjJ zj}_zgZB5>3jG`yY0|IJGC1rdLdyBeT_bQqZil%aRZo1u?=&9N&*;oVt74$)(lWz|4 zSyNOd#+3tnjhWZ-9SSJH3*LD1nyb5(M9-5qQSsg)R|LoM%`EiH4KciL8p4|3 zzVIQ}=w4B5tzJqCcEjNdDPpAD4jJ&!DvwhH=cn!Q-Ps76x~z2r9V9k_-j2h{ueox0 z6uY?i6Uwrl#RQR7T(qV21yD~-o`7KvYag;9k9l%$7Lh9W34;XL5zE7~Fh!hfhO7}w zg`*=Jjz~w7(Sf61mto>MGP`w@!xC&3W_{qtnuWP*vVBLe>L8>+vwFen-GmB8YH|xH z6mW40VN(TH?u3i^4Sp_2*C`j*3KzMppjUzsE^vT*rL`Th5H7gxpt;zLV5Fwhk;Pn$ z0N|n(Kb^_qD4?Qs4s&Edx=y*+{2FD^t!;#~4Ooctl~TM9qgY7vMe#i$gaG7$((!ug z9`cdbw%&5$r!7zQIUG}%&Rv0}ucI=v8mVP+4+7twbM_-=A`+^1$~T_sm6_`))d)8v zx6z!VDLrND4W^}$t@D>ibq-wuE?Di;r8>)W(|Y^k*8ZrD+0@oYYNKwowvNDBI8Rgv zgCWHrdEvH&3=uMpaL1If;Bem89*2BSya?|lO5^Lk00yqYaOcY1T2?RY9a7Tb%A3O> zU*1XU5?0kx_Jp=Ql_ln`=Zfj*)h)26_b$ae#Nly9BiVRF%~@tOk}KaUyfS`rC<|kB zna=S9HDs1;WR~xjhsJU-@fPq)34!~eS>d}vtKy>T>BiE5SduZ^IO)L=K@pYwBJ_<(Nc4g%1+$MYL`bGdD+JJ ztH{$<^xcxRJc`8Iy}GMY=tVQp*z_Ven%YJQDDjp;Y3i0N8N_+clFC>is_+h!5xsqH z#^8zuDdeMwxg%nJi+y(_wsnV6hO!_V^9{U*PV#RO6(kP}Io^*(9CR`o<*7LcB2LM| zbBrMpuilgJl9rj5Tx`d3ma5J3rB(=P&oG-qa1vNY6IOzE4WSjbD|6xZRKf_&1i!kq27z*sx-8SC2lG?yGjv)&$^&y_Q8;0k5(H5|9au*5@V(-PV zU&rFvHWq*uquJpLFp&d@T3`ShEc%$-KsY$Rg`C0MvE)CY8MnZDLnOs%V~Vd2PIVZp>U)(%gVI} zvgr!sH^)rUk89Lpg?@aEk4fiR6#CZ#u%ygGjS+Qr7|mV*2ci^RS{8!YuUqTif-e*i z8y+*qhQ^w2l$x)X;kIJO&&7J~<&;~n+JxBfejuc1wkO*!Pn3-K3@V&BWW)eKV3xPi zZtF!k9|}ey=Et*^*J2TP){bc>1p+gkY|BSfTMmd&OgK7hSmkgY5}XOlXw?aogE0RZ zR!3_#Ha~8b2I_`5x&S_kKm^N8QNDu~i1s)E7CRdzP0fo?E&ABg>0UeXob2^v)uaE< ztR-5{l+&Eq&8I~Q4bfZ<<=gks06ebBj6 zmDy0sJATIC8BN`s6HJfj%*2L!8@!Yaj5xxv`* z*Mf6i!2?b7=kW`E;%ysEcy{;SPnx+%O6?h}#!v>MT3KkW!4NGQY%fg{OVV5XV$+J7 zo7=iY%t%>pE~vmit4xG|6I~V2vf2UK=2 zbjJcaxjI)01n<()lWIoxGe-`Hnq&HzXJF@lOg^MYfmkhg(>Yc%vPg!#XCP#@m$^Ki zV-0$fto^o0u=~RzHGAG|J&#qwICD$YOW%+qS*asN3Em zELx{+TJpE-$_pq;X&xCy;Eu`_mgQ;oZows@%*SBe1LB29=Kj|eN!`&{3&4Q}zl5z5 z=EI9bF>Se0hZ%)Y%oz_CWr&(f)3iZZu`9O=c{G-@jq1=Hb??|8yxyQYfXaK|FQP*= zdq9^^mym6=KpR;r@QaXbw1R(c<>LoDL8zwcQQSCXC*r|vc3~PH>DQcZ;+gYKc4--| z_>19c)&j6MtC7Ydf;)1U5IlR!msc4%(3&T1WKk=F)<24uO!y{!) z5&0Z`qBXbnLCh;EW~Rhk*RN*ufSB10JXqV<0Y`^oAm^+WcRRXo}@YS2F5{nG#N5JyX0AsEN2;o#lc z6Kv!DKcr*U0^x9F?hI*HEIjC8d>UUt@r~s&YYDsiUpT(oByg7Cf? zDv!cG^i-+VJ^zN*>-`1JyMpsqLGR_jl65Sa3%lm6{jey-L2|AzG*8-suH`iH!VQRE z6&F*YL+iP1kN^u@`Hq@*a&6sNhfke?TMh3gwKVyka2CLvJt?;v>13fDY2C;-NgSJ6 ziL(qAZ138uhMR z@fl0iS0EY@*mRTK0I-UuFh>Z%Q*1NKP{lHPC*HW2ff-%RiHHta@n5NXv!}5NGQlx} zI9GWA^{5xlS0K>VJzi7K^!PCOsk{mM@}g+i#ykcA=wC6-z*r2nlVI@ZwPIZw8p=<6 zpare-#E>5kJdi^r-+PH|SHN);>VFA5*}A=QRTGjo)wixj11p0x6r)|Vrk=85!*PQs zSoJ457rKI}I~iw%K?gft^)#4Kl0z=;yVT51;IG;jshQ1v5haeA*)tHvzNgd|)tQPw znTb|lojB65P8G_Cr&VZAsZg-`OSGtM?5`@7ttz#hb`$r#!5!_SQjh-2x!B+Wi0t<$ z9&08owFR$*n!+%&L+WrpYX!w4iKBSGeoMu(cByz2OGm}4(9lpkg+2Rj(3NqeAx-h( zsaIzr8eHvM@yL@_Jj$3lS1hx-u{9@1b=HZ|n|%ipsd#1PZg}4Zf`uiwjhhh=#bY`Y zk3Uqr*M$HrwFrT9#bYChqnECDS59#h@8pBp_!TFF!M+SnEA@ z@-&|0<(FU4H0g|>{W>!85>|R3m0`BA82KQT$6gSrJb)npQke_;(Fr5N$OzAfaZ(n? zC^n7Wlo(ui6LXQk1w5XsNVXgn?HSttL4jAfJ z@9dVc;mwu1mzgcrfiLmx4wxQUs!XO|BkQh5hOnMX9L((0`>3?ULMqW3W-uEQF|J_^*_CR?)- z$6&uQI6wE~smbbxfRWKIb4mvI@R7v3*AdB<^;IWZecuZUFy zC`#5tig^R#7A{v}(sPV8^ckAsf(>H%3SOhDh@Azz}DL|^x7-96g4ffAjE+I zCI)M*|KV`Ra2WrG;=ec=IQ1ApIJ7+Pz5k)xV2foM|6Ia9m-5e__~$bIxg2*)4mpcT znAk6daiIcrdJ>sZn5$)Kv62V zMtv+48kGmk-wThWjdAmUh>>bmotmGf-8|Vw-jH{_B-p@e<(_%EA$67Bx1*a=j zut=G<6rEl7LZ~mFM~}Et6#o_FMb2tuKbW%2S*FA7nLMIMf73Q<&!Go>81ePJ5Nd7N zHujY`*?>L~=P~h={)S~AJX?blA4e-bK7`5e_?QAdQX{WyxZq)Q?J{h1 zsJBXS9+7Qh&7n@2Cnz7(_=$mO=kw8?5c8VUKWAIH1)$AXlf#*| zc`BvC+Al3-AyZz1r)L#>!dYvgmpW=+f~;>Ow`b94ln3uz&7NbPf^`k!BD9;eS*Ey& z{y53mCw0aS^u;UAgJ;!jZO2R$m+A=RAW|+@l(<({DCNY56;Z@*rV)P&!)-mJh)aCL z{Xh)H@`3&+Gk9hGQXysDo(4nvZ=Ie|CYZN1-G~&}qd{724{Z(tOB0eIG*dTE` zTodr()&_26z`8~e2hxbwV`gHl%pjgg;!%ot9hAe7)89z^m?A!A=7@7X&B8kUfJp6D znmY4cEI6*=Y6QkO$&{MVCpBh=_1jfa`9G%tokXDV0u-!%0;TDOIDqjZbUe>n>-cQO zA0+XWl{)StxUAOhD(*ZTSI4;Du*d~1#nms;xc4)zRpNqGG!8RIJo$-cyxnDdfMWa$ z^td--n(^GTh4D3vOO3$>z=VA3?on``%=NfLq4J=aBhC__fy38R3O{QwoG(C2K?)Bt zBSyyRjkgRz_jcCXDc zP2jpF?*#lL@~rEyATFMoZS>;tY zj6|9s>lu=xk3Gg;1$37Jn!_0VPu(xht5{)Prg23}Yzzh_a3u`e#@&jwBj8U@6u0SO za3CuynghJUBkKS`a3BLR?*QkDazwmI?r}$SM9>EN4aIpq>eqE?vSM~9sh z(nnodEl4?Po|L~JfVQH?@krrJMA^o)GmuRktD0nv&c~Q^Ev5@8tdc^!Lpe5~$wSt* z$4Oi=K}6aQRl;r6sWV_a;km>y^_C{U`r8T~L~9UPSpS6Nah#&xGjWhUGfLgF+paS9T#vcpjgDjbvR z{Z*wnwoKxpVVu5(UT?$LQjxaxEE?tcr=n9bT45|a1`P);k27!^Gr14E9NS+g zyku+Yl-!cF)^mSjK3S*3YEL7o2&p~&IH$F*^Q0wQ`ep^rrswgm7~7&j=0NPgBIwoZ zuXLdXf(?X^X0OHnvk+>(_GH&@G{cJr$g=y ztxo}31ay2QJ`_HZYtBOQZET0#!2*!x##;WcjBmweuVI)-c0#`r^u0X#`-Q&p zE!;N^?M$}!O$@e;`3OQ74?d=Z;bRhY4S^px_|4X_4x@hrqdZJ>4*Djx=G-;Ca#syp z{{qU1yPdtsiQGOj8BR)G%|<9CJcKYEO+yrsbAm8oHI>=ZehZ1$!k!AZ-4YX$?fWH; z2eWOD>b{Q=J)19U((?FLsB0i|0Ypbh!&D|9qadwS{?f&?VTa%5^6#=!#V!xR`X?)o z^-nM$W#iPoQ#NW7vUW$gshe`zhVj)M+qlimSgtZ2Wi&TYxo7_U6)U^fOt}WMi?n zjVkTM{xiU}e7Fg|Rn08)&Y?Qm*ZK*7O3{gVWq0ChY9g8pOXmMLMf!HnSpd&LFRgrQ zYqA#HWKk+WE(Etl$Q!^%MUW}P;ozO$#A-dGD67!IH9_iX$2EW$oUWmj z$GH(XXdwq}^jWLjF&a#R)6b(XbKzoIKRl$)+#$YP$8SPB4m|A+*@diiNGCBPW6NkI5Z#TpuzDXX59?+!Cn|X5wd%3AL!SrIQt&I6a$s0It;BvMp5?L z;-t(eY;ek^Mo&c9s(+?#!f>QgITa4iCpnP# zJ_*p;PQX2z&~)%M?IG=e@!~CqFL+>ToOT!%My+2_2PvdvEsjDAypQVQj738g+#diZ z^F+Sxl{QL1nl!$3Kxo~-N@M;RmMiB_Y?3W;pKS9J|KpOUCs;NYLd z4eu9)h{9-`Y4{KtyWwE8j00j-OXkEVy2q}6oQ|>yi;=L(9Z)RH1YML91*H;S3pf)} z<3Izi;}irTxFa8QYMPZ^Q?MTb!_+A24G2TV#Fb;vYUZ3m_9r?X>##G7We|2a9;$IJ zR8sy%?euI+EbcxQS>y!7ghqr^f2=?X8v8=@Z>afiXYvVhq}tcn6ce-|WLh4!Tl`M z6v&89bqmD5NYXF@JUj_2UgGa#E<-+kUX#kjkwbJ2rTA5flJ&NYdq+bcOr!r_j_kuF zJ8cZ|l6eO4Th0}15teOCK_&pBbws*oonyqwg0Dzv^J{m!e24au?K42x%IV-8wzBRu zkT`9F+5}(aB33MQ#6xTw*gWjlh)pX9F~Zka*6Yx&l8w?%H?f^w2e)Xa$MEAGP(_I$ zs|s>8n_7Pnxj9Wlj}z4nOI;}%)nxQ3Ga3sJxL!G!c}(SGG~6d0}UeCRjWkJ)g^0n)09io&;hlLELAMdDeS3= zZNHb@=JmSmq8wg%bMIG159V}nDJB@BvvHXy+7i$I#Iw^vkUCoE9*!eeXQH8#7LtAh ze+=h@CgsO!ron-p?tV!w%#HGys7INJV=$kpTa02g*S{4{#n$6RV%7l1FwHlb!t1FR z@tSE^lR6Tm!l;MiHG`uyN6PU)bjbkvf0^q4sE9aMtj#Z|4g5B0y9{(|AzbvVd}M&| zin}KcQ!vUz^9v-e^UYEkXqDj!W@u`C55-4^9T}{81A&;iuYPf3A$l72aAb)S`#g9u z_rrZfDps$)J&vas-Zg?QrH7@5KJ4&42Wl0C6w| zUwGM!cW-bO0`C=~!>V0JZp}X@eO^YPgH^RBV2+hmZDkW9U=a-h=gAS|o7T+_K*msh ze1ZyBm5>g%b}3a@1tW1ilq-O~AbY}4<$Xu;{j*Kbi3*$#K(TxmeScvF&Zw+_4#E@O zyvCbkIGIH=hrL)1Ul^SNf9+I&-dU8#Az@laEIS5}oBKoop037}Y93$_HIq@Y3UGaP zWna-fuy=i1cc|F)*@>bkj%#72!m}UN6DZv_lIVSK613i=@Pu>t=in>|7S#lKFV#JB zhcCN!4g459p{>XN#bve0`6wkc+hg@d)CK3eL5+F~zHJoaNzYMs+R9k}jRjwYWbXi@ z3`|F$2wMqwjU%uF{Xf|l6GKa5i+^f7L@s=xX9EpSY4FuGB@M8AI9SgCp$eBDypJ&< z61;CIj8DW~T82;40h8-cAvC6K%#2WzEAa)ybJTQU3JwRE=-a61Wp_I!5HC>CAW0Wa zdn=$9Dk>S_XeBe8XYd#Kca|Io5@zv-^X38k78X2s3^EZp#0xH*k%!EZG2FWn7OuJx zmEQrwY{NEwLZ<^)@7}cwt}4+!V91&OcxzC)|LFJQWMnADkhFpNeYEJeBNq%(U<&8N zWH>D5uWgI~Q}SujJz)4*ZFky$L-~9^vVYaa%D(I&{v?e5it~ZsyksL@7jaf0#4f@U zB4Z(IF~ltsk))lCuUVx=?odcce$$XL^twJJ5>nrLii*XC>Bja(Y0dul!cxzYwPiJZ zTIYdj0vBTj7sA(9VZf&*hXyKpPEDfgCUPP+u9#@R4k0%2{ueR-ARkM;3Y}OP4}GWi zG(?aM^zu^KZ0}LBwx&N$?X;a%TK#$AWPAt8Z2bR-d-wRLs_XARR|uCdL5ZS>M2U(5 z8bxhlL}xU}1SS>lsHmyfNWGQn45EdaI7EquQCe+lpIWuG-`2KP?N=)pZwVklQMp(Z zZxyT3la31BxM`W+`?L0$NrJV%=fB67BQG;K(Oo2 zejttDZ?%6@@&=mPJBl}OG`WmxN84frjA|0T7;VF7dKv$^3hIeToY1K)R53O@;?gzc zd6?wVPp*;5{U?9^i1st8Cv0ti;mv#l>rbTZNX8thZ(}=z64j9l<%TT)H8Su~4BT6t zyZ^MhwC}h-@?O=k(@lU%XZGZ~H+$=q1#yEtE6 zSrFsq)kw0oG;uAD`FR5q->+R4N{Tnpc8Ij&N)t199A7ywab{iZ@@t3&p@`?J>%B~5 z?5cY2)27e4MDjk<6-ke$*#F;@^k-`xpN<7eq1x ztP@C1vogyfnT~qyQLRrt%fA);TgJbxg4&luH+}~;q{F9U6kHr0q2VO+3nE^P-2&7K z>D$ivs$j0?kx=iJ$YDz*`e+&&dw|Mmf#YDF-j@M9Lrc!$Fxk%1_`dudm>5v+^=oQ7 zIzLT_9DWP~y*ioSXZg1|Ke1l(+o!r0@CTDu`)mZoAJd)*iW1+vZd=ZlNgh0!U39HF zW+-*CqRiIjCIpEZX=X_kf3M@kH$n@rwi@ow3O3tJJV}x$X`?9q?RuvOua<^PBh{vX ziv~|)Ir8qXjk(AZnt5@mYIJNw61x>pyozLMhf(e& z=|ik(xg+*%S~d8Wt7%b1`ubk&nvYiM5xJG^-mVN<9S-KC1tczn=fVv|K^ZgrS=x`f5C6YTa!ALc(5?|zbbeS6F3{_a}ErFhSv}p{Y z0wVw?F5{G7s@^VHEWFi>6|2_{z2KJ|O^@^X2i#~P(;`MB3e`44g})r_5~BRA?eDCl zNDQXIsVi)JQG%%K2Y*kOApi1(Y2#a?Q2D2xfn@I2&LfE)G5um`v_nM_#6pkh08zWB zSlrqlAR|$)Q7`v%oqMm6d3@Pe)Esin$`Tq^Rf)snI}Hc-;^$Wyi& z>VS#3o+fyjzdsnjTGP5fDV~nA1%cXkPO``b+FcIJF(z}{30zva33S8%g(g6SDeymd z7BI~$fd8e6XUWWyF=?AsM{da`>-qQIB%NY?jc~wNm?>0m@UEMV-oo&&_~g^_;_Tc$ zkj)h+Hjnb&Gs=FnUCCw>lZ zre(XLFIsO72ugMKDK#it>K{kj5VRlCyVMgx3EJ}dl=>f+3Vd8CZ8h{RbwN<-tUjeC zXG_g-rL@n`yVN$QR&9;yQ|eup$~}mbf`G)!na{i?bgcp4TWG<@pbc*}$|Q_Yb*%oh zH*BtSshRSxCTNr?asP-SuM7uIrXz7MThg;t@{>+Vju0Z_aZ_CDK8_1M#+PX}Qd*TL zUufkwvg%TPK#=7xeabI)A3rn=cC~-BE3d^Z(z2tOj$0K8muqE^>#9CgOm`pu5PVes z0E5<*ZwA9%$e$X_icflj;U<@g{&$ptK`T*i7^k^rE)Gimi?h-_U^v=+oD_V#l0EWABKGMLTg)s+)DXy85gOcw((;E!q-N%E2k4^LeFlf2X zfnohb14Chu<-R`U|LQ(&5;Pg$!LGbk{oY`>)aB|3ay9m;Vv75CU+_`=YkukuYRHQZ z1N^tcT{yC;+BaoNg5F0BR~zv@ zC0=r=#B+2dbQ&UC;wM3gllqjHmn|{VmC(VDY>DciL|LB_2WCqgY2%O+xEpyl>UA*k zzvF%1Gu>-V@5ZeobA&pbQiLRerQ=^a-D_1CqPoIpO#WE zmk8&{U>+z=J`6VQY6 z{!1S&VuR@WBdN+Zh>rOqzuYFoy?l|C$=lH`dCx?iy;$gisGKD__%oJqVRNg>*J0Hy z6o{pknEcaP+1nh(c?CG8Nq*;emjG?&w}mO^T4O!+m~#YMk(99h0b}1qp91}go)AA?myO&Yx|)kN1VlG-##dhuA7%ZO#GAjB{4A&03YEU{uZRp3Q~W?dz$ziHlk?iW^t8R>Nas2Q0xd>JWoZY zd&Sv0p8h}_s?!0ELY5C@=nWDS2=^}c$DQt+i<8VEnqSCT_uk3`wzgv z604jsx|0(SC2hA!E>9~2(oN*H5gmr3K^Cto{NMjj&6rBU z;mul3()||Rqi43jzC8spAhsc4?aIV~Dij~mw&X7Iwk^@JlHRxN9(%jT-Uj!6i=}SX zGQx;{!*GKRJVbr9E4hU89(^N=J4IK1QBRgdxh(od7A+LiGOQ=d_i|bEjVxLk$g++F zGid0PTo(834m-hZx7Oo#KGsxf?kndq&rz5UZJ_;*Iie`9+2*_YLru?Zk6Q1G zukM?RSgg(Du8=tI4gv3 zDG}J;G3=pbiB%k@tVO6WA9vlnlp#S(nMn&iiVudErf*x@W{C_}3J2JKhKV&fpj7l8 z?x?f@MoI=b^ZVvJpm)wMKxOKbVnNQ|W5VnO=mUG_Tx~fCOCX-tEbuTR*yHb!KIWI& zyeFncz0aaGBSSyOUHQaV>Y#r7sP|SV-k^t;yT}SgmdbwN|L%HYczZ-MD>KXMlgEF~ zdA*{~6R)O0Yj<5MJh|ZzEx{JeOsgTA#aftj3SFzCjA*v3VtE+Bf`AcC>+gREhvN3D zdNwg!=(OaA73^GU2Xig+KgoNSy({9p9Bi|F$%L)3)cCr7gaIEF&hySGN`6^iC&H`?X+gX&j3iIH%yk+bq#J|vP%!OvkOs+htW!LBM`Wr^b_3JdD zzP11v*ICv^;t#BA5-q|KEpP{mOOszw;S5TuuV4JHu&(&M>A`v{ivL`1zoAs5_HT)0 z>0d2=Wk+7#d;FU~pCbHwCwLE^h5a^chQ@#CSQW7j1}h*p7Z#C7U~gO|L(#Sd8E3Dll^$ z1u0mEB1eERn4ul8o=sk?Y7XZvw)RDU>g~TYa`<+2+JC55C^dZ z=iF2fKNy}9qs?H!yn@$g*CcuwxXPvfguhtdJkj2H%w&19GamDo4ZD^P)ihof*dP<7 zI{epgCc?&E9$QMdh@E%$+R2FJEOeU!|`LA*D*>=a(S84>qqP4i0m#?+4H*MtVmROZ- zkUE>#;$SCn=eSU-qLK%#w)I!~{+VZ^$(0PTi+`O3=^=gJ)nMybpb?w-V(Uk9>uh{8 z>)xuUtF3#h@`iC9f9t(r{96?n+rhEE{HDcjzlJNop@>0iZCCQc`nF+}pmKff>p$wy zd^k|bV@80CayT@LLz=suc{1B$dX2MMyRfS1&KxGVA4@lO=JmeNvgKVRB)h}$YW*4C z(%sCc;|I2M=bb2hv1LbIm}ft$boo-+%Uf>kuy?xLI=|c&^6dTvI@?h?yLpKI?8mNR zd}xxZHTx~Cu#(h8c~!~ZX8q-lelwv~1qNcACg2O@x3E~s&w1Yipa6u`QPss8{>=JP zZmqI(xBtuhl~(!Ne;`_efpWIg^f6F=p&;WG|M@NoJ3VYRDQxV?-rgtz56deIPk!E~ zXUgvarJNBueuj+Xr$lRa#Um0M!cN-O?w>@3u)3NlFQQj?0F_l4^l{NGH%-zRJ+IbFG<907UaIsGt4l=+^)`mlOyge#-H3Ys&XoUcl*I7E)%d@F zbf~qE?z8g5*bf+;Ww3JE`HZ(fB^~4Mr4FTvFRdm=sx-`_=TPJn*SG6c4=vN^nSJ~@ z=|FoeO~vQ|J)yIRUYxc-+_A$yvC5c!p-$+`Lk0IT&_#aLF~_C&$IuV!Y9cz}O=RZi z97L}9C81FZs8e%Fu&7u4VghcGzbV7<+^(uxvR(awwtA-{rPVRYPO{RwGrO-4cPbxZ zLwz>jsl#~#ulUOxe0imtgVC4u%mHK;_Aj0-tm^)WYP!SbIhrau-x&l7I{?d%S<7ub z@amqWU%^4ptt=4Bhbk2nA6?-^XzV2JO&k3By8d-`Ph5pUX4!S3*K z8n-QZ1j&Uc%*;!ur7wB@KDJ*%EJ9%i63{-ZKQIAd{MWW$vhq*g_{>6Bkf}rPz?bRH zk%GrS+D>l2KJf#v0}sD9LGV7y?|RXp+osvq|6-N~QhGN75uWg9-`3L0DJstjh~C%oYw%3o$7!90l)enq|QUvaq3JXTf<>w0{g)?Q_F?&PMp zIcwlNFX;&pil^i$tTcW6=_O;A1R--Ru20K{iv>5Y_$PNV9>*G*Ng?b@;Lw&DznSq& z^0qZ-TYoqt+cc@BDzrfNQbbax7W>mV0U$_#w6n#TM);d9rQ2WXOduWh$Iwr0-~ELG zp(R5aynjXs7p0vX0wd=|GAo;sJH{r8iQbqlCOJM(J~2cJsZMNd@}jsd@WxvFYJ=|V zT?3to#2aCWn=y$*9+wqKnoFNu=)wD$@LI#Yd$1iKI{I>Ta=Sm3t}#WAQj{8gWshhh zzU}@8Ey2!ew_>pi?P#M;Dt@f;3S%Y|m_Y#=Grtu4{6?siQ0v7-!AINhA(u=RZqum@$`_ zt7)T;OU(By!3QC@#WT+I6FhlrsMR)al0U-r;Y9Q9VKI9vel!PF(T`XApFiacPT0y2 zwF=`y1Amz)ema~*&r590{o;`6E@%=PTtJB>GzhLK9QgipBf)?iQ4E{`T~h$2zc4Zy$Y+Xeow3>!<&hU%k% zS|}FkE}u4Y;>qL?DbG$HChHU6WTTuekO((iZCsE>{5?5q3Z8`)e4%fy$<&jT&Ae$B z6me2NX&l3A3`fMNso4BHf27(k%qSdne2C;TC&=q}*neuY{W!I=YiyaHJnt{6Pt z;mu6NL-rA75|DHr%#J@%{Rv0tg}k|{^;X3SJZVs8WY9XjVJig(xleVtF~b`sp|Sw( zIaYmk9-4XMTo-@A?3tH3hb{fa< zX5BuA*1N|tx}f=OkW>#El6BBdqf4 zxW~aU6N4iNwi3o7#AL=G;oV**_#!PBE)IWEJM(T@cl*@IjA~mT{ zdT}^c$cqx}k6`MM#G#2{_RTUnp-=4)g#-XkBg42z{7|pZ{TSM`I8*51XF6$&5~4)8 zqUMG%!S}>#$-)A!)~N79=&}zLzDGjfn1-|YSikUMqy4YU_EY={51W#mta|>ppbo=P z^Awg>AE%QpscgjWxKPJgX!I=PomCBm9sVz&p!5%^Q7#&+TZu8DCH=H1+ZfV)G)>9v zV-qEjtt$!=CFv@j`HCawZey2!zo=l_y8J=7y9J0nB{g`KKZ#mng_{kpSx&Mz-Q=AD z=y?OS|D&Idh6oG zddu)R0wdx>;8)doapH0z8>9zEQ>C|J29^*w?HNd!)q!-tQE|!l7vB*PO4k6#O=^gPjAtNNEP%Abk$ZUv&Z~`!VG6`-7wDvcOkojj(Mz@>Hfg00=glt0=&1BKy9xSMS+;owr~%Yy7V+u<;$k_?XR)$M>GiDzG!~ zw-0M@y}PN|ou=8HCj3R_p-Q%0tRH#wgIOKTG9B;N)n7f&l%MbV5X{YYd1KCgi+zY+ z$?fyvp8)g|zvJfVQaWXm0$vQFOX&*bSVNAHL5@kia?}K!j&Tk})9KHh8O&sN1eFBP zse6J}-y!t8lZB-~S$YU9H-9;QFDR~pgFW?f6MsYV=QMXm*H(9OMa6~7A;kD50_3*A zZV6?+K2GTBDR#gE*XT}quc z3dtQsb@0XPT*k^-!a=OFW#>nU<3Mab;O-KT+eIe6;AU1_W~<%a|x&#{k*JxMAGlImQ3!e4*EoqTn*n?j3j zqVLIF4CRB+t!@0S_r6_r)cQK_BMe#Wx}7wimdu1yGyYh9$7h-R&K`$Xf-3EQ`mk|` zgV|<4!Wc%52U`1vpM<9pm+pdSVC z0rwZhLy_8d68-)+KeIC3kDLWZt*@W?EIUHW=Fxs?aG0n)^)*+#n3p);-+T^ig^H@$ zxw7~|4OskA7MNiSky9(=lxv5}R~59P)Y_GnHh({dMV$b&`&{adlILNonW6gaSU+G5 zw?jt~K4PiaD4<(}CSiF9RGThWA1rQu{9Hm8!G7}m1GSOp#QgpIzn zy3_cqL~Kyc2n!p$mm;+rTzuBL(4?-M^0yat5e3$Wl!c|tyg#lRgpAnL{d?-b_Cyc- zjJAJuY5_IzSJdYbSyf#6{<6gRns-#F&xiCT{$0FrnWX9^-Wv5;#TESU3+XY#-j9t* zO|dI7ID!2>vHPxCz3ub1swI1B3)~epMo}aDtbk(XE>7LKusMjSvFOHPD zRpzOv7q$>Wfl7{m;y*0pOD%FZkouW&F?>MN% z@+9l{-=(Td`3+%Wb?xy)4LdDIO=#PZUbOU=I^aB7eKSE^G`Q@m{^@vAo24yMO*E9m zabfQ8JtjA#CriMwd#JU+D}rt@t)xoJsJ6WfdiP&FH%sf!K^Jr()Jpz5sz{79uQH9Y z^TjN-)8@&A-FK5!n?@pCYm=ipVpcgYEwVF52t1TFQPjWtbQB2SOfF40QQYVc@{n-N z<#>{~FPOJEUKdbFAsun(bU5vat(E}lDHzb^*e#Ci7Wdbk9YHwoRJrWFFzC^2NkEv< z(4@8g=&OVk--FCz*?bN)t|Ey+cI%=ejPU9YMHgea_cvc@#^mB-kIB;+^>sVuLdnWP_=ijEo7cJ~jA!f8R3!PvJXtwY%m|cEHdDx~`*-m(qk8 z3fp4SzsjW!B(;f@UFq^%T7`15->&5P%(Ib9S3~W`^Gjoxi)Rc!f7m_k+rV!_`oRB&-{UM#H2I{BB2WET2Zry~k5@@*ee=2lH4t|m{H`LVGdqEs zd2v^&jqJ&BeeWDE@*?*DBri#sFbD}`yv~)qt(n2;=;ZX?LY$Mb!?LzP!WR2#fXo+3 zgA5>jE z>+JrvT{P4NLN=?%mc49cwp)6WEhA=^Ha}5q-~_v-FBJ;+Ef)zd!@j63~EhxAbgS@xs#%3PT!sj#ZBK^kM zsbGV+So&IJ^Q7fc+Y!p_-A1>4T zQj_-zIMbs>6c+Ph0!Q`X^M_G0e7=xJkM69EsCuv@uqDU>Y}nu=>C%;S7$U?~X|UPY zjs>I6Ww`b;F1^Tb6Toeb^YL+o|zh z*2(`|iDxL$@}&wSP|LeToH@1pGovfBc;tEspaS^h;voj{K;Q+huWT7A)fak^PR#vC z#5w5Z-xiA^9%pZV?|fQlyHB6%+U}F%(78;4BZePoz zEBQD6O1`PtNp9?)~L3|PjZbyZd`#E-=iz}7A5gH&TD&OBfru^ zfP<2F=)G@;Gv#akEwHjD7OW8N2&yML_wI*KEgpO$FnZ)VP-z0ecQ846EyJwCJ5tl~ z+kULy(uf|UJDSe9PpDt=*lOlk`|_MYsVSMlH!VoRcJF36<9_c=AAr>^_dW|v-jdwe zf6lHZ#wpem5n~`CL!sseUS*qkeyX0i-kxa)o%(SwZ|@HCU4Iq-v{LR zPl$Xo<$uBbwUyJ2&D0Sz-eiq0vWPuu-|(?Jn)`lA(79RmQB3&-K8i6`QwjaXze)&X z&Q&->5c^SFCYoAcRkCMl=w-;zRECmX%CfqV=& zgf1Q|LMQU3BW(`Y+R43NML?J@8hEy>*vM<7&j}{tALUx}#90~2%Uzvi={VZ$zeK{Y^lkKO6X$u=Oj4v$tI9MwE>*^2$cgXf4LGXzw z%AU=#^b)mZOR8Jd!)Iizq;~AwZYnI5*zC-sY#q1!S|%acFuWGoYm#kxFH8E&xzn%Y z!rDw0Eedmm@3UnaZH(_Jr|kYuNmEM?wiPU0Y}V6Q>JO5E+4h{z)E^K{MFgnt&_)hM z*cKdzI^&eU_XyVkhXVKu#cH0AttWnL)O(_YA9809b_hPVe6&l(i~$iH!akp7Vm0?w zDD%jecb^R9>0w%O;C2&7iGc*Q3L6Z;poLlfUpMbUQaz@hY|OS6PKwy*+ZU{}k;%%A$3KsuXb52`oW0uF z_v`J9BFR6_>g{ST&Z-jE?t zYDL%wz)4xN@E)sHq&^+^@0D}?M*KLQ5XOn)se*{tMRW*xE8h#OY?BZJf_`Vp-~U3; zOjKpcFQx}lI1U0jd*{f z81n7sVDJ{s4Q&^oR(n44MpDbab<1m_2wq>Jg=(=i$Liw3Xu>23qko23?_hhG@`ot{ zj}4Rh_9w7FMGAxUzRSFbFs?gMy_L%Zq!II0w;!U3gd4}uSFPhcQDzY5Y~6on6s9eIOZ|V*_fJwGHF=>5(PY1#Cd+>#p5a<{ zjs6M~ziafTRFvFRl(c(rH{+u;7=Le#Bvo2rJ7a78O4p5ZC@uP@ zb))|ia>Fd&G&t9$e|WAr{BI2qRq_l&5P87PQsB?}sk|w!s63j`fULu`98AVJbf(v2 z3^38-#_VhmLW8NmAhpU2QZ^L_QG#w)lY07=o~bxHfXl}dbA8->Dn{3_#$OLK{|}q$ zisQ7(FQbO+6wJUjPB>+M45MLV|7?T*5;{(aSyn>u|NCYKi<^4dDL>F`_12t&|H$bY z)xmI@#K7dP;`qbD|35jE5mBHKOpsB@IUzc-67v*2N;G(55>JIxg@k5&Jsy*t}M4V8?4<$tNTfGCfX? z)#i&0t;SQsbN2LZ*E91xc8WqS^5Ou^b`u#+wzHIj;O*b*kOJ0b3ad$-;_qy5LMgp< z+f4ayIq+hA>^*2XGcgMlV3Qj}(Er^0^y5qN32EZ_>ZE>v4tdEZq@!Dsekdth?U#q= z=e027i;l?8`;6be&^-v$nL6#tKx<(8l-zC@Fc+hpK9 zx3Z-Su;)3u1(F=BP{Gd;rc?MpGFUB1vna#qg;~;%qpJChlMD)32UDz<3oP?2E5Mme z$#sqTGt2+`Gq&I!`80rmRoU5C4xqPx$jKEyaw3b@+4-FOd`SUpB-`ChJce#jGDoA` znf-fY_ds}DAobemAd4&@l#?Wz)kzRdqUvn*uIP9eR>yTG1}&D3qa0*c%wxhwfz61m z9Bo>5p76Ey_gDLF-)N2h7h^tlHd_u3#eQS0@Xd~4yM)chiJ{$L!&mojyTyik>h8lG z0qy#y(_HeIFydui0{4k)KG7&0oc&W;kW>OVxcKwb_T!J}%uvwY`?(xyj_L@t|F-f; z09g`xY?U<;I_agBe{AfeqU19=^-hGzD7~UjBI$OPy7i9iHH{gnjAmZ$VB{=%;&N3Pu9EM5XOEA!pmoGBCy3ep?yVoGv9u zHxmC4Y!NprSKG~(@Hf}k02-@#NPrbwN6=X&$(nwL`Ed~iYaSAOLJNKm4EU->1f$uu z9AUSBYtRBABsjzr9ql3%0Ges(Z$*QE0zeGUxDUt(C{|Lj9eo~Oq=3opFtc##%HuU~ z1*US1a8K_?2F8Pl58}$+k}Z3Su@o(YN&M4)oZ9mAg z`ruH{&zQr=ld2&Y7ZM-q18$qV_p$|f8@kLI1S+zA4ClmE%a^A9kN19VgQ@-79F$^% ze`G9Glu||Q^2Cs~hqMB41lpdubz3uE^N@^Y@gX&q$~oF4$^{E%Sr@fySr>njeMr1X z%ewJ6mi0o!CzkbTB*BZM>Da38d->->ig?l8!KsCGgX|}3{IxJVs9+=$r22?MugaBme`Pfd z=|F$y1cxc@`#b%q%CFAWxQ+a7b~M)PPF`OaT@2!$piUy zZGVuo!$WdVw54+v!)s{!AzIAOm|!F>u}>l>=_GlbR>I@NMmh1B`6mR=0YtV=^2AIn zBVwW|@UOmIcxOq(Hvl3!Dpwb4`;%&Ou3Oj$5>x9mZdUx?Y!&Z12Y2~cWmJsM9|J0z zX|mT`w|~L(DENfzQo&N>16`}!yIt!@q-DE0G$~X^k9E#+eSA=z_dN2mf9o&E+4d*p zqU}&?Kjx&a`cK-&!6r+gV6)a%D25o-g(NIX8-Q*Ehrl6!7R4kk5nik!tzO=IAH{0M z7oUYBJxK@p@3dJy=JPYt;GTuF>=uK3oYg7^*;8n;|VN)YMqvcai<^s)X3_qniXe;*(j)#9A#qnqAp@*7B4+ z`d)jw==oY=7V0}hc<;A(wye0HSL~>c0K-Jfj%fcp9P@U@W`5#x#}!?}cD%)3yWX8V z!XO$gsQ3DVaSOe%aCrK`irZQ`>^84(201I+4tZ`r5bWJsM5j6b$@T*biXPB4FcJ zh%E>G6-VU0_>Fk1vHba~(AjxnkJz6Xl(D@LuR-S$FL+{VF-xKDhjtUyb08>uhLC|x z+ns*PIu+i4njczlE^y^2H#0buuBZ=L|G9#|@lehg*p{dJN6{lR8se`%1CD6YH(*F- z(*MFE&}i^8Q7E#_N2&#t#CRzWm+Fg9(~}c_bR{KW$jOQG?AL&m#U4M18osjlMx{*H z%G#1D4`HFnSAY7@_Q+fOYI{U`wQY}RFREuNIokG! z7<$_yVyc<)OWqY^lO`j=DYl>&i%<0I23*6iN>?hzW3v!qN|?=W-v zJ20!H|nDjV*3%!QUGh$-ejiy)_1g)%+G|YkGbOtIP-Z?ul(XuSsJO> zY!#POHu`_LlWN!a*JC8cpPFfkA0ADeArpi}3tVL&zKgH$3x*^S$$#-Jxk+-O^eb8r zN;J|$m73Tq=K%i$KoDMNe(~urt;yVpUf=*T4JgU*a|rhT_Y5$X9WLDh4q1wBp)Gh! zXKs4Ssw7PO;~zpD^}uwTCx{CyGj-MOkaIX_=N&4T+tuuiNv2wTqb*tclk~79t@f9? zq(M42tLq;s&UcY5+|RxOn+Vl-NtyaI+@vCRxGd(q>j;%#LiIZH@@a5Fg-94QoG8XZ zl%A&@R<@DnGl(8{!D}@n3xXdfoO5ZlcrrJ28d+%Y;@M-z85tnpMyzbVV9#$VZ+_FE zMi|@vtzDcIJ)H1vkPO;N7n?OO*e1IIF&p}TNCG#o-WML^#>dbgDf5Kl*qj}WCx&4+ z2XsOUQerVkFO5QyQ}Dl+5Ojispr}8{0rN@;AYApGi4Ux+?PBoWtJ>?fZHQTS(W~d3 zbq{7RiF-iPbry>W6MGWF4YeMn6WHUSlk>+J*YJ~L1hE&L<#}a$sTe79QQ^0PFB)TT zxn8&r9`ra`1=CpXb;x6XOw@ZG#Kup&`#gHnGsmiP_*sEh3x|O=x+`uJI%f}YDKS6r z!a_k3O$itovzaYKn65xXAP`Q0gE`d{T5yN>Rojqgay93PpNn510hVhRc4QHQz$nwB z-jI51b4R1ZGwga+nW)^3=e;E`gcjUR6$*jl?yOnm$HodA+x_?9prT8?(vng{?mj%fo!J%_W}otPbAu3R($&uS3fTVpYCuF)H~}K|J`M8=#8S zjWWxH(H^k+7k{PBFRHW(kxmMtUZ)dLW+7%RIy>lz)KGG-%2TOF zD?4~j$TZmTwV1T!H{&|)(#qzW8f*QLagifQx-3Y#+9mDZR8t$@N3;M(TJ3*jsd3>> z_hJ^2sP|49Vwl0^4+k?!)%)}B&NAObiHaabn-tiX4F_)SOuA7jUIC{V z0`W&uXJN7?(283{0g|6#{?a$7&u5~08-E`6hQRxR;+%c*I;fRl)-$58HU5qs~H^baX$%L%s+5t2KJCNNilKOCxn5I$LPH&fDy5j(Xd0%G((Q z!Zi~=;}Jh?Nj(M7J4xwy@h>_Ic;~Y)=^dYx}Csjg`23{u6Ky>Dd z2$A@T6z5sPMO?W8889#0myBr5)E!T}AxIDIHvbR@`~Ml;2lj&t(B5rF@2x|F1-6!= z-6yaB+H{WRE#f`ftrf8k!HM3=f_NtgUNEQPKcGn%v56sr;*!^;RNc!{C{FadE!^To zb9mam2|rJj-|hCy{JVc>Xg-0YWRT>!qza+=2%$Mn8>{_t0}qF&pvuz(n@VXtps2B0 zjt{)({{z%!q31#yOVhQe{)5|b&w^{|z8>c^o|WxCN6upC)B7;*A@q#too*p~#klXPY-aqb>|Qe=x;2E&^A0HrWE+ptO*oJO%FzD~U0K#}HyI-?{F?fvD znVa;My2HRCoqHi8@jrdYsw4UVu@Rj4!yMnOn=B#G(L17eQ!Thz!^i8gL`FgSjl#(~ zuE9tB2d$38rAi88B2#MEztg4G`kO1a}>vRQ(!v5|nhAF3NoSYue z*yUz2!s>VLK@%0$@PYh=@(L>n^1;garIWi6GBr5X&w~Wu^mUr88mH^3x)&h0nAfMe zbtHLR=~Gp%DqU=k?DGB8->g$dN{SgrYd;__d?wU0rCaY;X6qO@}qC;j; zw-33)Sdc;`rJ+4;*HL){m7&6Q*05$kK|#C(k#wLsu8_j?AyzcTO%$`OA9KaN?2bCd z`Lf9yiLp`d4o4BG;#IObvQXsVnUz_!ZNmt1_pTkPP7!x2oS(Q!>}NKs+g$ENdQGKP z!H;z!<_^8lG9n>{K>x4%i-byVrDim_@2i2F`rggVXoZ~pwPENG{alF$`RoY09`}+A zPPiNLcc6}}MvQ0?fz+ov^1a_96SYy2Medj!96 zQ9BM(pVZr|@}fgSLX_!ZaaGVA_mr=z_qgzxbEW==KM@Qe_rvUQOm^iig%0RtH>#fEvrTsq-NC;?=2OanA*bpHR?IWkIqI%KycWdk zc55TXJNJL`7b-gfdo$?dzpot&=o+GSijS63?U=uGfRkS@%~tmXblSVi|K{C_^e7~s zVg`dP=M<9X8LuSpPe-WeZVHJc@Iw#mB_soQr!(|SNgo0xUMs6~z0Sq1PYlomvaGQ_ z0ecI?RV{mXHPABZsqi1LE!M=VOzSL4NL^KSW};e?hOrSw>z}e%bCP&i+`$l0Tu~D^ z7ZoM6;BmbLXT)4k8H5>OR-^d=pPOl_=^w8Nnf*{W$i%GiKR!@!Nc3+?9)HoqcvP3| z#Ro0cU3zx!9!qBM3v0mLa1nvEI>3bA@h%>t7GP&|NnHuLen(|9{h{L*v7Lm!%oYd( zG4;G`9RXon%scG1o%>C)UxM1B!FyQrrMgtj8B_zER&jlz@Mdr}Pf%tU@75L1Zfy)Y zqj;&2vjPidHKGwQjqr{(1z>%m%uP>qHU<(>ga(v_v9*lQ$U=IojFe6XY=BU7y=~U~ zSjCMB-aFoAnnOwX9xp+h3HUH{; z1ml(M`(|nDFNT;S-u+eNQuG z(Q+eJe5tc+k$cH+`GFDsDTv2*QuetQ`x45p_YfR!$m*xiAi@Mz4)o7uP-47EbX08?f|tQR{}^>r?G<(S8$NOinCzhy3Q$NcGxn_ZpX&Qb2nA6D8O0cMV^i7L;Gpi|Gl{e> z`VoZ-lwdwCfVU&+3F$=mm#D+DBIXuB2xQpM#gE$bm>T&D39t|ctPD1Am-8h?^&RL?l zJJ+aHx)spey}Tp4ex;>@4WkZq2DUg`r;;X-Q&%t(sVcN!Bsp^TXKo)GFUTxYFOY9u zqFq{z6+YLHDvCBmJiFcK4J~ax)upE7^NAZjD;uY#jwdKgxpe^&UlrUpIhceX*jUs* zVlM%&w3f?_#jBf6nGoiv23% zXGfkwRUv*h&cEX-;{ymweZ>W$WD{-cy{{OD$VkU76u<_#HWj-d&z}$S0{eOM3i29K zm(1hH;*8q-M0@X30cr^hyo9J9p___n9&&9g++k774z_-3xRSXl)JveI_4)xy`6rfZ zprujqf>iO5taUx(y=H{N=h@NPe@Wnd6Y^p{f&qWPoA8);m+J+Pr|c_u5PW}D-(k12 zOf0%5_?n&g-}BC?8g$MMx*=&TKUHt*AK~(E#gJHtfuv>alV}Ykhr9 zztDTUU-0I>g3*lkMf=(_-dFTJH{OSWui5b~;XMMT4}`_2mRS?tly|&nWt#p(^(QO| z$5!1c7bg&|rgUWU8L}1Z?6UQCD)*mJ9X#l?y;a+G|GGWZQ`mRbF@EVj4poilM(@^_ zvAn}(x^LRmE7Pvks-lN7`;iHjcrkhUNGJG<$o~`hWP>LDEYUw=3H{dQK1?Og|I#Ug z$7!lWhBs;^cN0gJ+-#g-RHnQ&XgDhvqa_EbRZU5}qq5oRSnc2DI~n=Zpj0?O z>~nbM9*S&TL286|q!pwJHhsLn-;2QJ!?TGR8s|YQB)V(jQyJ|MqeALK%L*mp14~a* zi_A=7Of@&uGB?zh#YsEJnmt-!9Q5~4Vpg*LCBNYVSZ#mIFWg)m3tbJNXZa_qhFBTg zel|?N1Z;n?fhlo~=NdNxS3A83d=e(qo_&K8d2v{@W0349)0jj+p8kV; zmAR9at=FvWtZ>fs^b|Gb`7`bpY8a`7CZz*Py~z5IrPS~7)|(b1K@YRMr(WN4{U05C z&C=pn-aR`LbuaC8|IzwynD0nZD>$1D9Z7BqxoOFfN2HBhmMeD466alOg*!LX8#ntB z=zpV61MvCA3lsm9KKEXbrOyvwI_hbbK0o69o9T1&`;I;z3QC0oN?gJ_@vpOW1*yO0 zJxhw;NS|<%)ZwMN*J!ffEGr0KhUqB?=pO_=be1f?RRq#bCRU=P{AoR_woX3imD{yhc z&g}opI+OTkQw?k}zK7UGg}mNai&npfZDByP3rWh&=%Pxi5^H87;)kl9t|qvXfk$sw zJ?m!nJnu4d*~_Vj*mTr$6sfz+N~3|JIE>(#079>0@Zs|`hvG7aAQG|Q$STPnup48m z{?N-+ZuMdRwbQWsX#AL-HSFG4ChXqemy`-A%&hK0&`vhAr+GI?>P+oBy7$~RV!roX z%?$qX!avPTV8C!w5WHmP!0pev^(CwSvj6Ig4JY1b6oTL8jJH5yrmiq;G%L zKO7p4q1!#o@hINJd8!QOC}T~^KMWjeq{W!Gp4kixpztsW*1|BvjuLHX&cJJ)RlC0J zWX{i?ypew!__vmStN7PZ-!^q=cHw?LQ$DrBptoP9yb~?~Z}?5fLGSrNSR)jx_P^AR>?TB$lgSV>tqm8>s3}TsO!mN^WOodqpP$3(ffkZhr zi|*uQbxg9aGHstnQ{yMEbTfsgrms3W zRK`>f&z`v0Zb|Rm`wIu#^d%aaM1dFm#~#~-9{ENWMQe6D@o* zxki^dC9qO2s=Sx()_bq{jrVG^Px5)9JRo3hu37!rA63=A$mCUr+P9Tt%1>P;*&-{} zpUaz$3*S!;HmMRd8qt}I2ndWe>3n058pp3{yVXsP;dp|n8E>W9iE^F-J@ zpoH@JQKg_yYd~ym4gFPw&a}SFYHVJWZBXjqEp^vH@eT@0LqYuBs z_1kGrpYrx~`XJsl79yx*wsDYuuyH2Dwq+=}Vz#!I#JwJH+=O9B!{HXD(1LcujrnM{ z3m(!>l=pl2ru&T0o4mgpesd=G)3LJ2?7}eQ^D90cugPBiX4*gX8aU_CXiWA;*m1Zi zrsim3I~DCpyz9^CuQ5!@Yu_h}oByX&Kp6Q;RSR+)P)FwouX2qCfZlJAi&P{o%b<0b+Zh0w*SlB8Ztt?__I$Eh^PZ*X( zGxm@4TJ2UQHcpTTk*$=*f`r{7Q=b2XA=%-X@@v^`2Ct^0M`;~u?w9MaTZf(j;e^Yl zp=Wt335ym>Eij)^j?vojSt@7i>rVQB-&K2F-V*>`M$E(5AEIs@qa8&9+XiSa=-GC* z_i%M~ClX{rxn~gdF7N&nwa71Yjptalf0ALlgNVSD14<8a@=MF`@SMHALWW-51xom) z^*MWeSXL07ZYLV9Tlh}7D~q`C?@7+!{`C%%CtmX3{GCJA+NXOX>m=Tox8qs&Yf2wJ z6ugmZ!tePZgI2PmL>F#V=OsRuY**WP&7Lu@2$8}T&=>tf-b6+z4CWlVkaF77r9R^c zpXFj{H<~#H%c$v3gB<`G38DQT_Exop+98x@OHJFAkLq`-{7769LL{k~@rQQ^k6-aOVJMW#tA z~mjz946zD-2i&qBPbC@J2yc(Kg z_4F^o{T#y90_fzsz!ZAna#m}&A|f99CvE*1KRiLq%CY8ic&qb9MUr1uTzkC!#>YpJ z>HPTq_BTYzki-Pw{r#6=LMQWM6TB{c96$c7A4l@z3jG)pX-RAAD?XeeW$^)|$CQp= zQSr-cSf+7NF202yZm&_jp-b6Lc5iu&?nB=tVq7k0m{G;<#I5x;shD34-n0sg1@VZK znnq0(3oAl+gBLAp@Ho!7-rE2y%KV?vsrGV*LRIO}KibphBt~BQux!duJj#5I*5|eF z8Q@tAcWN9e{@h;>r9r_;WNhvBOo#EK|Lgt|TfGFKB7}*34iO!T;u8qw7!RxQ&mjia zrdb*Gpc{VGe{7se$|7y~jC~AaPwbKIs3(P3-$6?J;r|Ma>;9bnN$>vSOx(b=jnovV zh}Db~Xac_;fMU3hWKJK$=&P4!4hla87dYr7`&(y!FSNgn_P717`uc$Vy~_SJ+TYXd zZ=L-;-2PVZ7d!ti2p_)O=`CLIlN>O4Vi>?+-uf@ap$mIQ?eb7;d32^TIky%6f-HW~ z0hDbkZarPAa7UstHLU~zt!?Byef(#IeVi41?7wthDooGSu6_AfT8p*~!<&5>+XS5@ zQ6V~B5XHxlJN<#S?zu~zOp{X4LT;9erOJIgdlAv2o!g6(?~bY$r@#EU=`n8@9;G`o z@j?9E6))Cg^y?~st~2$z)NvV?C3fd0o}@xku~@sL3uHzsq|pOmkc}gtZ5oDG`t)gZ z3WdA7hW9`QL+gyDO6N(;sD1rvj>Ht_9iiVFu032!*U~@z(f&IuI(A)j+rOfnI}7pk zi*|OOn2HT8^B+A(-2U?6A!1OZrV{32kbvzv;$WTJF^r{iekj>l=HJvi_Y`u!{b(+? z>o-98u2?0FFAWE=gC#;)6qvMGBJ}w?yxZ6|Ra+c)s4_XgIoW zw?YhCUmCm!?1&HR{5EFM>Dw}cKriUI4Q*2^n8ND%)X8H|Shv(CXZm^mAt(gY)BDbj z7hdSO&oRahEmr>>odNYiE9Z&@Dv+>ocns=5tr!*3N=>sL3ON zI5|zk?dLkgt(`^jQkPU@DMbwgR_*LAPMqd`6vmmsqJdHSSW=&PEh5=VJb^=UzgH=9-gKAwCWRp^MInXj^1stnD;Jth#~h z+nzaaJB^UIg~aWNztPyJZZ<75z3W>XZP}TJ^l|lHmr`L~Xn4`U(D1r}Va|Ou^G-KD zjB*(wIhKZbk9mV#d$G{+0sK3y1mCBDWom3Vuk~e08?LkprNM$a!%E+%bYJPNW0_S- z-J;a(i9go2_-diPWxF1sq&m5!d2nT3H`l(aySt4a7%Z+YT($cI5t%kSM>au(a4YCF zNj5>G73C%=J3-TWCMekrW70$excr_8>hC6qEv#G<|CjSqHxTXr%z;~ISQ9jyxd7Me z1_HQYu9*lj-q6G=Ha*w4>FF7-%@2F|*Sh&(E;KE#+^*=v4cL3@JlcEJ~b#|8rW zGY4uC>V2hcRN4lmt)*W!Lz=$Rn8=vxnVX;z5{Le``Dtf<9KFN;Rz`-y-Pea&Z={QU z@d;2`+sO=eaR=|wHckXCTx!3j0^`z8juuZ^7vIZkUPWqa@vI5NZ$4EvaMD_L83fqc?wn%RV!WcS?K!(pM`GV^nrFo zoCPfEVbz?D1TYIL&1^K2xfOPN0jJ0JO*#1#Ln8`#L+y&tg16~beeDa0L;Z#$>@pQr z8wKq!Yd}}tUVQdv9u8$NX*YB~WV!zqCwu}XMscP008UKu(xjT1!xEh37}b>gGIBN7 ztA2UjwM9+I|1rFR6^i2WxXGocy(5`ESWcjZ$@H z^>s^eCn9WK6=2X?S2J=fyZfQmW=h3sm&dP-)~-$*>>tbEbw0*{MY3Q;0G8PJ04(Y0 z(b|{euP}Iqy~9Sj!$#W4;BlSlJNWwKG6ue^0JfUXHMF&>5{0CfY}E6!hhIc%OaFk=S7K$vKfAgt5PO%{d> zuv%vC2Itg=T%z$m#khks1VR6_gEi_PfATm^ff?FIlb2Lh^qP8q##pzd!x;~TnJuu_ zHy0Qi6d*ME^L{&rm=$TA(PoZSGWn&(v{^NM=&nz!0NJxoY)<=Y3pVvikl zZ77>*23&BrhclyE^cCiU6$l+43uNh!_BTBBZA01?|3GiKLdM#x2PwxKwGrv#Mv52^ z1H%vT&!n6G8N)9-%qghYQH&&th39M7ERCK-iXh7DL98rlK>I{OhN78@QKU;wP8H=z zPmN&=y;H@r)_2D^)<-fgV-~IbEZ$VReBK$!D=Mq==W-gfuCi(&G8GTqLQVx$X-=8Z z1O%~kgv>da_(AsF6)9RcJUuj<;pahyB7Dd~_;pk_wavI{SLj#EQz!3=rC5MMeJq^42!Id$E51j^ETZin^$z&Z|%r)Q4H=YxV8?Hqc#K0CjfoJT0%IR7NlyYddCx%43a7<dbM;#QS`MxsL5b5Bv{no80*z2s!mLW4Z<1d%fPrN|P7y;q zl7qWk7j+SR?>!!DfdN}j9HNB=%gA*U$*pr=LXC!Z^|gNwEg<3sAp&O)OPpF?yLHaK zCxsp>I;5$s@RaJK4~ZWjzdNqvuXnrTAyPwGUdmUz^!KSq5eO=P*f<7c-={5tvK$+^MM8TBfcJI4<)G&US!S~@@!7~5-gR{Nuep{H3 zS@j_!p09drSl$_9btb5+54Q)Jf{~5G>C4uI}>aKKrr3?!Gicv z>0%z&9^zj@`%m=6=Qc!ES%a@b>BJgb#mIr=QNp0YSlf|Nyg|go3>*9{_{eA3S**4z z7Mk4U7~l;(T2^~x1> z2M&A7H{D5fEYqRv7e;Hl=CJqaP8Ve2VJjY|_!A=L9(>P^w+$`Wx)fSa4Bzfk@8_Bu zuj{F4l%$^c$q>|5_U7CA{fFryO6b=o%qEL1K-MY^7|w7B!AcyftL@Cz3e&TZlnaecPZjKi~#MoZ-Qu9yZu$(D@0fD z4+qyH9C!U7*!XEm?i!D~z>5WG#N1tDah`TXs626m3VinzE5OdllLJ*}1m}ghT$q`x zovt)%XME&IltN-6d{ve>K!vJYp~NuDOwF~^6D3XFHSomcMHAM;o+8^muk*Iod7mnK zKG}sdW0yuWTcEi*bKaZ*sZ;Jj{@mx_=Hgom3 zL@^v;&Oyl+iyB%#N{r40=18Us!dss$N$g8Cd`tt=preP-J` zg}Z1ZUR2+9K^_?j;sx8@;oumjF#2#x?U4o!5x9Y6y_R)Xt?(>4@w+14Njj7w!AOo> zyz2^&ROvNH$($j6&cDToOa>B@|HH|NN2pYE=Ie6MDV3{$3>J15f(ue6)97k(Jc<8cAeG`nbKBXVChKcjhZg%*KJ@8df{H^))S z|3nJr^Gx}HL1GaBO;LUWy)awWU&4$j)amHons)HGP_TQiJ@n#!55&(58MF5JYsMRK z>hKYA#!lJ)*ioUm0@b^`1fP(rxw|bemuKZE%Z^m-3Q)`5-~gJjwuUIVE?cs@cga9j zcTFWH9**03p7(luI3pd+Z-35I8DQfj>i8Z?`12G-n<}64KYEl-=HdT{fzi}?yr08Y zzreDL+|#m5+>b|k$bT9@a$*;%Umor)snVzUB+P**? zqStWKEwYhO2)XXPQRnS2c>3)S_u}Nl(=nV9*ybz1-Bo;oUiekF@Sq9&WCN{L(j${! zsUDn+TA}z{)XV|EV8e`$nOZ&`Y#+B9CyjaRE^DDhs3&uEG`SgtXC7`p$RC6K%?S6= zn*ND`^d=?DAR>qVPo<$!zpN))FE}xkKBRhIavylTQ?JkP8ms+`YwYo#Mbr39$biUd z&#GL5EVeXUk#1!+Nj6!foFb#29?hhy7u)MsQAo7ovnjT~RCOH5ZquvFG}?kH}!ykq(l zZ}94qZmG+!J|*JDnhkde;cQ|3S+*SQvKlCVq1s zJ?8zhzVoYMwu+aNoZK`jdimDrQ@y$m@ZaH9z5+PG11}{gugH zY-P9B^()E8_Wd%cM}Rr7`XT zz25-z9|rZ#?qd--=9>#l2@0GTO`-VtuYC}hINA>f4wyJzf)wR}fMU4Df4h{#EzZbs zCAP-JN<4a#894yKMHXMm#=#Cr+qAOWI5dpvK4bU|?S8`;mT(pd1W8|}6YA6bD%@A2 zscBW_>*fDU9rX9Jv}l#Tr*+dG%sY4I1byWoD4(j-+3o%(cy_8d^sO;1_BRFzuD+M3 z4|#+sAPAAsgxexs+Hf9HSWWRiu0`QKJ36zteC{slDP7My*OazYPn!f`luURNDhj7- zAIEi&?9t)5q%p~*3&7?f*JktVJ>kuoJGYa>b#_*$pDSz!r-9=lq;yuk1CtYr56aKu zs#AYX1nB`Z%=7Co6__01??j39|2!s}^^HoGb*i#T{`B|BWnH7Jii5k@U&|MwGYxzk zM9A}}=tDWp!Qd{&8P1=7s{Ou+%KR1xz_LoD@JA`@-SA0E66yMbNus+$u@|wJ_s2nD zrv>*Z4pBVTR$Rt|z(w7oi8o#P8h`y;Po%_lEimFr6d1abS}XaJ-rL)bb9ESlI?4S= z`Ty8^_xLD_>+yecg9Q>dAi;3cRf3{XjHYUEq3*&)cV&Z7P_ZHx11VO-G}#5LToN}} zmWNe7t!=HfE$!F#V{2Rev{nHxB;2opXc6ytX?@~)L8~FiWqApjO&r)`%p$I z_=bq|^BjF-@!a1qu5#Jk&S11p(zkhBuillhyYmUC;A2#8elC}gYC=N6=mw@qo+~qR z{RX4p9~bdBr60xqTV2gT7D(YiXH&O+1U(d=og1hr3+81ZHyL>sQW$~|gnP{imoe1Mr>I(7BH?VYcz}2DV$A3y!?*K$JUq+SUg^jRPZh_B z^v5`b`{X~>8>e8X=7S(|AI`dc(f7}&y8ZphRXHU{MtjhJ65sMlhi@5AQU+lh3fHc3 zfo0!<8E20c?&(YVOY#$Hlw>7IZs8e24j!DIBMa}TWSuZz>E90A{`%xV&d?vR1qehv zNqF_B=XkhB<9KIdV#(%l&L$tbza)3!w?$jOTn)M=+1t7N@Y&x%C?HIlNF}e9F0EAT z_v-{p^_K-=<*uh%P_2kaY2HMyXh%*Y7q628a0}KyK?)>sUUFQ1Lav`CMoL<=$wAa4 zcHNXXDVgJ1p;X`s_Hp9QeIcsgN~I`th2nZUq=h972utopyj`P&B^WyK-NF<&LCz@* zz1&j$j2CLS`e24w=qxpz5u@@pyR#hg+`@J(__3h=FU2ys-!t@2g!_ymuB(oDjz!>> zklM-;=CxE&%9vp4GGsS0I=e!4^liTAPImDvO7y+UjHRnSZ&coRsroFb`q+;$t1j17 z`&?HY^?8m)$ega6O=2blaVQ8I?lb-g6kqVlPvrT30|ougj^{GbMu&Yyo4UYp0lNt9?Mc5vuTL{y;ra|6 zpDdb72xM#t-y95Tt}!)k=pLyfyt4B4_=GI;JGqdh_|zO$0=H-A5HIii-4?2O5Axvz ziLZ{1KHvyf!&Dw<_Z3IOg{HhQCpg+8xfzzNBzmO(LS$NbuY*dG3Rx@44cGrrASu|a z1K%-ypkK%s`V`;eqb2V-lCto)6Zh+-?*r(FKArFi3MgWmmdjbXQf|XnEFJ!8NzUl( z$Q6ft#ZD+#_)=q3>0O0OlZ!7<>p{SnSm-yl(D>Tx;pVnPT*kQK=9ar$(F4a~T>G%!f1_CRV!v&V%R-N@5b!CvFuS{pn95@FyRyDo8O@sEH&yv$LX%MHN~ei z@WSa1^9M|1syycgheOO=DBEH-uO1eCuZ1PBATvvft#tub%5Qwe5vm>nY5&Udq-?Z& z2r@ile$0ImS6q0|Pz`30@)o0x!bq5o@OM!CFx$s664P1w`n{=`q`avTZ{(Y5h%Uhz zW>%!XIZCEH#9%n{UMyahBnDTa{yl(1*^J{L9qn`Z56PB~g|b+c(*HvW_EB(hivZL% z7oQ#`Py?yb@hq}@A;TqFx~RBKR(AYkfz$$&4Jvh8Hu*AfPCB7jA)Y%FAHJ~2SXsC@%*g}jk(V&7>>3&_MW&w%bvdwxwvQlLd)=; z8am*Cg0O}Q!VCJ;Y?0T}qa_=Ky1kx)CG!l+%^qqLpm0f`UJ5pj1dPkXEZgAd5?FKw zmub6fr6BVQ=E9>eaM5$9eoJA%b8I1&*OU|@U$p>R>eD14um!N5$pjL7N>dg^&UOZB zP}0iV+?%?1y1}4%=$lD4<^y;FuIqjhm2o#RBy~X>LzJhUOXoAiKe~)t%M*s0&w`Pf z53BzKsDL{M;(fs#6da5$JjTSx#VXC+Yv9kg(+?wK5oDM7XBM?Bj(o!Dm2a)-EeUG* zS*i0yE>y!HSUnvEPUwpG8;`&!%}-}-PoR=F<(Px3O2pnuEG&4zeN@kSa=$&DX@93d z5oKT6spT>?=zooQmV*keI|QPm-{#QS6a`YMbjAaH%?s4s_x!ih6O+Cwp=?c$>8RQ4 z8tdF_Q*?m_Hi};}dx)Ww@x_6fgYB*N&|~XQk@l*+o=kmsEc7TG?XBK$f%#h%cNVQr zyjz|WrKN$fyPfT=g|6lpfubrAedjpJ6|wQf&=KOI2R@B%&4!ZvP6T(63(dj%1u0s} zz{cfVy6(&U8p> zY%nW1SKrq9&`C(IO7HIf^rF$2LEd1>>x3nEd<2BdQZTdDR#yOKa z%rDPm0l6X>&RzQYk`KC{C3>zFXNvHfK5Aj1f)@7f3C5JGyLL|yB1`|Pa)D^Jti`Zx z3o!UlWgLdd4;v=G)qdrOSV<1Kr}N)$-t~sa{`UqwuSI<3Dkeku>uI6CwHuTb%6t;e zyF&9sy!67k^D^+l61@s8UCR6kdcF#;MMis$&-mOg=h&6T=RO1DtNNg)w$k%12fkpS z2%8@z{~Fl@_|}N_r$6r7>W|;FU$jwtp3miq@G2r*E7p^qBZQ5$zS5RjPwV_5kq14c z!;n2V4@BQ_C31Y=e9d}ARQhj)u-rll*Yo|eTffQ$keSAIpNAW3XTZ&)tLbVUOrgs2 zf*rEN_S|NB2HKvUJxP2RbJ-;pC~IZ{nWxot|3OgS)tmz#hdPR@c^9A2j&XWR5$&jS zH9w<5u5va1S$QU6a*V9ej>)d(sWdv;@vZP_(T*9e=GRmiHLm7?By8J7BJ4Z|qg$Lo z&w4C`8k6z77QV!__V0bxT|jKZK-W6@?AtH@R8};9>G~HrwK&-J3!?n4b)4+HD+Azf zixa>lmd$;IrXz1Fl|!Aq+}}#fIkc&a7DjS>eaxWfgdamDcx*K@Yl+L>AyxaDO;>XR z@K7})8F2ZBE~Xrx2UhQ&i%0k?284&Xo>Iz03rs)KQ%d#$ZVs1Wd@Ntd&3@0Jh5Mm= zUOzXK{0eaYs7WQ$T}$WXIpjRT>uSCl+@)DgSCsd3c`bcm$rQEM!V_w{h@cEO!2{JP zzGs1ov=~BC%?Hx3XPLhQtBoAn`kUWv%YT>*OnD3jW60J&=cA$NNr`#0XcT9Ya6A;Y?f=q`9!J*8o`45xU%3_a~lBh z(cF$3?U?IoUX5qXomS{WDzq+qYqX;Qng=0S6S1PUO1cI5W9iGp#C?w^f^@lVr1W}W zAy4VZFvE~1@{_ySxuSFrv1O)w75Ow!x*?#4{~@ybo-3p<*V>O=tHE1Q7YX~69D-MZ z@0wW~ZMl|n$0;RKeDP~PmmZv&nCMe-V~O{ZuUF!mLE{QR0Q1vp?XGe)zs1s486WGG zzRs(SU6#m>UB*GsbQy`ZKMMNp!tSW7qR6H~jHs)=2fz6Zy{a@OCoZs%D%@X2KHRg) zweG5p#A%hTr*R5ZSdXe2{JE9JRUPU(zsj?IQP1F^{lMi^W%iOX>vJlNmP*fui=5iL zXTqg4$fJKhL%U?0r z)jWVNa)JB)WrN$CeEG>=mq6zLN8t;d1l$7G6ZDji=lp0w54?#9kyC2@o_8YSttNf5 z($iMyx~9$7bnO2SuPR?W(iea37J58Tv)f!OXJ^(eEjc~v31?#*;sxvHQt6=~FeCp3 zL#Mjl_59I54F?s8aH$DDrA-i#!0hm;O7U25cuIW1J3=oKyZs~(aL4|pqO$PKw`OS?K zsYGka0D<*ea`q!9@0WQFM84zq99%pLzSGB%kx(Ct;)K`^XcEMRIQfG4EW1Uayfp_{ z1s9(!0^HfIrzI6~5U<&Wgq7%E4jb{$mpr8)NN^hkK-i=~e7uQ43JL|lKIyPebQb9| z@+>?{Kn)Mkpt{qbep`ftq@i@_lYmckkF4NGJ0R`rUCrN+DZ;kx5!8jna>6Pi1XgR^ z);Zd+BC8hSXh$#{igwHnU$JHm2~jROO|9e6j`{i_3^mxyN zUvw_6(7RBzT%ifGQ%3FT((BzlE%Kxq7BW0^beOf1^J<1Ugi|?Gjr2i zz9FX`IVY+hi9|skrduRZrYwh=&`V4eTg9S!z_%MEh$eDlo`l(DN)sTpy8QI_qXLf= z4|>&|#%o@xV--yfmRzO$2O+;8n7*wf0kSsFsvMJ0*X3xej+kAh#Bz@=}uCqxg zN=*6%U-X>xjHp6%C!=Klf|5YNcTN$p8#;X*=&hO~3dl#Z2cg@~kc&+*1pW}3T_FaW zB_J_;4L#*B@1#=EP4ALM_6W0?7_yJ?%H9a+p~yw7jB>W9BtJph07@F_mml`R_r5Bdy$Yc4WKId_<* zfffCeywcLeT~5xX%axD%IF^}HnWM~GT2LmNr&Wy z;?;$aTaWOAyj5ul5xd*xVX^=V3T@BJ{0Ey@P*8Ds??u^Ry&i}!D)hz2_h5Q%@W*b+ z<-e2v`L+LijNSiSsjT$a^4v3o@1PN%WJNv?#N$ILNxkc_Ja?43-lHD5MBLk*NJXEO ziR=W!woM;33IZB=?{(nlZDO4Y`OFCB3c@o7wltx{=AaJIg&2d&CX$%CW!dNnC)ay6 z)PKpg_y%mn*16^A?}o&=uBUF4mls+4*3Tc{E8PS4bagzGTgR?C>(6fU%RQg~#D00t z=9b7y#+CVM5(z!6W7?G&SLT9R_?SD*C3QKpj#|NVrR-o%gWn{B*FiethJ4ahsC1-Z z=9WpK5wgJ5`7w^WjUcxk?ORPnx3Wh+q2RUbiTS zIVlg8Ie|VLU#C!3eC&ZPNekm z*~W^MeZA#Yz>~LHR=3cj4)zDegRAi*=B@#u9j1X4X=T9prGyFcBB&X94vJ7=UgCN^ zJmg?BCkek6&wKAHDOI>Z?oZ^ID={So79C?2@m!qBY~+2az8KGgQF(-$!gL+yd_-bJ zoyNP(d9O)Mcsmi6D50O4dtRgz454U$9o>N#XSh<)jUG838NMa&HKbsI5a{79aX7S= z*F*)4b@3!o89j0=(r-)N6RDKFNtt*#dgN$?TXKFhj`JR{;tX$Y;+bw##Q+LT7cn=1 zzD3&RyO(zU^-J0*vqex3vKVC6m%nawuwE93=ra$|XPa0vN5*)VNo(u~7>OV+S*soC zs0wW<*q5_2nDdcH!=i`t>f4%>+%Ly^9M36Og?o& zG|~#(tO6$tk)=(jiut2mytK18H#RljKWZkndGC#@6#kO4E2fm39loGqa#nbR>!}LY zQ-Lhc;rT_SJWDX?vY`Of*gH3v^K!6sXXH!SIaU-$UJAsoJbevA>1cF1gE<>joG~G{ zsqGOaW_Sb_-oitoZ3W>%pJyjx)D4M2t9cC5kvMg=Y}FI_t8)<`@J32szG7GA`(w4v zL{4lf=WvS)A#-bFUjuz>JAgZCXf{$f*V3n1BSc;#+z-HJ?xH$T00qUw8ehqp?x9u6- zwr6zPo>6U^%42dj6b9o9Kh9N+LJT29u2zt zVT~I!G#7~Ogt4DvM>)M-NO-{bv7V+|VcaFjj7{JS*JWPfAl=lvQj^3>(F!2(eUkWK zG%Lwf2K~XQ#9N3lxWI-WcO`lpa>hJ-2SI_D0tQ-<)qbxCF0x%qnQM;ry!VfwJ>^=; zxH&W_7}6#>1LJaId8hY4?O>cUmN!s&@?&}Z#q)}=6_^P)72Sd8S9yyD25Pu*<{w{J z>x(r94zO)cu9qZX*c~r#9w=+4 zq+0oZNTrT&5^piX%b{q zf+nKjmb9FZ4v3DDlZ4$E`65(GFxQ2o7y}Ku%oKmjNO8SRA%&Vh$_S4sf3!^HYzbc_ zm2rpIKsxnW`rGap z9Uu2imRD*7(xR83^np>;2Sr=CkUMcwL^k)n7XTwm3g!&bVJ+?;_ZK6ahc!v74@MiM zFR~OLF9u*9q)0IUbkLv0@}2>tKMSPH08)Rafz;n=Amuv~$dM-k>CfUKe-|KCSXUtZ zS<3{jLhhe(KqDT8@^DkC~{{{gp^{;0|N`&&EdjTV@jXd7|&aM zh;`MohuQd*qI?@N&W|mf(l%Aoew*i+OAfS|ZJy^8bDN5Pu!W9Fl&;CgF@FFBTgjtp z62{@~Uudd%1Jyz`b%O88)VH~o7T9&+T0jZCe@YH-orjhzENXK2)8b=Kl^p(8@yTAp zD7c3Lqlc5>f@G14&}cy&0U#l2Q9lH0?LC+^m6f`8Jnz`U026Ck1x#Cf9Pr{a2|*R# z<|f$;j@#UHNF4g5`ZJYyHF26Qtpj3sxu}E)Lh8`pQJZejFI9Z(nHJ*Zi4s(Q@}8lq z0WI>#MqG748HbAf?W`x^ywvS-n+{QPJwlTDw)& zm|Nad9V%0%+$gwH>N55ui+ZTEI|v;BzbBVpc!w@bWOTeU8OurZG|S#n3>J*g*apd@ z_(cl}Ys|l^2aB?0+goT3l|6<%M9h6xGpgRSjR$3~%d5YM|1jW*3L&mdfn#$bAsF*0O~ zwCDhMhN^}3G_oxGlwh09?qQxwzLh%TyO;*lgR&orL=pBdhB+qPZT=JSXTUS&BNn6Z zAPS0pA$50zPvKpG2vlLW`P|{uu>P2`L{_-{4RKS9cxs-S*YYr>IsI*=%`u&o<&SEJ zK42(fdB39yQE&;UA=au7k=Lalu{;#M$SC+eDbzThP6mQbXA4AaqF;_3=6WLaIIkA8 zX0UfKt<@T*=TOPSV6)ZylD0Q}w2V~r`!1HhWYL9m8QMXsSx!Y%HCgnz*b+<^M44=sPSJHD zRDrQ@U=4}NE67g)+#VKi5aH7p1sJHYK-N@3QIp~=7BNpdS;Se>kyE{TGXrv#$e|Pf zu{=L@%3fQ2!ZV|L`J!Yu^3Ic(Sl-WsCnx<%sV|oIFp*pG>Iv*60<63__*ADDdEYf9 zgh?2gwq3w|B@ca<0CreDH1s%y>+_dG?Djq z??hH0RVgXgDRt!0W(}n;ZGKwoL5QAnr+Xt+#$C=TOoOL ztf9$v858O`TatSar7UYEZgGRJX_2F*;f@255$LVl&S|{^U4?kTt@zAir>O~Zio!_s zEW>z$jq$nc^T@0U+ikuQknR7KUSnczRZW%ZR5TH<91+ZU1+#kr!!}TM8d68|H4>{! zvW1>+Hhf1td4^(3ZP2*O&Asc=+_)?4L#8l z&2>jE6f?lRXca_dm{3?a}C%9R2CWM@0hl|S9%xV4!4L4u@D+ETXkwkKV|5S6m z4w){hB=c2e$-LeyFsP6y)sq@n^QpoR*Kle*`lusPx<;&;m{a#*pedGnRMrhHJdJZJ znZ63MV-oKu`YY&;q@eVWD>XZPjgKt{uUB9k#*sJTVa zhD6_ETewWAA-dMyS`2&lS_9(0?Pk^fWU{HIIEg+|w59|aIE?=_h5U57GludSJ+)sn zW4~7j@9IlB5(CT!E@pmca@fiZ$UeN!nar_rT7B3_x5U@(m0}KaM_cS8wuSS9(FLlw ztiFlSXrxrQp01K(D}=e|oHWT=8r_tQ-brMT`S^!`SaBYkkt=AXPGn9c2^uL|E&e#A z5eoE-q`+XyOw_%srjPcp{b#Cw?|mFCXcW0Uq)Y3g)G2+G6(x;Sky1&St#nbksHDt< zX!y;4(0n-xQz|J1KzK5hlr!04qLT6; zYov@-ax{G+%VAr5!cG48)HD6@2^VQ)l)a*i@+etcEAmMy2P`O~Y_OG4*8g0Gxt{8z zjKZy?d=8ddF!cc`{yB*do91TuT1{lXoSP#Obh?^VftZq(W4KvPz!&Y5$-OL3AKSCe zj{2SLx!LwqAuLeZDYuYVU`uJI90qq?&Ha#fDD9Med@Ajfkn*6NvQc@ivei`3PFX8G zDB3CChN%4|xb7Q95h%5|nnfd2DYiUN(nyDxv*}r?IZ?Oe#Wv9%%cz$AKZ$}$ zA}sp!D5z-0ptFvOo*c5&D>aoCsnjU5E7fW$GqswE#0BCHmxwEBDh&XTQd44~yRPne~dps%BCr(m)uK4B9-%?&#$C#a5c?B7IH_9u>Lbqt` zl|3PfLi?$?M9NX8&ZV8yS1y7dSL!P$W4M}!$);PWuXvYv+KTuRy_GpyePxcqBdunm zbXd5dVWHv4Rah)-oRg}sJo5rIa^|^;F_0|<+A7_vu)Odgv?Gd2T3UtW3G$frqOlOy z(vhJ=zTyVYb-Ixqn;X-$?o z(PX*Z)@0FSDrEJoaJZfISw2FsKuB+fK1*GOK8x5{BHGetc|y*D>qMUghquXO?M>;k z0EW_MIbCOg-9(>d@50*wJ2yNlQ=dfv{1XoMl_HB!$u!NK-}k6Nw3DQ zbXlwLHG+b;}bcNa_TOtWD znX#foSYkO8dgOluCm$|x!ws>672Y|uPvUII9P;+b8D5c8&%}S44({FD9$6EJ`?FAJ;P<%tJ*Iw-s^25( z_bc^#Sp9Zr`@r@YvH98`rMHk^z&pQ=c*Y9~kHw0}O~W0HO1h%0fUy0xI-8vm2fOq_ z*R$r8|I3or3V%@R+3vb0D)T6M`0|DxuBTf|*Kb%`sDxJAUD2C~SvwuJL#30i_*iTZ zZmr+9_MjbFnGS7PTMI*}(q5Pj{e8T8o(dbB4*RhkCc*=&%zNyxe*%~ebKN8NvFz~I zGs3SUd~Gdaa8+_^ModLUj^Aa3yEDQ?#9`Oql#;4XQ6<&iZHJ15e_e0=x9zaERM?f& zSU=GY+fLY8Z=udT#twZ_LJuGbk+kR7p}$c1&z7)L3FCZukR=_whdUo~cr4?qWQVf( zp00$4uBTt4!y~=Rx78Z0Z3cjA*&m`mY^3D9^)4j%Y2tYMaM*^#TqVf63$8T60Q~i8ovJ0c*BRHkQRQb*4OFQiz*>9l>rsV6SWNkh-5>iQ5^9-VzlCtM@6m@DdPCBj6D7gn=qsN`FnOCjakHs|$Y`-pJz{%@%Y z#XkN=_ek*t?W~JykHA`(SDmcD%T<8|J4tCyg$WefjM0wK^R6WO(-IhzzY-Cw7&gLWRG4;K`hww4;uZfnZ*oi?X~FJxm|Wt3wM*2A8^@sGdCrXOXdgX7+qa9oQ5+O#$XU4&g$^0d@|n zh3vX}-Jp1ael(-wERWM1T0<8!R<3l=x6TVa`kL8^K zAVCyW&QYJlj(3`ab=1+%CF(6H&J6ukKvgGkUSZh-N|Ql^kd~o9=r2tkVXC`xa?$tD z{v@n|^s(w6eGC^L`U-@YxwZ$VdmB-VwsqFmDc|Ed-=qI&zWHyUdeB>SZXc~{Ah*7; zs1^M*E0WLxO)bXc%%F9RHYnl@=U^x^P!kx9sE%t6qr{F#Bwt?2fp5OHm^X3$xnXCh z#y6UqalCyN%4vlW_4Y>qFu4c1kB(wY`-D$7uFdC^yS}&KbLOn)6uaK#lec49M^t)t z)W0h==mT<#LzDjF_Bv{@{mSJ{t!?(5>a(g!TUiGrWEt~Gw#6>PThfZ*k3G=?S=Ny_ zJ#CG_E1Wa3I_!~FBaqHbzL>z~R5@RIJg5hP!l`!BEaN;8EVwCQMXr}KY~;Bd_GB(d zZWP~)!{%3{7R6P;S@hGE3*9u9t)iY#P2O#A8CZfFb$R$y5miLj7Y14nLK zJfdKKHsq3u7ZzR$dR}rhkE587G1gfnWnxN*ZGA^xq#zXgmJ_YXqiw@=P*z0QmpDyT z`j~9s0Q7+O4#L1m&Y^0Ai|Es*-e+{09=&Th}XVwDCiM zKwZb;j3uVvLia8>lf9^vz&UuA`-nbjOFTX#s$4D%pp%HX$I3qFb#plp#jzIlbwgih|kvG4xX zsTW5@?@Y&S>=ehSNxuKqajP=p5)soYZ9_iAvY@c&c7;VNSCNVH8L;=U{U{{flq=e@ zt*91S$hw{*{Ce&}W_&i}NiwV(hT=0gLOj1DaR$}8b+rRCt2IA(Q^>suG0#%Aaoed) z%s%$^#jA46=l+Z)2xc&xoV{j2p;tzaIML8L(kI+EdL%cTFOs?neOv6Jkn#MZ(muuj z)ui8w-$C-osea=rL60r>%I^yPC-PJli5z7VYaHLD0UPT+C4=i$-2645?gl zaVG|iIZb9qA2l1Wt4I{&NbWAWeM|wh7~LF4VV}^P6x!$1#z?iXBSiuR%(XOysR;#}OMP{%R?bLITnJ z9Hucxe&?d>tuN-A-^3eri>8)lq>qSNM@bO5IIY_55KSy|~vn6f;^ou+Ng#!Ay zff}(~K16}OKKib89eyvM=eD-{3xT``-a@Q+?4bl|GanT(tm4PgQ`7?JIT4`=I8iL3 zrOw&A=&g!|3+8o0x&v6u)3>S_!LVt&m2n{Su7L`r_A34ob)Xz4mAX=6IA%V&; zkM4m%z`-MKU3!303K4w_f4PcSSsRV4rV4dRUm9IdoEsT#c->>%c|RN&E*f*DIH!y` zOPro#9NwR1MS7Uu?_ee?C6?40Om`#If%mFz;8JF#PItK-ojgfcbS7j;AHBhtF{Pbxda2>4W7f=)| zW%wD2y$L^8gNKD@O9}rxUrO|hiP!NOM4;xAwb?TNja}xSC3CE&IwHxqkwv?!u;WO!GJAJ?2^Duj*|)^zofPV#6#-2ll^xnHvA1fL$PzV?s5S8t{m|CzKsJR(gWQtG-ydZ;L1b!$j6i8FS%Xdan@GGMhM`v1$aM{rVu>Rb3Br-U@sBa!>9YP zuwAG#_;gFjAeFb(fu(i?&sM3D+Rjyh`{2=H7n2TdqF5Myx6H9Zd0=#ePu^G5=l;wY zl#IqKdWRAISZ{>6;ELEKcENYwlf7_FNbq9l!_eSBY|L-|hU0mYL%30;^1$-jB1PfO zEBZ(%Hz{>7EmX*bsR`7`jWJL{{hbl;9Bh~!s1dCtR07oB611KhI3vw}0*f}Xtg&KO zn&Bt)bnt8nBf`k#V9KsT8Kc^vYxp$;#!r)~0qd=og*Dv4!AET9X8hJ8j$%DBDpnIM z`z&4f;$)H7-4m9WM;10H^^{QUs58((7GxS#964lrJ}KQoP@u+?3HP_Z2zUo&Q`R>m zYCEsUgxm2W2}L8Fo$oain^Ju|fu$3D(;o%k)h%_o5%iqe8H)DQ`$6gFykm@qtaigh zd{yRKgIF1pSkxoRYBKZD+a#r00%;=mn{HF+ZX(_ly591 z=-vbYh-}g36LbJAgz{-(=B&tJP{9EPp*&lur{_xs~55Fp(ODMh9F*fXv~F z5S?d@FCoMB2UeF`)rCB7hg|;sq1e!isV#WuC46N97TLT)*)83ZPI~dL2BL=#%wMQ@ zMm9eISLI^G6@s;(+nI14^QuBtqdumgI|QhQ%(PKPvAC z5n&W&cbLCBq|_n{5|5epZjrS%UrM?=<;CbsqMsQhm06n#OflxRUb)R@NMEBJwO_Ph z;1n)qbhr$L*#SDbp~NhqgcH|bRpY45!tW<;3Ai3VtOsyuGJuT^S!t37<}+Cfjw3DO zWjq>`EV@;CAJ+E*>^;*K2Ue_+(YyfzLf{U$?Dgj@IZ|-q_wvz0f7tyj#vLub$hJ2o z%G}0b>CxVz-ZYPyiX~7~GJBf!u(fDVM!z~Re;i=U2Mi0s*MV@q>fn$VmmYpqs*mp+ zmG=|Ug{!5HD-~F1wnmQ}m?v8KW;r3|0Ge#uW>Ux|R7vaUZ!iyYpM_m%vS=l!$V~c` z+;7QA{LMUIN6gm|%T$C6(=%zgL>Y#~)K!XYkzwYaB*&R>l<8GZZ&=KY8orh7DlxU8L17rx9a1u&jsIUB&d;U3W$#g0U=`RDtU z;Aw|h1eufzy*tbcS4ukRJYkHq%}(K+Gi(X1E*lI&e~y4MP#< z$x6eNr$1_5MMB`M1cBt)j}Ll~j`1I9-Ul#eR+!($37pB2aY8pP+MqXjPm(2M91(NZ z@w~>jnBlF*F+BOf8CYxMl{x5s)$Pg2zU5(8g*Ut1yXUyK@pyK)2fta7UfilVwCB;R z@Je61--Q{c0?at=P-dL=I>CS)=BLPf~ znKGQatOfihW&4}55$rEOpb6-5J+OiK{PKxA=wR(9UUQcN}+?%b$3Th(kZ>VTJu-O+~cp3`4Zr^hM zX`yAIl3Voq&nO5@@kM76C16fT1YOT>Gt2+3sL(gKWQg&RV9rK_s#&i4c1s=5SijDS zrh~Z)0F+6qc@mf0kQibslXBCKl;$7o1lLOf5R2>Wis~c(pDc^!6j?p|nqev5%hEA| zQ%WgHt)R)G8^_Vzru4MjVN&NDnr6WPKZfirVB{mw!pe$*UwRY^TFIjQWRpx|$%LVd z(xW*GOT44zU?pb74$4;@hebZ#_mW1iHT+>)3j$fHmwJ8~LqG(!P<)LT@E{4xj#2D3Ag@JO2U(fh;T z@jN7PEk&^7;Fcedq2#-(mKoqTH`&pV1w0^5MaBkSDTmUVT=!%_7nyA#dGfoOMclhI z=~^o9c*`@+-@}a_$#X58g~zIz+L`0;J= z_pGMt98!AXH_4(0$Kl_f__6x_6=-}R(m26kH9tW3?ck&woqVLwQT7%JRgV%JpI9K+ z>J7x7EOGZF*)}e(sunbyy}XjK-ZZ**q|dSe+JKX~&uBC8px@Bk4GL`FOZM(?zG>t>?RZt+}s*Mfs zsi*SfxjcCjiCsVp{+!RI!^fqWo4Rl2rdPj4Gfl6?_p1T@q)mMdcC<@VuaKs$CtQt| zm?ZHjg9tKv>!5k|K#lCwn0)LxseO#xMV0^7v!YnEe-x@)k6=DM(`A;|mGamvj?C}t z>1rh5awqwxrR&T0iXEw~si6@$OtNUk6*A5TWd=6_oY)>sP0>Kj3k-9T~%gG|HyQm`BsRYVjZ@Zs_$)d%o$hLc=xE*HU^OUfd6DRO9 zAGlI$f;Zg9*EqUlkgItlvF4-C$#fh7GMUfejk+Ce0m4g<$@;mXMAkx4-DqQx71fPp z^4+vi7SBx84J}X7@>I39XUJ4B$aWyaA#csAa?yQ=c>~oTWUVkSP*wM$YLNm6K9?sw zZ@X6f3ZG}_73_;$(I4T*;u>qGESjPC7%#z5+XR2^A*|q)DGU6i=J98tCKcu@I9h-s z8uuBivib2dYCTTeLHAtn2~5LyBll3dzYM8R-%4X6$ACn<3`>i(#Uk_mR5|7&s$}>S z;vXg>EXcVkp6X_&l69?av93NF*fmVJnH*9~0RP@Bt|!}@K4eyLKW zYnXTHZGt~Z$Ba39pLEQxRmZ%|^8^9aF~iRx*b+TqJNpYdrr3I#U=x5?J))MDG^1j^ zwpMq^_u!HTu}xShS+$6-l%S~CG?4dS#*P+*i}X$w6@OFFUyjP)4~onfWn(ww)>IYx z-GX_PrQR(FrXuvRmQnXqUAOelbk#Lerg7S-_;0;8t9jKYavsj+>GzHVpvY}x?1Pfg zZV%S2Z-m=m>+3HWEn1KQ?IfUO`c_EtQXZsuP@Q6g7hXM#gdIR4R8xMd7Oxf=?s3 zI|M)997=g$Tss1R?h!nn7!g<|zDtLVr^7NZyk82#}ZvY^ki7BWz)&LNgJh|uj=b5>;4b9=yOM&&gw^SyV>Z^}OGS30H68Eb}o1{Y==cG+yfB%oZetN}P$LUC9eX9^$qps+SGJ z_y0`cvSIL-l*x4#_O`C4KjTIKOP6dFqYu-@_yvt5n!Nl)vTEFr$A*(*5z|V9xn( zv{yPC!;6JCEs+%)owcW5@U0dO+8lejFk}dTdeYnMych70rhD-f*t@`R_*KrNywd4{ z{ldnN(bM%5Ht|{z(I!rDJzqsTUorbVN?lv!mhn26@#H2C*%xlEWoJ8*?8g@jC3|h` zqC~#>!p2t5+877xJrQwkPV}J!dCzRWhABqohfFaOLzttq}G~O!09z~y309nwLIr}P~;h$x_JH;?iWG;SK#G75u&?Y;hfso zxfG*F8|-k3{#Q?f>;Q#4bo|v)IW$AK>l<}@e{5ePDeU2T3gC0R6{zHD?IN$GOQp|m z{^WYfdd&F6SCUB>@8*Zq$Wfx-h{=sGALlQN%Y2^C7M5b_h;5VSP2Ooi7O_^=S#WiP zGzP||uEOdN-XHB3sA-bS2p#K3cq{JA3ZG@D00js(@?bBNo|L6%*fxv+b1s?^`i;nL z;FV?Tm||%4j!MIU-A$3xC@>(gWeOHvc}y0)a~ULYlWS=fB_|%?e4h!~)=UA>;pZ|6 zoN-;B!BR{Y=ex z$0}jvK961RWX$~Wxk4VO7*)>>Ru|1$p#cr!VJk$)XjP0#r?h--D^=(UcK* z1`U4sbNQZslRdfbIR~=R#W!nlT=38#7TE0i(D-=hib7tWoj(%EybG!Ge1zBNpdC19 zu6I)k@)GP}dAVHgYW_QDv!&TO!%6y5L$S>7}X* z&O4zC)}3K>0kYX3_P)>9lB1e`s(I6gqBO#+M`^_Ffem29^@!LJeoErGsRjWEJ4Smp z?!8#|K==yX5Wu1N$dF%9@9;9eQMl}r>Xt---73`>x-(hyG(|v>;^Pm>DNvsI($AT+ z@oTKFO?(-f+jh}w`b8_u6GtG9piy0ae~brjo)h%KUIGWCImUuqAF5Ih7t6Sr!bXYc zYbBXO#v%^RDhtKn&S<`Q=saFkXYAo&S$WA#Xp6aqY?jQQchQKwo@^}?m`ai?YD4fA zs@V$i=lXdwx~FlZt&rmIYt&qPD@s5?k#OI{2DOl#si5loZ(wE)jC^%|?=&t4qVGY7 zH<)L$&IC%|YhVh^7G=jvUEJ4rDWg)AD{5}J<*z5ARW+@2I6PB%R8~q1 z)M=<0EID#VS820FH7^G2SYY*a@%{2%=cO>xprOZA=GL)SVXitX9>kMm#hW} z{Ot2uaAl?9k^tQaa1}(dHp&uG7Qp`~ex%!ie)zX!b z$Hx!))tl?_`B{QUV;BqoFZwI`lmZV2Otlw6#W#eVu`%Wr$ke6m4;V#?d6iF1!C&yl z3U=_t;i0SfY>kJmW-+1!Eq5&yy)7d$n}-W~28~LGQI~t@mIaL4 z|6EyQ)~eNRDmX_!6@`qd>3$S}1efW)b4Jqm{hE@T3thSUry$CPHlY@NV^$q?(as1c zDXZ)B{+;_ttZv=VU$MUG$nbiOY)cp-={bl-nj)CZq_P_F3U^uqMkPR~bUpzD9vs9< z7X9i1py4?2h)j#!<|hvdSz{77WfJ(oVJ3l>Nf2NXR5J+*Gbe!z5Tj(xf$6C^fSHX~ zF$HSQUa024D%G@y6^hFo@X8#h5X1-3A0$$6Ugm%-zS>$Sx40DIk36OjzpG^qi`mrA zOimZHmv?+z%?Cg%b%|cMNF(SpPp3WdAcI+?J;i3e_PEU)?HOTy^%Iq*%=}DwFeI!4 zyygMz37D^FPqq1i_DnW6Y0q@?S?!r^{zZE#%-?HIo%t*6Sztb>JVqnUGgoN847b^+ z{nBhRto_m*=4|a>CH^VeFXX|j)_x%yreFJozL*u--y;6=wVxU7Fx}d}NBsS@f4}&h z+J8X&M}I7>H%###)c%9we_#7M#J^wrMHR!`sr^pzw`jjCH0G1qUo8GdwcjoNhqYgs z3|p!FWfI=3{T1S0p#5I)&(VIdonlVc{%Y}062IQAsyCa)rpLKgz*F!l+d@p~j8r+s z^XzTGZXGg2hZGXBNr&`Km85V+CUv^3U@-|F(FxyCr|RQThc0+XhwRoNBM4ckL!Q$i zP?Casb;x5nq=JxzI^?H1#7juM4*9+gK~7yTTZcp>#A=?_EiVfh+vNPOK#X5Gt5_n> z_bzaJX+T59i@XF4Ym$r6KI`c#6o>S<{Ur0f!FaJtsZKux8e;N1bO6h{_ z?X5k-gO_KGUEaUfwXffWa`pzI2f?aVXRv&IaOsBdmV_%Xn9cb9`LrOI+)7yF zlPY6tu(d5Gat5OmcIo(h9Uu8DK+%I+1ge!J$_`&fdjcLxxH*6Ve8A)H?Tm!*GKc-W z3nOFjGE4oviz6dxoI9A@7TFc#v0kf^@Xzf0gH_v!{)FhBiKu+0{u}vy1Fo$B7kWRf zy7tb)voqa30Xo7Z83rjm2-4Ow^l;G-M(jo!8=e3jTy2lpJ6#6q#f*W%{@&}TWXm1`r)-+2!f}#+225b|+QkEmh@MbDoLODXGa^*cWQjv{B-udHmJ7&hQfk&kG+i zsifFXy29X=RyS&v38$52Wjb*)fa=Vhc^7YrcR?Rm@h*R8Kvr@jxk%X!5}Q|ben_kF zio9Jmv&3Q96V6vMu#6VMTyuOcBCxdX3-=*J&M8Xqi~!6LY7igF6`wC2M<%K?z(x&y z^jTzMOJ%9%;LDgS8Uhij(Yi#gX1Tu&8-6PhN+kb|#K|dIgG3;SUqx9YxJV^nDDrNk zz+}-HRmN7cNk@NmqokH-uZn)iioR7xKa+|+S4H1#MUNwzH%ZBMCgCEFIFq^}IyL;i zm28NV63gSIQ=}K{GCmTI*b96Q54OnO#}QSq8Wgm`Qw8edeT-T$=?guu z3f!~xq$y8U-LuUJC(lqr26a%;Pd{SmS<7Tb8||?(>DQ=we_9#bZyO0m4_#>b=ut$; z?RouvkA5vry`%EtFrPW+we5lny5W~03(IkuGZ4^ z#j?1=gPQIZi}(3E<;ofUDDxBY>Gk?C*6Vc;56(JVPZugZOxE--0;R3IjTA_3RkT(1 zKGwO~VRlZB?vWut)_)WDo>)V35D@Y*Ix0ur=GCk5M|yh>M~1e0r)PyvGw&j~(&Yu& z%sG;mt>WpFGhO_0==QkVJ|dL2-B8ish>-XPcFPobTh0!6n_iAJn2C2&U%<%UE+-sg zH~R-VMcF7H0nE2(6PRS?3Z^Nfflyk|YPTXGqg?$`EA?jTbec<+E zn&TPzF|Be%m*ZClZnn<;Cj_Ss-1MXF_L1^ql*fr%bE?>`$6pU2Rugz<0Q+mS)PCAs z&XoHQb}tfaj@nK!eW@b!5!>~2n>t1g=qk=k#dodZGF^prmJsGF0VGkA*Shy1T6*js z!)M`fhOxpHEHw9V!hspKLr2sh76)gXqha$+x>KK@&1Td${e+&-Cs@_dF>8NI$H;|l zZ@3n6()iy*%B72Z^CG4jA#-&I@?1ubIF>o3ndFyKAFo!oN*3)IDrn$Z8Y4|~%q$8; zS4Gy_WYL>QPLa1n7Db0wNoCA;5=%iTT`bfuP=^tM$$MI?6i|hfCKi*BnJ47~%sUhS zc*!s*(g~0Nq6_Ayf+BpXTtW9H`fka4^*35GKk@NuIfRmBuv(6S`rawi0IQVN`Z}ki z*gE5L`;B)v$CKA2S2r?P6V+>y>+gvSG{Xqv^f944B&!N+lZ|2fdT+SEyg|h(M*0Bn zNv^lvlU&bxk{b6o&U{zBCpqzC?@8{MB~aP#Nj|5)rrYmH9^=-dJQXey%b(*5$A3gaJ3lCS z;DCTD4@%1OKyqG-T8=w0z;^HmFi zT4!-WugD?N!5f5bO_vilv^zi7CvK5bIdPkJiX66O%UN67BT|kI;jc%qkeI#a9<24jtsq6%RZYi$o;gUv5qp8!CvOQ&uqpbO( z%^?D;AWnLbuz|e@V1-G=JY9gMDg3nxY`csA)YRnsa7&I%8Y|%kYB19F7~1wp0Bf2| z$Jif~fXWf)fKfW1YpF}Xpw1tletciDNK}ZT z9p7^;JtUQ!*gyMkr++-O;v@~MM!aRg{q3%BVBbR426_XXDsrdy`iVC4 z>ywvcq_6a8qmlQ3sz^>45qA9b2^lEkOnJT-{_9X|5qE$%W#pMC6vlbs2D>TEr%EO9 z>x=2X!So-8Cch+CbWt(y1u&0P947|JqES|KqZJ+b-WHy(=L5?@$?$ZpOL35PAAF5G zf0vRFFX0Y|>-#^H(0HgW!Ujc@>3?P7TyCS<7Pb8XfXT_Kks>SJbLwE{Cgw!4DER zw%Y=9uYhaGqHpTD^&2U-a1ImXB|=`kVca-Nt13i6kwdwrQPMNW&pqm-Mr79LBJ46( z&ZegLtbi@2o;LMnbzXx7H4MPCEd_zP?u!Gl1@6@LtU6@xkhfME?^GG4`D>O7F5iZ8 zt6~vu;kHN4GdCz{-AoqjG-a`P_x%qjiFLry*2Z8~E`6UDL83>q7w+J1t`&l?hbAy;la55kb3w|%oSGLWyJtMV<53UBhc9zU9M*y@J=dXD+C zgfzL4q5Nh=E;lwu4`hd(yyC%DeLh_lIfvzCnAwZr+2W8jLDF&Ycy}^U;9b7CB)fh8 z;N+pVbB?K@4#sYg1li&2LvM5N7O)Q9Y*_}J=IJSv@K%tZvv!HTx6D!C&2DVejNo{# zENI-#9p?8^MlXccmbX;O`;qc2pIZzda$PWX5kZug##%0ioAr@h~*()>0K@F9=_nsa8^cR2Yi&=P77-j4C<8 zMidh&-BaZn`_inCF=V*lP?hJUh5@y{rgt3&tKvU%9E4TboFk!;gCT>w@Cd?B7qYqX z%0umC7}aJ{Pd5L2i&{&{Zb4^-aC+*)=yQ!yFFRMkQuC-#m*_^fmBPwdZ7%4XQ^L)b znj#7x@A|36`@yUp=hD#C&!)s;Hq+Ejch z!u)||eHqI;Yl#d^ z-o(Y~E4LAv)k3i6UiCd1X9EcJ|Ns4es|Sjgv+DEbwA}WX{(jeT_gd~+%l)~UCuq0`;g^6YPoAIcciq9p?#q_D*>az>+(#_;LCalkxeF|J zw&hN;+{-O@gyj}nZm#8aEYbCTXt{4#?hBUtl;u8Rx%XM_Ld%_Pxf3k+8|-f|aP?n=vDWw}pS?orFN=lf)Sni)J_ZODC+;SINZk^>$vD}d5 zj<($MEO(IQIxY9BW(%Jz_chDiXt_^X?kdauf#rrRx7Kp2E!S(gBQ3X|9dEVM?w6F? zFI%=(C;aWdJxhhZ{Cn+A+OFM4U)Jsk8C<%YXXl(G-pVk+G0EX}_#6>O*fHC2r=#9+ z8@{_7QydcTDcM{Er~2eBv17a~WEV}@hS33;T-%-Fje4b(8paR=cO z9YKd5cQ|2F)UTv=5WB!pM`?2%Gx5*Vf&4~DClJjg?hbs@_`c1GA4v)FC%5VXHCy;8Qqsq#~IGvuSG$zv7=kZa4oU^lcq=3NFkfB#hs!M$xUxsaIv) zrcz6s#NMh#_e`tW|9kb_K|Rs}JwkSU(#L|y(!cVjzjdFcza@|UleBgG*Z4n|9=MG! z!94vZb3lTbN<|kw^7)*`dKr=vpR=K2{XtykKa0= zD|cK>ZBou0T{yot&bWQnxI5?0ye)j&opW!PQGeTvTjvNpoH%FJthy5-#!a*{`oiHE zHM2?3aHmRtJ1rIJHH)<LI9>WyUl>n`P{M^p&2ox2pMP~4Y z{cDf0P%HUw|4n3m%uso&7#+bf{U>uYtmvg(?kt5fa|oYCxspdyT%p|cl&E`H#{ECZ zXOD~2EM-e+LZ!c6t#)pQ;{sY$=J*DEBJRZ*?xYh!>=YwaOu0&H_lWd@;OLd~n80J# zknZ0T&a2Ae!EL)rwdZT$x|UEW-RMi**a6B&f#gQa&rVsW_G&B8NdL%RrJ6Oe zMo1ay726&?yS`bebb7|P)d+l>nxuuoJ?OvA^QSZKVAC4AOmQbVZsPwW;HZM%8wZTC zvTE*2uudS8vC`aVm^2fKO>?rsZKm6xdB7>w?3e>|vM$jaH-X;%v-_jF)=Mu5 z7R?~>ze*=%>G%m?oW{Pc^HRrON4!lv#)A7YpBkY1df$;Qr~7o>#|vI-sxy_cKf9NY%1ypD&d}lrs#MdJWQ>sU$8wLUQG*xLz3UjaN(k)Z8BH&-llVZb~m?l z3SD1U%E7_9wVqF7US7wStn@-HZek{kW2N^oPbN~tSm;QI*c%C-pi)&47vk$*=lf66 z3Zy=2y2fz>wf$%H1c3S~F!kTTBUDk~o<#a8Rm;DIe*+aZcjunh{V<7G!N+THC0_8fYszu-+XOgf!3g{ByR=2>6NyLXnuWW!ccAHg znXFZpv%eHq{>0U*RA#8GPqK;&2cpwS*|J`BtwTnkyZy4=Zs9$p|Lwob`b9C~KVN?| zy|THCuKdMSDY&n}C(!AAf?fX{pcQ)69n6v@z20VyqU>(F&ttdqUyYT_u1Z=uj&Th_ zS7eULNdCL}8W=&Ll$`tWD`RQbrn0J^e_!vvn|;%OMzHFdZp#zM|C16<)_>_aC^(gF zM_0L`-LjLF(>1+al9r2zTuJG{V{29K5~%I9VQ{6&hbV&)yqV z?}YhP~T&s7+Qldxs_AG9RQ|*^3G-70z1zWK6`bcd1vU zVl)j93=(>x(@OekMOg(K^xs6*4574s3o0uZEEMWnZGWooObXr=pVH;(HR0s((r%%0 zvXV?lQK+uzyTadnt`pah>uAjc{@G=BE!S?>*DWu-`gEV;WaUd6r6}Ew$B>bz+E7G~HdDD<~o^cl-f6iPbyUHu4DH(;7ZI>XjM){Mz zt7VRgY(iF#^n9I0xU5ZjeoN0uetU)UkzS;SGfv<@rBq$NaI-qkv=qHRS$ct9Qj4rg zw^~-g%+)~Xa(cDYwK!C(|9!sBy(M|O;$4~!_@F2D%?_9GI3%~Tdrai zLuz~XqC+xivV;T`X38ogdk>M-=s#QDtM@zUTz*oXxL%{!OSsKz$WCw~-)Zjf#P~G- zqU%fV2L(TPd{~SJ@?|%=U&j0V>lR5wOptJX} zf+ZrCoXgz%dcBv~ACj+os5lR~v%Ye{ZQ<(rJ8Nb&z+&H_ut)D(W@Vg$WM${%=Jn|5 z?A5zZe!(fO!oK}Z?O!zDwBmt-1`io}`Wfys&-#DZ`wpf9x)Yj{K?K2qt?vsdrvK7C_i@#z^Ens{Bxkj7SFDVn%UjXFL(4L<@Q zCvwP2ual|H%+hMpGmxT*OI7zrLsHW-G$;dXDz8Cfsa>RBGh3@hYW$!isgZ@UJ4J+q zq5`PcKO;RARU~Js<1#Zfsr^!hs1vd>lG75^X((zCsR60vH2i`Umy$e`FWxGW+z&|{81X|SXM~_$v}TEA4<m zEnb^1QARG#tljBaO&ZMw`XyAIpc#}LFUrykL_JMKJ881Dm_HafWS#OemE?FkWuazKE)_#C zS(k)gb`xks43y^~`PbV=%#;wPixboq76bg>Ru~Gh>B=t#8JhRAG?}Cvjs9?P5kjZx zG#RPMm}nS`GzMrIWTjBRPsJa5-m%Ex6S zW|44aY3oUzpKtxK$mwW8Ytxg{bfqg56+_FPh9BK{6dxD93q)Akk<2RR#w=MNUWurGUGDE z8|Bsl%m2SJ!b({fpDP=Mry@)r4oKIua^dOa!nx(b_$bOK|IBjXS>?iYvB%OqLW=k< zPu;Yvl$5g9dYBo*e;mUfXjPaoGZU3EFfh=RuI8S?U2t8sPgoCg3-*quXH*#6!Kl@| z`WCTCvA7t^)@I|ffCMvV78U>t{G;H17c8KD>0e#k%L)HpPJ1~f7(MfQ|HiVH{~61V zW@Z04_GbU&CFHQOe`O|BZo{`k&qe`yUe-TDn1^EYk&bj#Ll zpKafJ8KDZ;0OLhUwpK|G!=TU$*}r z-LSqsoxfrIx7!~a+%h$_CEd{rR|2kv>Q1=S#0S1!&@N z52-2r(W-O$Afs>>+_6z|ghq?|Q9-a>R$^8r9+Gy*O5qFdfHOLk)lQp{OwU3)#bv}N zQ3S3aO}~sR+%O9v8KpaPk9*RNQTlSRe3bi>&NRy z>v5n{jMtCP(L0-(;zT!Y+&COmJw5ed`iVH5)vecUL!q|iNfu9`rrs1f78bh-z`&<(WT!2+#`jQJt%00GU-Bk zXk4bQ9qzRUX~NRAS=!QgAxQ_f8l9pKI^v^XCp?&UW(*(C_5$VhKt7n;C@m5nGaIF&o^oNz zI{`&y^Z5mSHx*4`3lnC6GAP|B4)q)5Qz@AW`dF0H$Gxg4v*ju({(m1Av z3e_0yT6Ab%f*mV_il4y;*LkYXO3ADcZ(1&%GgqMPiZCjJi&U~m9c~}IOguR=6CSHs znlVdnOO%OwhDZx$aAVBFoiX2@DAQ8OEMFB*HZCX2mp9WSmUFaVj*+&^5q;noZfCI3 zTE|i!6~NgB2Y)5=sazjB=5xP;uV+z5-@0c)ebom$`MT*Md{qhIz7~;PedV(5)fCL0 z>+CDpz}E!%c^g3I1Pi7efbVfZ^Rp0dqhvNZOW2@XJWj*R+k)i|g!_4rvKM6eabg_D zkqr|pnN_%%mUK2?(J2D?kUXVDqHL5SkLT>nm|cPuv-7elurQcu1%0ciC4CDSPV%lw z=E_xMDy_XI=d57Pe#9~6-q4hJaE`vVTANtw(mpm}oSg}?n`!0A>x248AI`;sx#&QV z=FG+0jJbF@7)VDgUW&Se_2<0QrZZ+naoh|GHshc5$3VCVY$w=T{Q>4NNN#VVI2sr5t=nu0W2;FA zKP%>^tH@eYJIS2~xz(awZ@Uq}dVJZhY8O71uEyCQ5bZ1*2 zOKG8eUfAtE)|oREUn-Nz{{S>mDid-q#g)tFUfGmcbIweuRm4i;N)P==$JoVK13-mT zo~s#iJ!8Whb+%};lTjK+bNQ610VRZS_GZi;{c4Z?vk$lC$9mWrcuNOq<57+{H{@}g zl4D9gjM=IdOvO1c%K~$Qsa6m>X7@8?I{GV(Y^4b>-2Vs&2!oqn)naXm6w)tu@6qh5Q_htww)2 zw`i`KotdX*2j{DqZi$*DELO9~kJN^3j6DK1>?`!KQFxXOn}vKMk*5~&Rz^M*@>!J2 z=Uzj}9JmE)WwD7TwJ8s_;VRj%N?3aC%~9>@L*QpP|BeqP}IOBUzuPV1iov)U~5RS)Fh z=NQ(Vu#@;x5FlU5%j;nrC-f=5cE)nf7R))pnmI>WF=y;8&fzMpdo45O#Kkk(53Is0 zdAl(M#!M|!=HwlwcH%m#NpEUXtO?p!%^c%1lGc7^p*4>+3zO2&Ha9Eg23xzq)^4!1 zTZUj~evIZIy|t3HhR#?wJ*nM>LdNcbN+jK}e#+y7wbhcjVBg@^SAK298m6;hhO6+h z0!oM##tZS4ui-rv3e$YM51XAaWu^zsc)8Lv-bTjxIV)3U)ek#5zX}WUH}^VYKZE8| zdHlSyW%4+8>>U=T_&F6uwxK?8*P1e|cLcM9ZKO2}*_peWDYLvL%_PxYkS80kAF+(-Gho_;kIL^s}Il-Pzu%}bFeSxh(*tZLRWvt0vtTU7k*@^!U6iQ*tcaQr^)D~wx zQX9q7*eY7AmgjY^ZO)vzx$3f2L(q?(JA8klPqDwcVNG)LlID#S=5Y|#(e~(zP;^2^ zG@=93{>In}&?V9#*NWvrW+G%fAftkeML8MHQUO~9GZTy<{+y=^Kv{t}?@)QcOkcv- zVoP{oYda9x?y3;Z0VzNu%8wW=4+0pUzl zY*%PgU}cbxYdvJhzKNC`y9YW-x*$xvhZV=RQ<%mF*6o+#4q6-MGdE-AnEUJu_Mu1% zRtQi0_9(}V)^^&{UlmV#I^yYECzt0e6wJa;SxEa1*WQA)_qJlTk=D#M+|ppK71k9` zTaM*|_`QRnA?QP+IDT9k#|e9oHM2*b(;mcyTd{DQQ#$FeH&GnQu=f(iU@`uBz;jS8 zo$>g2Wya)jT)K*--?wHx&e*Up>sA=#Qb55ZTS3VxAdcy*jpKP;jN`;JP^@e{)C%jB3gg=z;~VSNOlf>$ zzpc)(C7?xAkE4<~Ca7@6v^R=(cfnbGvkAX{VhE!TVdKb(EVd@cMuP58-jD3qNB1kR zT*Rx-xH8v+Za53OBT+Nrsn7$6jX5gp^=Ks#CE5+~Zim#&j|v8yDB-Sp5d*D=GtF;s-S1ST==e?wkoV zj&o5m7hNaJ*=|O8-K&^F$IeVRu?TvGiF3rzm}7IBaKf3NY@x%MTpq{wOSw4cqHW5t z&p~1R1>J~G^x~M07$*KbZ?vDnr0c=D>X+go8>@}-IV-W()l*w|8LJ-YfVQfz{y4JS793j#nl05=E{-2dr|3M*m>SMbkAn=7EXt9`@q0JU3|6Ov zy*n0pD4+ISjx7bHBnoXN{!~lYKnxTA1E`f4CjJ?yE`|AZ+=)*&ioPm&cjF#cbwsEI2UFbtK`oCxUJxIEpElJI&C<%g6gI|#9kzi;||-f z!)MIcUaS+#b#`nq_A8uiim^JyQd=2Z*1ZGrO%mD~=*zK{p!-sJ2W^;99NoL2JchFh z&3WVeSSMc<&)@r4W1TdX=kK%oDYX1L>4CE=)(Q{Y7kXeV^T1l>5njPyd@jNn3FjiN zwH3}P)~p%bL!!NQ;rtmkOgIj#(rN?JXas zG=I>2vIp4{`HbTDelsq|y^=Yz-EUcJUMTHg!rbG}9d!2hc3`TRWxE&cZ^1A<7huf- zimX^$og-_4@ol}#yg-_}!hM<@iw))2FQBocFWPfaWhCf6-*)t&7|$SUl-AUYHAS2w(&^cVJWYIN5yqE`G|HRb zm1CDdx&dTU3zSRa18LCX<#fi&T7+{fn&fG0K;9IxREAMJ%AoNqFQ2o)8rRRF*o>a> z7+^!n?`_7cyy*D@>Mg=Lfi;}rZ0{L~IzV9qMZKi7NViv{!-oZAXM$Ss@{H$QyJ2k; z!^9VXR21gNfdhLf4rNmrrTO@0RoGLYk5T!QANt63{7#u{_RtaQ$( z`8k6<0{gCSUyk(xDO05U#&P_3z&?QcU;aw4a%=*m#Z&2Fc4ZisR1! zj;sJO4?xOP=t$!gh z`Rb0j=|0oNz~4inISy{P4@tnir8n+N!iDo%!!5Ya$6D6_e{VPfly47v;NtU8na*_I zi!*1WJ??w$nDb_=@BFCB0NP5mp-FSD0@chAU(G& ziz|@oA2yR?yFe?Y{CxjdGS^7-leal@^>V`ap>YU8yWG$&+#73!d&y&mH{;JE{J6xk7@S3%VFUi$!N-9|xy~rp*$dBTYMSHzV+)?$e~M>v`OI1??EQG$ zz#RDgGw1gvN&oIEIkpkBPt>1(Hi-GE$}m13_`r68ZDCjJ>F($Ycjy88mfGTrH5?oD zNm*GAG`8?Ng;ZCU4II;eE{Jsr`Urca)IZfXa;y>Po+#r7J{jaBjcF$(o^O~lCof^# zQ=fjBk23^F%44|+=SPr~hqJ`E+z)%&nBdInz&JI|7g(3nShv-ff9i}%#U6!j1r-d= zT8_q<&K%pGEZ_WPF`QXviW-- zX^K-E8-a8}zO5LCrRy6%4>3k`7$cZB;(RV0D_2hAJ|470EMJ)a;+(}87sr-?bHOt` zJezaT;T|(V0lS$nmqO}GJfp#SZxb#&f#L9c#KI5p{20e`L!7?^`^$9V*I(gTh1AYw zmthysc2N)aCf2O7_xp;t2n1r?bag$FuVc;deSZ z%VP-__c82W*cY8qxARP_4UKFAtKwO8#RMnxyCZY*x7&=fT_Jy32rIis5bWlFzQou) zWvz8@YTLYqPQ|KXzp9A3T$`ab=}EnqkJbkvAI3Ul`0-A^fqCIKFkz4Jug*Ug%ojV#phA0tw+Nfe*xqwM|Wu8*$l?a~oWix5EeE)sU0r9e}uO73}bzUtGZs|M|rg?C_so zT)__i`Nb9N@V~_`ZhICMr%MXLyKjNnI!#(8zQ&7UYzZIRE$#iRbe$%GJ_E}{Y(#Kq zRJ*W{#tA7Y><3PK1CIZYgt4V2!T5?OC_|%RYzU?KmumI|fvpEZdu*WqbUP;yF&n6FEg4mJ8G z#bw|lKutyz?#*6G#aDe9$^D=g-h?A9B6JDCacK$UGV&`$1ZmBeADOI6A+7Bgud$>U zejN?W)^L#`3WeJk=$|x4fw@#K+nrAD(fOiR zO)&!cJ)r!Z(i5^$G-#?uv~>s<#DCW)zIzC{G-@lo&lsTTABV5FLgUgBLjo9kR~ea- z3AMY&rQrR`dnTduouAN2%!ksDO-Oqrr12g6q^wrKc&MQm>2V3+noNAq7!PZ{3)`oa zh6A&sST$L%x9i`ZzRnxcDK2v$W1sLnYt-NDXTCh~!zPrkmU6dCNWjOCnRv^Ux2p8% z(+p))MzW5#0y`!2L_(MTWE4E=qcE{y^obR{YpWM6Ns&>k6F(A$;B&LoNPO#xS4}(N zEnFeZfyR)Aq9fBI%0%F|hX#0D1~10v@@M`j>6sdFI5k$5Nk^tjWiWP<+Dbv$JEeX@ zq%l}a$@@8F{S@K&$d1GqTQ2Gv$$xmq*iY0NA%m<8Jh_%eN3s^V&a!|k;rm)iqxX3$ zr>CZ3{Dvl{q1`R0hsEBmFWDEirRHR^Mv5-@kWdPw2!%=gZCIcVsi4)0Rmyr@U2(z29b<`SJw9v2&_9bac zUrhVsYg$PO=Bp4{K@E(HHf1rqB2nxpU!pLY0+Q(y;q;6lkX*yJ|K<46)N)}#$&}0Q z#k<)l8t9C1h0k_VB2W@mM;4;&mX@JOOvZ4@h>A-_8GQ-dNIGbwvRIt1TGQ-4J(LO@C)sZ_d1R_AjHks%nGCq><*QmV<)8LKG^ zMPf!ou|&Sq(&E?%gmq#ig2XUc4Aroa!b0(3=~&U)GXEq^{J`)yd`#?@rPE=sF@s4& zQu<&?hf$OxC_O$4(&Nfn6`G!w7^cIHmxnQn&%D!zh#6{{pkQ6stYn?27hA&ulJQ-) zE{lE42o`sw1-cwad~arDK}o{oo9)Y}}J0 z>n@^HNCK9t1_D{z*%uElfVn0QtD4MM# zjc7;hq=7$zA#4MML(|g-W@)Kqu+}_^;kOmWJ}pZ{?o7P(Nm;3&*_48Pv0Z#T)e@eZ zn54^Oiz$}O$gjM{Y4A-(dsT!cgT4-zcArpQZy}CGNlM%h(Zc62uB0`OU$`TLZGMtT zxP}}n@Nwwz5eN&bkW32kL(y)U1KrY+$vXn`BROGt<&PKGgN3mlnLbdH26Ny;m&~MO zoD8c9Col}nc4-O5UBmX9pp!FnVk>AjLQ(bk z7%~MSSQ3T0W@$2pl;!D7aWrbg9Bn8pXpWdDR^5cpAnG6tI{a$KQI-h*PH}0N3Q&eU z<43e8mMO@=dzj4PC{DEY10lE25y)G`s19tF{gV@;SVve4$2h4IrD%Avc}2We6sz=# zQV`9)5%)P!H@1`Jb-DKRz-cE9>k2kOe#GEg0wdc;u@T}LEefFA(PSxpmlWfugT(Ef zY~L=RJ8mYVS?kJ0;MkB7n5~J&>73Fdf&)XPO~zT#iF^r%Q44j#It4aVsgU(8UPFr$M+zZ}@;>1(V>l;R(! z#aC5yDuQ1J_)aE${liE(7D0SFe>@F!BE7o=b|U^P>gCrD{#1mah29H%=1 zVVEWZ{z`DfjfYjS7UFQ#pX?Wz96wMjd`c|In- z97@O)Q7@uaM5*v#F{~1?@EsxFb`d4HiDEcb#JVCrZXw7UM4T_8lutsv7%x0k!S^vO zg?v(azQ6eQc>)EDeM@=Yx0J8jOsJo1RQ|#n81yb0Z$E9YfGV;4SIJX;ZHTCMsEFP| z`BYZ<3vacsu+D-U-rwTu@p(&my(q6NU$_{~_Y=y^742I05A?@-T)ZB65ds!B7xaJA z^3|d~Qva4$t9<{e0*%WrpRc_B=L3x8@u`K;RbD=?oV?yhnz8cozDzV2J^Z-j{2%EbTTcED^>1pdPr2s4)L*O1 zjn7M_#yQH{Q~#FoVQ(pa{w?J_%gGyS|MK`$y`}ukw_G0%y`_BKTgv;2oekfHW6hh?gFpiTsn7RG(Bw`LGl(Vfpm(VM$KH^67sR zmhwqhUS5iquzdO-g{6EF{*iq7bSdBebtJn~07?6SLgeY3>IS0YA-*D@uTYpJh|VGS zvVcE(5K_y`fBfK+}@5Y>$zZ1{3gfohNz1a~9wb}Z;h?I3wQh;$w;Vh)hnG6qEHV?m_P z3{X7~Tr2S5r9jee1&Hcf1tPi6K-4CDM$f0815&yhj6(JyqVw86d#PS~3H7R9d;PO| z{ZEztdS_>X2JK$!J^Fu2!+*Y>*A4;k6#3dAp#JA|`xk2X|9nWq=85+|CQ5$!Sb+sP zHfKmqHPMtGUlEw4D+L_Aq!1nq81o%T-VE@(5c2z$Us$fDz$>oN&xJ@|5x2gfd^=Ho z8{|(cD_Z8kzF&58S?`x>C@p{FQa(plY$#nX^u3^FF&740@OoMP^!e`!pTI}TNFk>( zOlbdeoTEtCzg!NkT=<^~=B+foPb*)~znxBY_&r+~K6#D)orU<^Z~;}30{X5IFkfu9 zRJIhB#w8tZ<+h-`k*?s5LVjt_a}&d~$I^APgOE;pFkN9{SX`u-n;51&m9Cp1g1n;; zWCk(pB!+XvFr8WG>LG^Rg&^}3!yaPzaj;MxKE>cKgBYf>0$uaPa1|lQvc+&!F{~29 zYB7Aiy-;2?FCB(w@_^JgQh(%z3iaqkWa79n5$7jF`75@&AQvrS4-q3p3==U_#9$G9Mf4HTQ$)3h zDiM_;KJF&eb6><_5$}k2Q^X<>3q`yn;&~Czh*%)vQ4tS`Xb>@9#1$eg5>YCDwiuow z;#d*0MbwHID`J?4!6F8T=p&+9M3sne+hsJrrTN)dGQ5cSp9ppd-u3t3DDjfk4_XJP zPH8=$@gmK6QbDg3(u#i-)`hT$NQ(Z`S6IMb%)eH!L#~MB*Q@ej+GpsJka$`H-UIal z(fpkU+6p=Yx(9NAF+4$?L8+o)qK`O{L0d)eUi1`rx-}P@= z01KYvLzEDotg%DzCrUUBv>m)2cn?J1)8?15I#5wAA_916F$QR`55vH8>0`1d@T3^zXAnAM(&f7{CIX7fU%2s!Zc7cc)2gm z1Bk0J5N$v_VE||yc)1Tw^2HgL0zDB=xDIp)JmFrEm;2fzU!0%f244+dmxKpFD)7ZX z&otb-4=~{oZhd)P3+$N={UH+zoC6}8sJP3fDRdG3*^6Nc-crxx+R7oaJK#~*Kik72wu3O)?DMdS^@h(UrrvA{1uq(cEPeK2Er zR6bDh;Ym1eJlckM!Zn~R;PZi!uTH{l6EIHT?~+gtx&s-)%_2{DS>y>VCJHiy%|xCs zTI30JBCiKZzCHUCCA1H;w{JmHBA zjMasF0dU|(%tP>6pi4fM8SrZ0S0I`T1;87du-Abv2Er8p%O!c>IuMQ9eBh?7m=B0I z0Jm=wd7$K1lkk_F@G}WtmxNZk7`qR7!UdqC;PZeZ;1kmqGIIZyo(4X=GuiZeK;rp%_c>gBaf_S-4OY(QQ_gBm##1ozdT>>xnODXqTNq#LmJ%OHa zCOmcnUlTlGCde7Q9!UNthl7{on;9r1*NL9B>9z`30eUe!u6op;0X_c#)2;Z{s3x~20H_X!G8|zae81qTlkYgU2SfDp2~eXlE23;8NxtE$bb(6 zc7R{T-Neg$G?K5zB_MB%2SW1eSP8t`za#m6458 z;;VA(m|C!T0Wh&T+J$(z&qngy*a@v6&H=3hpAQ^b zN9YFW32}E0pYb&9#g>=qa4EVphEC+ ze~0Av@F$QS@`MiXZI}RF?$eO`9d?Gl!gYuzOa?6iFZXFE_h%^gNhtSYNWKgG-s6}v z`~VVm04c!}Zf^zuFDR3+08|8?@D}I{c)9;U@=f?HC>MDNAA$7X<$eRn|6mTNZjuR) z zgd0FBz{~yR$$jdPZ@ulH0PM4bCqUld3Bx1M*Wd{kgGx{i-qU0aBQZ}AFZWX?_hTpb zbtm_QM}G3UcSkwMOXw7ZISZb!2IvxaLLbmk@Pu3sj9c)8n?SkX@vat|(i7_^{NKs_ z-jPqd*&r?A3B#kYj)2FzTC8CoL0<0rj(p&yf$||kI1#i6JmGAS&jU8?i*+3Fa=&-v z|Lz8;81aPG@NahsJfWM&%YEFD|GOxV8b0#~CxdLj1WC-)sE_b*4j=k|hzLx%7?CjQIHemBs42>g(PuLNufBAFDRZWwGh2zi0(5g5;iCu{>E8DHSU zQ6TVyyFe7b8`x5hwG`*=R={Kst$QiJl_I|yxJTsA1Fc76{X$+FU?7O*bujRe$Ug?= zjKLhF@_{wR3T4&=-UVGk{C!}^c#I|Rp}-sv=`$ACVFK)sj(DK?M8t!)04|<{xd}cG zxEVxt-U3`Y1v(&p1yD5=b>aB{;VjT@#Lou0PZRh`z@DHph>r%^Oou+;3CDp{I@kwz z0>rQXz=OHivruLMu-6Rq1^8&-6%f_+7+5$H{Q{n_<}AS`gqvoANBkC`=^XSmcnhFQ z9_#~N4Gdh0y#RbL@EC}mZ@4YP*auM?344e<;bIW!mIw4&F6iJ5bX^HO-$!2H8IT%0 zVZz5k{}NtY4gC>+33%oc*pp;{B_OJcaL_uz4%^qmclidPE){SbC>Qc8fQLaOe-t=! zBj!Hh^+095(00PQAkvet&nEOWWMY9=K@?vE{BASWK8(FWVErv96Fm9ZI<^)0!509R zZiCL?R{*~SMS{Nsboxvv#~C<}c-U<|(E9-PLC8D?wmvAd-4{4P_|# zE3^yzO`z@97~9}gzz7hvF%mczL^{t0eslz52{L)W&p;%z9as#axjVqmRPXcu@-U_TJ)lK@nm##)Ma z!r362M}#fUV9!H*0&pdW%3lpMh&IEMOyafsde+St80p=L^9>A+0J@_Kv@kg)=_yVBYV`1GOTo0o7eBirJgf+Ys z(BU_McLt6V`Nt*L3!Vu&NRlgmpxoko<#^3?cdcB%Uxw%6#Xvit1*WD7s0QL^7Uly4@RyKZ$#5sk!Vr&<$dZ6mo*qc) z^Iwv;WBn12ElQsVJP%9)V%q7G0d;``fMLLaz-(X&Fb|jtJOoSw76a3PZVLYASmq0S zAK@fm1~3D@SzI!P@XL!YBosCQHAsSFo9~@WdSB zaHX*d{=EoEpOkS5)=MUTXuN{;lZ6{jRIs0A;YuGUSOqMeQhx6#3U*7DuAZh~1+sAQ zbo`wnnf&7!3WlR)Y5t?L6l}XJee7J=7mKn~e!&6-J1R?Gv`E1=$iksZ6zsV?yiCFN z$ig-&6>Oa>ynU5|U6O@of1-c~in98`)+_k;K&1L(Hz-(3nZAoQDp+k<_+Y+*;b*>5 zeYKkuY?&>%HI=2uVw1p6T&4AS?ohC6 zvas_`1@p}=OONzYGM^b`;VB1{tjC11a8U*y}(^Kd|L5K0lQoh`;D|2wwQ0cm1yrMtKZgUT(z%S(ZrJM3b zlfJ$nu`l~l{jbv>_8Ne6oB8m9guGMGX3J#AR{)VMu>;2z0egY!gX9;DKa8o#_u&r% zeMI(ln2We-AnJdl+qJ?U@Vf~<3*-l~0ZEq~RP2H`K#=bS(* z1^h9J6G1Vco<@jwQK4i{dLk2lz4ag;Yp7H&*}Z*lpT<^{voLnzgYx0GH= zm+Hq?cYGS&rZAFui^?G?)GxGuI4WKsmKkGICIXNZ+D$qMHp>DM{T=m_?a7uxSyKCj zddvDi3YX~;gD7O<`!5^}mGd^2y!}G|OXUeD)Gd@7gDB%ZdUgEU)ZfaeOsRab1@ZF! z7ura+H@1g74~|A}*?tZdS`YpHG2SFav3q@SSUo7XSpk)vc&<2d6nXRNniH_29SUOP#)LVIZ5Q(L9=B?eU^ zSlZ^Yu_)I`kdgX8-Y;*XLG#jj%gQV311V0bSL$mazHIF-8$&Uq_#0nRxxa@(e@Heq z?uS?9d*jM^!8hCg%KAcDFUdx)-Z$U2{#Um_T6bye%llRs`|>z>eqp}6jRsPCsNa5n z|0`b)g@wK`&i6L8m#rC+J&pH2p?ty4LVe}u$jkZuT>aGkvT> zjPt!+{jXlj%Jx60z0&xX!ewK^SU+iIo7(@A z{m%mb7c77|^U_sfy6(!e2e)24bIp74titzIpSk$Fc=llWk`lAny)R0N+CFc3e(y7j zE-zl((Csa$UoxuX%ZpWCJdb%%Vj5QU+5K}@p1IzSdFC7$^UUQ;%(DkkS4xy&RbQ$^ zjG*Tfo}gIJ|KaBp^bCzi0itJOgi;(>o;MNWDV=yq|0|bNK9wbv_h-u_ed(E+69~^_ z`RB`o^nB1oL<)O|JUw&7GsC~s2C@UyNo5hy^FV6HpSjGCiFBvo=mYW(-BXspn00|6k?GZQo&glEod8i;=C zr{~jD7uk#K_qO&Y-Kj3J74>UB5ZQ%nM!&Dfo(m)N#+RxE%-IRgvD+gHl|$ns4m1Vy z5z;=v@S23;$@XfH1Bmp1KEzU2{2q}EKUZkZke$%a_G3^V%4RvBUZ8p)>X$!psdnLa z6r87Off|B*KtZ4`phys%^}RsuJyAG~H5#X)?ti3zsqU1A#yFOHvqapd;)!*oqX<*q z_5uAB{rkgygYhL3sQzXAo&;U{gOWf2-($LN#F8`%OA+d(Wr&s}Qswt-PHCh&$?@~V z9!S4CMB*okA7FNrUHLS^GeLbpe?fobF1;M8qkB9qbtf&NDQorYtXNBJ(xQ7-A<6-0X$ z>`(eTfk@Bt^OxcrLHvBB#70- zT~1_wDxb~`)j-g%y#Ay+wTIu6sS}!FzK+EHhxxh51K>n{q+6at!fOb+}qJP{v z3G>V4AJo4o>a>&g-q+~Qk9nG}SeN+mg?ZBCH2qd5*qqiIdS@XE&nirTK6+*B# zBF_0A)Ste%{yqJT=4&PB<%cuUr}#@Z$hRLvQaaOY_we^Ob*Rh|X^`&Tu(F|0e&U{=Du=QUv4$v+5X&#eAjx z^;Pqgmm_%&gv$~7)AP}P!~S%fDLoH2z!*k2v6ziKdFX@4c%Y0R$#wJ*f}igi>zUum6vN&gO_ z{!5XV0O|_D-uGwk-;vYkqVp%&r&NFRH~(y<35HJ$hCTMzwMeJ&PG_4>LH)45I+89l z!D*av!daShr=zs|a_k5*AP;}OO+-pK=x^4a%IEcuMcLH9WLIE|Q}CREG_pC_p4NL> z_ua-JT@A7XmDBxI`t$qma>)Ke`t#=>j1&G7m{=f4y;V3Hkj)D~G;gPYs&7PkU63sZ z`>QnONpFe3mR{7q?;@Ye3PdG?B0ztm{asK#oj)B$qufl;5cD{2J zp9bZFdST34wucO@N#Y(Q%~#3pgyrY2EBck5kJ0l8{+-pnz{VhY&hl3;j9m}dJpko9 z&O%}HaIf);FwQR`?i&z|c^$}Q5Yojn<*UZLq`Q&+DBm6BcSL@-O~^kAM021S$R6~z z7s}($1LixJn!(x6P_RTyo@hipz970 z%~i7d43Os`$Wecjf;dw;0m#M_EKj`dv?h^|8iaiHk7K<>IaQ7!yaY56M9+ZU0aXQ2 zA=HjHX@dpiXw8P+{C#dOlz#$$!xiPv0?`~@2cq?!bPu|PJ-{EbE+A77>Uhzn9LV}|70fV!dUw0On+p|GaY zGmgoish}KC=ld{C9OR^a{sY}fZC-z}eIrm12+ty0KM?xY>mtHKK{^nvqdh@n8yW{R z7cegW@S=XAn{q0@HTra$@a&W9qXlJvQb05xoX0?h`ijQG8|cpKuYjx_s4A!_x^1p7 zcl)9|-@910LJ{9qq?W)Ypo$=}>l@hLVF<=9)_dn1Y#~>LcP1iGb}vx0h`~UQNd4rmRX#b*TSoFJv2hB5@=hUCD3C-2l&DT=BA;a^7u}-3lY9Oko9;hLx z5vT#E4yYE$734_ci~5V~PV@$ssu}7Dc$pj!ClM9%l3s5bhlcXySAV@Sk`Awz_%G!@ z3;bt+|19utSwMJQh`w*4>v!<6nErUwi(eu49MA9xUl>NcxJr2a#S1!t5{?#;6Yx2Y z_g*|#Q25@95)mJZSnMRk7dI2IFq81n3xkOIdqS8KQ6b=Sa{)_4d~7d-i>r#*R=~nh z0vbfrzu?37I01_l0v4J>b<_*I%A@^@LLRG%*j7k4w0&`a;?aKnDEK#JFJKCU`Rna* zQdsBqwpJc=rmzhO4hWaFI14kxL}#q-&zPEp1hk9HUi8yPLvm}K>-NK_jdM4wd=xmh z*MO|3BNfl5*xk8mUtytp&VtxU`$BCz3_wagA=JG7YD=-bu1w)VXB&3BVgOt-;NXV*q``{h{GcTOCcRp*yuN26{ZdGxdE zj5$xd?tOpc>tVkPKJnz|9d)Wz|M#znUkf_WgC6CV^sm@s+*AGD567MOr1!QLSAwowQ(ynwFl-RJZV28SG3Uvw z)a=c(uC%>4`ssz?(%O@O=Hw5uevC9&)ishaUaT(dqu5--^8J|^|l;`&8 zhQ`B(y6zt2G2bw!UC$j)Zw0x3)BD#ESu3vgR5ps`<|`T>=)C#G@c|hho~d|#*-o!* zE^WiPsr4>JEU4a*<+fRAc6C*aCGX^%Oc^~sw+GvzPk#K(+T!VzAOGB7nzH+>K&!2- z7Yhrge?tN2%G*n&0qPw|eE%IY$Z#YQ~OA zbhC_WF~lamz^%@D>kC^Rwsw7X=aV*{O}+JO#5t?V75DX=eEsL^Riekle0g?i_syL) zkLu{rHl;1+tq+n;ho&B_lhS~7@ldp_$!yR$qgNfg4McF z{y}Cn0<+={etKf$&np|Jtb8%>QqG7KwxcGcojn{or;%Zh6I*@5(QDQ2I_A^9eYUXA zp=-;h)w+22#^treCljyQ`uvnQ4b*D;_Ae+np<3 z=#%W5c5Hyj!fAWd`a%0kuHBem_1$NCSDU10uHGKjC3;MH%D$(K4>Y;<^{A7J>UdWk4rh)%E98wRx4N6s~vy1eOa61svlEu z@?67#H^&UH-LiC(e%Sa1MH9YFXuYgLAM+JId$+pMsD{3NR=cm@u6_51>oiMhjkdeI zwBICCAN^^cea)XgDf)D58|QA{+6Km`e9hZjIoNu4?Qi|=EFQb}*UA0X&ADFPtKi(d z)TnXkk0%u^4*2qmXI)MoDc)dRylqa()*)f)~*_l53BDjADwFLQu2k>8mB8Yf4Z!>6m@*w1FvbUdttrU!5P8( z6}7e9di<8#=u@^W!ga=#EA3j|-<9sMXSu?>W&Yml<2w#goUZ&RH|YJwitE}W<^1vf z!wL%$GG-)4d(O!9n;1I$c3%gNF>^oItc+GP<`Sb+1+#%7IXE@|MYbsRQuu-Po&2gk%1UN^m6enaXO6d+@{9hqhr#X6M`J%9cD1;(+mhPr z&7BfVFKitUIP}n9hp6nROCwHv&TcF$j;=Vj_lA_8_SR8-mpebtRewXDdHA<+MVs%f zc+fB6qDo((z{ zm$cGadYsnk+E!|+OGb?eMJ;y&V>A zsU10K$G4%u73cWm9Zp#AW%I`&n?IhhZU4@xs}&87cn)(}yvHiaLz~B4&xrexI~aXt zPTcHu9pko+TlU4s6Tht(`rev}SGKT6jdp#|;L+Yso;RuRzQc^)IF|+Mx;=cr<;> z>fVdJ8~VoA`Qqc`$`yCnkB85`kQpI;a@$*eYmu@>v8~FwErmN8Xx}%lRC&V5_p_>9 zba*_m&%CDFKdAHbC4X(d!wYV8ub^taH{?RW)=2et2U;Z`*y}go;XsqzotxI&tYe~b zEcnSK9827;;NLtK4*0Qqj$Ok(nKknVTRpt!86VT6?>F}*98w3=99q+V$vORxEruU+ zdF=e<%ze2A?~&CaM?Gv^&~jfqm((?;qBi&JmATq1-zFUgB(L9pcC9wfF>QgqYa_?r zYYU>bWpw`Pa@9#^@i;MADj+@Ki6hJpKvOcvGr<&^GO z`}a*h@EG|~i0_reW|uWno2;=ubzp$gg>#2iRN11fsr}v~`hsl}*REl;-FsReOT5~A zX4S=CJxN`4HQ_y%o4Qrb11_Cwb-Tf>$8!>{v`Vg1d-(_c4i+_A@Ah2t&GBB_&6iII zNeR}U&@D3jXddBOm|pwpgpUh@9NdDe8`r$4S>e03)n!%Ntuc>kcW^ElU18DoYj-Vf zI;_~>Qd~*|tYq2ddZHMJ8alx*F!(soZs>;Q)fi$^aUyl?HG zt5r5fD@%Sgx!beSuNAcJ8yf9;nmf!nFy`q!zo4OG+YilrvbfgffwzWVbV#ZH+c>7{ zvT?-oTBjUZUmmaS-F@eQpDPUCd}pY~b6t~?$DQY&Ur?!1joz7eOq)Laq{M0Sw#lso`9~^ht>vT;1<;HsZ zmWMhld~w0FN90B8{(BeIvDfWT9IMsiJM)-Y%Dn5!uY=epJC$u*4L9uS)~*;cE~?M? zE;F+fx>|=O<$S(f@yzDBNz$}2>y~Y}o%8tlp5J00ES=KW+hoGS16@N})%ZEDQAD3X zYY*G+X07bHY>D=t*Ah>lKK02cm^_H{YrOWGM!JS4=Q{Yd?ZI+2BQIKyo!BeySxlKUEHI>+xHi;jQ=f9eBlSllk;UhV?hDMujx+9nXSm z+a_D*t$Va;|E8bDj+py&Y392Ra@8H#krmTZBm3;HFzbf_?(OE+PEMM4HN8S%?}Tfg zdyjnfeXe!x<5f|??bT!1hHDmurqKuH1vTMvTKa@Vna233Qn@tC-zKr){b$ae)MVs= z4XIauQESdu$j{)O{}z;en+u;6J+~&|A^^|#&IcqJ`)As$iwm7h) zwO{XG-;0O3HOd)TXSCUm>kheIYP^2c*Y$?}v}?T>TCO<~ z^R&jj4RHgHL@Iq9UIZN5`{5neb|+e-Jli|Y<7q)pRyfMH+vQ#t2Ys<*=E}97^wsZe zZIwP-J+M)oi;I1a+I2KrTCu^h!4)^xZf&nyuU`|GeZzhG`Z<+da*wQIYh9N%U0qY{ zd9~o9gr90%HPe3gi(`vJLD#Zaui_Q+AJ}gGdh>eonLk;^)~k53&T{9WO5djsuboaG zGV?^=lU$iw&#nle{c0YZsX2 zecI;GR!2vlofd7MDNDBOcyzSZIHmf^!^Nl8 zyLD2Ot7>s;g14tO3ci)wcS&)>1a3rM<+ILg&Pg`S@5Y+hyVw8N++<4hnD#dcf34-y z`1ndY&vTPc)Gz)~kuzmL`^9YCwdIEc2Ap%;J(T-S>o9W3y1J_yWb__&qQ(7X<6{Pg z7vvvQFG(MN{B&A`N8IFqntjJNyX(1f-^JiAx9@Hp{lz-#yb!~xYZu-3jhKAFNxxqC z)yxUcS3KSRNoS}1SDN0gol(j49YfH)=Eal!2c`b9Zh!x$D|DUG+_FdY9w)p;ABoGaI?v{6|M_j~ z+D`BILG>ppUGV+7K92hQ;PHvW=6lZi#bWNftL_dajVvEG99>jt$N1V#F&kqhFOCTt z=IXm}@17HuEA(?mf0LQkGBN7ovwlldqi5$^jB7dIsGa-O9p-CHTWsIAio3aW(L&|q zQ4d;O`Yp6YaNRmq12SXUj`8Z6|MLd@#&wGCn@qT1xaFnW(jjp|?m^Q#KcsiryRi}b zJn3-z7{qp6?sh?YB*Bkz*+T+gQ9etPe*j1y)H7g6hYxC`&rMNp!UGiXa`j=C_ zY`lMOkFCe8c7IsuqG9(!o7IQU-P*NvV2LNIpwBzxxZSkn^aYdL8YP-l>D%qf48M6l zF6q+lUf=D-9kySsJwLiuYT=-x=g&Dk8SugVhTY%a=F+9;hng3Pz6s3MRjG1tiGEw3 z+Ey1{Y)Xk~yT4b9p>16r*4lM(En4#I%C?3re;mBE>8cwk=U28^-I{Y=@L6tysT-Qq z?Vq=G)fXo76;;Ok(01STNBc6bC^in*H$SxLESr@rk`6tezpdhk_t!6t3v3kKaNn8c z*}p6=p4R=l`?D7(KIoa0RrM>wEJ75n*jQngq`@X=~l(ie;C@+~O~G zSe#mT@oJ5QV^7^O-F*JUGRs3dX8C;eLY3C4xK{B6*L}C&_Zq#sQ|tICNv#bxKDpJIkUd!{foO7KSRv&WY zdSzt7h|~j9eUCpGYqH|r=?O_^3;Zk`Rj>2k&)qeyvOwd7f}-aQ-$qvayTt)B?q8@g z%o9_)NH_M{Sl=pL?;$bNOPSZ!!k0 zgy!U}_arpj+yj@~$*hPU^=R=q`XSl1%qjHnkR{rUi?8TQs63q(HzCo;SpLzq(=z+d z7C&OYQ}Lgov3}0HJwd`>AD&srSJ3Q?79pLEG7SIVBXOndg7S{?xsuWKTT;{}yp-MN z{N)lySNzqC+|0;9ET30RR|T(KXw5U+aP-ED*4FB1_Oat$Igz4b+ccvhr1`TR+M0{D z6D5)N9-ecJSj$R1T6{__t4=f_#&zO6wOJ`EtcaAgw7_M@;+$0CYD65~>R$44nY)i* zy_MPDDjm2=H}KT>zpuUdx=ggV^|F!LmjwZrW+uq!Y;_B6q_-7KXdcmKG(6uVGVdFaj=?)IJCwBzA z(R|rBVSKts=A2#3DwpG#S2o$6XgZL(-&*CN~HdPi<$IDOOhkv{48{w>qD zJi*$adgU>-D@Xhd6K@u^r&`yo$?GG&JLvkkmn@yRYzf1rfk%T?z*1JNO;ze!u(=te4?^n2($KIuySyrOB(m+_YqFMd7JR zK56TLj~|(ypLt3_(LG~U!PrSnWg&r?<4aoattshf57F*$CNx<}^Uo%_yta>G#MwM+ znXfY_)`7OK;@kA>+e7F_b54XURFY}oS9E7Dm#i{Wa%WgX=6fxO%$KZq!&!JIK~Y{s z`tX>i(~XR0pChg(-Iq;!vms_;y6E(h2^od*+OmdKsm5uuM4pz>i4Eemh3;-5Y;%5p zb9$hPLSb-U`yjf(`eErF!_sf1PHO!k<7TbXK6l{p!)G*-q$5-5E(dO?O3piZ=y*$^ z?BVI5n*u|;P8b=t3_ep^;!)JPJ81Ui?EIzsDu>t5i>hB76un>fH1l1*r$>p?W7d>E z_l?+Vb|(AifyKp3BGWnX^5^M0Hf+}GlM{8sXC1T2rzv`lddS#?=QNxw4f}>HJHFK= zBlf)EC_k~_zcsubM?Ol~Y;a_Zi=5w!l|w@I>|hqvyf|8JQ~Q`dBM<2LL$+N z=`d4Eg)WwEq|ccrCB>vr1u#<-YU=DEKo+D z;vJe+X=XSHUHi@2Lt#4E8pSdk{j55s)TaY&OW#QcX+7@OaOE#8)8T@O;i|X!j@^5D zP;|v~^vfH4irZWC)<_#>_O;zmH+QLv;rp$ZOD?>teA=;bSAD3itJHp_SmuHXWxmqp zQSMg!O3chH9}dmxxE*a35ZEGDN&kDzY(gUQL*HxS(HT{@5$+ z!Ry>LRMq8?3T++iRPTAll3g})&uZgFqoBPVeh<|<{KqFB`?CIh=%bpms@wt4q^}AV z(jx_imAV++c|R^uZtt4X+h)>-!*XNpJk(F9S8m7_keIj6up@b=tjDW7vj&!mK}9~j zEhm>VS$KMgx4Tqijc1ygtKA+UkE?>2B?Wtrj+DypD=4AEZn#kFtRQ6QB$xh3EI*$= zPQ!4kK-`9h(T04ZcXv!!*Ad+O=7{$21A1%rwGA=QFpMEq$X}%QDVRMxt7YnDOOuwg zHxp-vJxx6*Z7h4lVDLb(eAkLtsq@Fj>yPU&92?wUo6hnHDVgzjtfOwQ+B2ed>6q}m zJi~p&LG6}DoTv<+Gtc-`X5^Q}p1)#XW%%K`$(nVxd4{(IRs31j16wrt#<(TsWoRT5 zdXYzjS3GCfvCPttGel0l*}wU82CI%3xG;?7tdG@*KOUuVxUXeMc)zU<9f7s#C9gUv{*_FJlN%2CKB`KU( zV0EX8zAxpxL`Fr;Cl!UyjB9rw(_BOi*;`go8R4g!6++#T~YotT|(0}LNxKqEK z8~pET)|O1UlX=eE=&PT?WT^_}jm2v}s`h8n)iexaSEubYZi-|*95#NJQpoMfq;HH5 zLH*YW>@$hmydiRtiEePLmBiG^h+;v@VPf)r2R1iFFOw}air|ZycSXqT+M_t#z-(Iy zLrG&sxqD?zebBT;;7fS8dqpb}Tei$pDRdMpLrIMul{=Y|GF!5*A*O^A!x=1~L zEM9r-Tff{h-YngU=}M1PL$1B287w%pEluo^h2qyO3Q~sC>`qinTvjrd<6c0hN3&-y zI5jyyH_*^DJ7WB3qm$y{4MK85Y@asQ#hYax3fY%?=u6HSi_gNc%cLG>B~3PIUl1~T zU2&hUE$_{;Rafi^QDL1HlKygS1hLxluO%&|^yfbE-(E@`ZXXm_WmF;j(yGc#98H0Y z2oYGXBor0$jBm}~Wn+BxwT={_T#W8L@Gc5_#1e}uT0{6JRY^;B_%Fzw=$t4_d}4T5 z9((Ry{boW!4N>YMLaFZxx4YzUhiys5#{B-+d3=?x{RWPg@t+r(3G_bqnc&(Wpd zc50DrDs*1|1ch&fN|%z&o~`<96Xvw;{)YY>XZE5Ci$vJra}D2RkGVK2bkpTV1A*=O zVjVuZ>Ao|}m{XVR@o^q-Ff>s7EZ^pdua7fKV#4}Vv_-o-&Y%AERJ;eS z$Ug9{vi{EW-8F`dmL>)bBEmx3Ze%*Iq=juG0)=E694rl;G_Hs9JFE-)R3Fk*AL4j) z>PhjExw~q#XSeGe6!lP&wBm2sP#6;y&DuP_UuD7ibrM;s`C?Mh?oU-MEXp;^`B=59 z-|Jp1))dVUsK|AY+Yl9*5O{xv9Y?DwDd`MeNoK;gR=n(^g{B! zq#sKUecWspD>789)FimY$6P%ByhYXehM2*Gv-{M^$8XG?ZyPSaG^wL|Y~sxI^(&zz?bm20L)M(&h8 zTa$3?@!xyt4y$&RzEdnbDiL;&*p@s;OyPCMzAd3&kWQL7r81V+hlvL<)|uR zm^01UVihOy*N=I|XDWUFD&Hla_gc-((&1uHsvI(Wp*Bn7uJ&w?~zpS*yN^#QovxWDVAjO!t^5$vDya%+fr_ zi}mg7afa$w;n$xl{CpOsTMu+t72zWLO>`r|o+s@~3T6D;#s;37EGg_1Bk{g5=vuB- z-0I>W(XFA!+07MwG(+pWVnSb@2 znp>Or$9O;0x#SaNQhsMcZDqr^DNl)U{mm8{S@f$SMrz42mbP$S3O6^lE;NqvvdNZx zV`Lnuvn|rl^;VpXk`l7MS3||*H^z>Mck&!2Mq8a3C~{Qi@^yh*GF$kQL)1b8LHnICEm-i$jGfxFp zvzda`9^_g8WXJMH>xvbm*aC{P`raLwvB>Mr3ZHQ^IO>*HIGTsA_51ed_!Zy9(KGWi z+nJ2KJ zZ+nG2Tq0wZJ(B%kCdX69%|b^sLM`!9$yDX1C39TWlr$7+TcsC-4)MsYM(~igQ#5aG zdVE}cM>$j9utu`w@j=bif}gJkyells?{Rg*r9i!jZ;Fr~7oi7lzY zWG2|PT6^g!#XVm$%kZtp7PZ7rhccE7c=+I|g^|9xgfyeEqm)y7=2KuQQ{SFQG4rL3 zqs^8#ahw*tcXj##>BD@>>chojH+_zt$x;7mVdOY>c7mX5Kf~z*rR;o$ls5eRB#qrV z^zng>Ms~IxLyxAEmNCcNU-Fl&SBh7yfvJeoM!N*YtdLXBKWDfmlq}olRI3u`EBc&u*qO;t6fBtn$-JmNgt|UL>J1r_+-0*iA zTf-}?B=4Pc!^wVWB`u%LAM~?#cV8gl_IyAAy-4ZoZ0W-x5BsjS)7E{~L4Ucv-^)qD z9c2RVYvzSyR!ur5C@maX%-laicQ-#Xs9x%@=qP1QZ^YgmxQDWwhVm%~t@D%!pD}KU* zrztVc>E&m~*jRUz&(~xw6ODJS`O1hQ#?dmi3pA^zC&fQ%wq)z-aKZyBhu%?CU}$EX zyHDdppv65{{3O($-cS4sLwBv6|5oAEO(!mf`Tp zFIFwF{)D4hwvmZ6O;^|~KCbNfy;YhcnRh42OEQw^m!4PLFXgyLn^a8rlr-q`O?J&K zZ!;q~MYZ@i6{#0yEq@oh@~IvF(q`N#C(ezDv^3|)x6zX zLysN((BJo{b?JJax{2?YDiip>xiG&)$S`hO2COXiaXz$I-)Yj`O4rE9e8{HqQpV&7 zwD6nE{F5Dt*I3^cmNt(S&eQkJbD%vweQKqOLVSf%TaGd(^+cfhsIPP9=sKWz>h$OC zQO^eCs?*1rS0o0%*&C)4b9;69nCE}BY|Pje%AaawYE)hl{?5VYi@}vwG}ohwM@A=` z1`F(Je|Oo@ZttvFn~b(f@9wwbKKp_)$E(eI$X=_aBWmMC`_G{Z9c=cWlpa-@QmOX- ztBFcn(*P;8K*rVWqA$g*cD*te@a;o$OiT_opRL2JFKex{2GwLIyV6W*=+cd9Uj{vzTc{%0piZA7ld+5wCQ=fWUNCQ6g;sN^Tk}VG z7qe{n<|zUm8=szMJyO4*k}~z6{bk1F%ZDE*#_?@pSJO(An0@?B#GPG*iEZ;&I(3-$ z=|kL{!)kMq8MX4Hy>zWq_Gh2^cNLF@RtVPJH&WyuBFu6WPyUuAueg7X?qyNU9M2Ks zH=IxjXFHf>@6bH#tgzX6DC@M}%|88i&r95sA0cCFPG99QbUIy$x$Ldz;msSa$$1^z z_Vu*p{4@0HU!I8=A!Qb*mBOJdX`LTls)aORE7 z-Q=vwS|oQq>0{D6f1^$Ji{rO_*jl$wa?*eu^A%J>?VM^Qn~&>zJ!G#uG(+@pS!r3s zewoaGd@;-Uv{mWT);`WZX4ZTnDPaPC;MED&D|0@TB(D^6)IJLbf4)dyM&|UmMY_f& zn-1~^zN^0Fv)wBp?)5wOD9`JQg|{ExRQ^(U@+e{5EqeKE;dK&&2XE4pY!2hQ{p>;? zFPFF-_Y@-&cPH%+FF6__E@9*JG}X)SMMt^FMD2dUPR+@C%Nz?0=Eh0Pwr+|KOG@*r zpW7g}jo;L%uT-AU*Ae3b=mx3%%DgTK*D1KqLtD{bO?)JuTo5XybykUymG)+pP@A{< zTZzbk#CXBom5lN0`R3gc-g7fIV<^Ur#*0KM8DO%;Kr7X zC_0JYd|b#m9+Th}Xl|hqzKegN%R*Y6j>X1{b6EC*t$FDU6}Jy^s`f+=q90Oanu{pJ z_mi$BYK&#v6 zx^t)8w1_GHn6AEUvDsmc(-LLDAx3`NQh%DoHR~6 zZL)B^qBmof6ytdupXkix^=2w8B6TBSyw^BNLFj9%rX=I0e4Z;!<;5nO?E1RRYi-o+ zeEPX5r>Vrr#V)XoVPy`{V5Y{nJ5JHPbSG_Oj-@J7z|rV@()B2l(9Aj`a}|QGv~}2% zW#VHNw$x2Cn*Hp>6#B8%LHm!%X=(B&TYqtsXWT+jc@ulchtRYl-QcN9FN~ra=(A4= zH=7#GQ#n;3l^!(Uq2t516814_uJwoYb4i4QsuIyXYqsqdEN=!o-gJWW2m zP?2rl{KO4`Po%B$6(2L{;HCYoH=*d>qu#rTVN*J5ll6;5;q2OP2RM0#L!NRG#ho>q{ypat)BMfuf*NB zYik{oyqg+*OH*=&&8YUX(Bq5{PI&cfxGjnzSX&bWh`?8Wa zK}*UuEo32WQTn(wbhhcJzFKET&<>17#T`k8_tA0n@=yI)%`Kl;rs8T+jG*&d?HLmn ztdk6WufbWf$NsW*ILq52IWakG4}aKELX-Y5=s=%jXM3yM`q|Q#v&P2sV`SUY*hOy5 z8-wly2%-{a)2B7-lATk7;uTzk4PVj@7-&x5KWHNz_}r~|_$L(Oovc%)MJUC16>V`! zG-aROd`9)7Wxqt(PaSEVrtA+IeHkYDo`%v3RAm_N?BXkgn|FjC`^rDhE*}*`R1#vB z&s*ax$-kL@&EL$5p$irlDEr81UFX<5uFY9>z1n3qw)tu}yvBtm3hYPG4WG zJcd^4bdR5{e$Q*$>@2`O;*c^kp48wHg}xReq>Lt57F-5Tm2bvX2S!465G zHxgBgrkZ9)GMprpLY$L@P->^HG^QbKgkQpT{sse1jP2$qTC7@8-PzcU0rg?it#fNm z3GWiMPfI{6d(H|^qc7e&=E)%XxC^cbr?<1hPaF%E5LO6zG|;tqt>@LR-gZwfc~<1H zs%Xm1X4zR%cD6INw<<9ja~*c6sVVbW+GJOott;Qi**hort&?sz`%?au%;GGA%PN3v zHO&3H6eFA-WTn`7h59j!Z*MzKn_%ZN;f1F%V`F?ppPY=WvM(84_s`d@iaf93<0=_k zpd$O=jf9$Vi09aMErf|GeZ|=V-OCvpI6Cga*KpcU&;zt73(D#rQ#b0%hEt{pL%g4_+HE->O zdjF2bouWncuJo$eP;QaZs_n}d>+5}XA+3Ghz^q5|kx?Nh4~=cES`ea^xZ%m(MvG6I zGu4Zx<>dD@j1qA&H8gU4A5b3G&rUmmvFR+`jD8}%BE#pi&DbFO`)f*Gx$7IIK1YR) z$!=>Zho*O^v{`6a5fd-CnslV5(iOZVMb*Rh66Td}bvh<%i*8w7TKZ+|;W}Hvh=G9? z(q~ug5-S|aWD48c`5a#@q$ah0(I#;NqGN~8;)1Y8j4G=qUNPF2l`4c@ddwSgaYRXj z>9LlAG`|yxMy;ks8S)MZj0X*BOatKsDssaUJy+Ijv)8?^>t`?fVwm@k!uSenb!Lka zL+*>Nugl_c*;;|ct|M4VV(f{nE^$7iPnWkA?-^*3Wr zU3%J#ZP{Z{Ft5)BQ-d5aU_cCPEIxD%s0RkV&!{y|prx|Mw^b{7=L6L+Lzfr7#y(_I@ zzVdVX`aIURhvpHXF|XwUw($FU%U>H3kadPFZCw{qGB~tcf6He^L&?tK6?zVYiRd#v z>9K7N*S4Z4GpE?p=#FB(0*dIjhGs_5f|BMu;aq4?NfzH(MxUIe{8j(RZQHkWHOHhi zZ|2M7P9?Gz%`d!GQcP=otI_anm&uZo8}z;n+BPsfsd8@03RCI8oS>)iPGS6}(l7ba zrn3{$Lnk$q8Qsscy^<_`a?(5{8I8*)6pdY;5@Ca`}Bsv`zUx$$wqBQgP0+ zREdyBG6PISRcW=M8RJgsnhM-vsznHBnogcCkP*$QePW-a)~xXIg7~-R4_Rui!^Dh^ z^4GT8f8H?r+}sD>Qj3>svO>nv!=i~b?B}x&e0afr_E28y^sM?qlyS-|I^!sp6}K;n zR^(&nBi&zLIO~GAS8S-|O2LN6!y{FS96U4IKP?OrD}7bd{;7Z+g`{3k(YDac z)@Qf<)=yosFWD*RXy9HuLNUbXmf^PzinoF@S@(m!C1^f*aU-s2JF2xlHUon9Q~p%(uy$^`%~ZWy>l5fGiCn# zy%UrYAM^F0kDDj$5&ewsR_rJ1A$_)n8eFn2caKt4cz4T!oo#lUHpSDzQRr^?VfzP# zr)G%7HJC^H$>%LTWSV{d3}KUdJ8%L+HQdlHA+COKVAEMMwsSHeaHJxXlTsY1r_Hp^ zrd^a}UlWSkxo3r(snMaCoWl=?tHoF7thdX5PggY3R_S9d=ESmlx}q`oFHPmIxr!ZG zJL|3u^SGpD-~7cn&qlbM76Q62a0}CAti;6)%Lta_F|M#P;l?I(?_lM~_~8Xs9!6 ziA*~>OEOb{(XPb&5HkOb>X2(5OYOCHPcfXYsQf#uKsAns=DSZVByJ%lx)+@Bq$*)p@EC+Y?jPND9iBOd-UY{q>$`xV#-} zQ9e4YM3oU$s%W(`UvK-}q7`-=;pUi;dUl*W(Uy5O70LRU>?w6^J>Zm=2A3f6{SW(13Nak2xOWVbBlOu*t5UbrN$Cy@4@eoqxe*yq<)2vrc<>%$Hu<%qcSz3nPhG0OF2^wTq^WrFl?zh_O8 zW=!!p5@;?RIOv=2E-A)s;fPR^B?1B3(jBd;t_uXci-o70(5aFn=Cukr^G_k3u!Efh z7MU2V-tEVCP3>B6m98Ldz8X_2=49VP!wUFVFTyL=+OWUgzOyZKqM_PrSy7R&P@)g} z{@~^ct-}+NmfRZak^5AcwpD;FI;c$K_R2{YH#Urzqd!ei{OVw{*7so}q#1LObJr5q z%ziPG6QC1hKZWUFm;Y9FkgSqgL|9#bwf~puu$LjP0xJ8qRu*{~E+CG4nYXqwvu@NS zZ|0(wL5ICg%zS+43gTw;rB6B!7jF^jd#>!uJhP?G58d;ZFfiJXApHD1`+gW|zPk|vp(}>oJ z+3bD8d@=*_78I;Ff8&Ey!%@wN-?G&9J0{hoF3>w5#~xf^=2JC3u!Ek=@4Zm*!Tgb9 zFTUF8_w27j73e9FCGQ=p7K~|Zt*E$+RvwITZJH+)88lFle)!l7o9u#_oXQMN*^y+& zoXu}cr%l{;R`86(q^jj~SE6GjvuMZYMo(FLG5P&T5B%JkCkh(wxs~+jg4N#w+B*W4 zY)S1%5l@aUoc6M)QpiVm(cv~LgHhMSe6@^TDfs#iFt1&^JBn2%_^e}YUlkR{g-Y{X zj$8SBxW}1)z4Azm_Fv2U>>TE}`(nSp2m4B)4hdK5=!2X>I_Ix1<18FoTT|^?{oCx? z#L9!$4w^gaq{Vf4H_qblyZ(%KYU6yQ8b1%;aQVS=74sAM`$QWf{a8ua;X7sK@qGyx zJeRo0H{m$DXz7)X!Oao+oLnWL_d9Q1BQ}0OYb0q!N7J{34mx;p?o+nO(5w{I5i2dk zN7;O7d~7UaOjP78etcx|*3SyBckgu&ne%1?s!O(a)R^;^$fJqTZGl8b)m7T#O2QMS67@SpU=M6 z+_-p^@PLzE3v%3OW-4*9CZ$^pmx!2ViwaMo578GIc_2*A%&~adRDp@+PLTyk{6v1$ zew~r-&5?D-{9dJeymstnzkar+L}==ZVGq;m1LXV73ly3=s$ZVJ?nHCd=O-iD{+?LP zSE9mPb~a>WZc^=L#{*Lvh=u2e8YgS*N?G18mPRP2F5lxBD?GvY=!JXn_b(@twoFl(w+8-l8~{bgx(gZ$kRqe38jOnFf8zu;sk2Kk+oM64+dfw5& z1@-)#r<2ZKI2V~QZ~mx+Rn3>_dC{SdE+>6@A+^O%zoCAMs=^*uNnPFcXgyD+Zm3YT zLbl3Zdikno?QM|3%F>0QBP19nlu^B9+vV1sAtSTzp`bJJ@T8qJ>-JV(BmCCw)<2#k z?8Gn-f7wpoz9YNHV&E6cKL=bO^!a?(|yCkqpHIKEZ06x=;})i(7F3kUj-EON4w z_%wkrTA)$3@kSh`)|%sJwlKg)?5M!E(OS~Y5)ZvI4PKOmYYfxB>1dL+ z{JQ+bi5iva3xozm?{rK}X;W~wT&lV{+V|{?58ncJ5T|1d7v?!%GR+p)93NN6_fVh6 zdb}ykab@2;`<$gR4nC7?-^lQ#m^k)jg%;M zIE^+n`fNaGFqcVf)y%#sSUGoO?%F}D4Z6&L;{1afm*<(U6SuZ&(4$q|QhI-7u-?AP z(awG$vMr4UlUhFpYK0~hv}%gP4l2mLx<_a8m94ECE`N&s%c)<$l?z*ph4wY0_-)gp z&z$PdKJ-&htfQsEcJB%LXzMv*WAIR7eX>&C`?Di;JT$f{n8w7^SJ%&%5|Z3l`bAMi zyt(1F1TF6=abS-6%&8|*RLM-1R@h98-&z}`7PL{o3)mOHTGpfTuDg?FDjJdg?y{W5=t$XZV}g&=rkrN5mS69; zNt2N4;g`Mq*HV<2fK?;yQk zRBOo<0j<3!?S1^EPk)h}pC!d&3p}$pe!=Z^q30XpcDZ%T6ge`^%I`)Y z`#o*{qsK2?(+o9r-7h@b$zLcVna1E8crxx$xIxms%8VUqXY)T_YC4e?ba2y{=8)~< z&ItRqKVC`mci8e_lkcsxVrB?J30sdftvkKvEJ@mSv32t~#Zz0#ob?7A8D`#hUCE`q z3l7Kn@JmQu*duh)&vjGSZuf8tLQ*og;qkZ?FPfGAH=7@f8`v=<^Uk4j7OQr_&R~S4 znT_drW2aDCBQ}x8f=^%rC$t|H4Q>2{UN&fRu$gGXRRy%6;8OHs9&6BsTT9S81$6d8 zpZTJVJU!4q_zvw*hOV56zMF|Yg>T^2Pnrf9Rd}tw0;5332zs2<>c}qR0!i;)(i$?X(DjMTBtg zF(h13+oD7rAKE(vH%8D18uz=i=sP2{O&1Nd$U;Af!aJhwBgVJl`#z!jl+k^3?tLcQ z(E?Y{8XENK&dY(FyzJV(Ga9KIcf<;4GtpsaLqadaukKtX`TxJ>tqr2g6Vc;>=;}_@ zpW!d)g#uS|y0?!xjUb3CqprL*LR|7fG?5xd5MA|_8%+Tx^(WS!CGahUU< z$5=x_i}0tv!Rbz2{1ZBi`3!VWlE-vlF8_NycJ8HZ%;BH&*8^!ayh#4S25AIQ&fReF zXSlEm*?}k0o;6&{|1~#BTRey`!2sD*J+f04+M1SXg_k#AW#kKEf)Lu4PwPL#1U=N( zT+|nZ36#xt9p_A>Yd+{7jA}Ld_vb8@9QSrai}m+$Q~kMdvFukiLh?WW@j#*z7kO=$ z$)i`myRNpYf%cEU3COj)U_=A{1Ar|(;|IF{{Ah~0|XVy8AAK4kkiC4(y!9D+(&m-IZ zb9RH6{Rer;Yb)Ky-_=%rKaRN5$(_dYvb)d3p5f zk>DjfI{8mVcDO~1Rbqq`G4{KkvPDwzMHWTrm;}1m9AWq=6suPC606put!>4T{nc>& zqZYz@v?Ve3MYgX*7=CYmCTiUoA(JoHe?c7Q#`R%KIuUDqXU{mkzw?) z8(Zj`hQ4EQd+y%14$s((P*2NIzYvQg*$7yTye=83YBnyZ*Agk#r z6I&#cBY#9DGl*)WBU;EhT~J@}w;J^bQd))nF8}VGDVf-!tDx_HFB8x==*5$$LK4Bz zO!skW6X*YjGNEFJPBrfI>Azzqe@-TlyMdKJ?gq$!EkyynTf@zDy?f~HpOXo;6Uei$ zow)VV*N^b*8-H`FSLmsley{WnKFT|wJ)Gka}s68iB3lD_VE=ZmQH`EF5s zbQ%}4%@K!Ph;@VrYHu@lv=r|ADSlS}7C*_jV+r!yo?O0y9C*hMEgIcPT2v|y8d zc=2`CbC#kuHlsGgxNR7ryFJjAYtda)G>7vVkP@ylLc8F-AbL?i*FxUqir#@AU5$PU z7_Rie-5^vXX;nZ!d>};h;r25NT@QNDoi;2rvQ}t?eh&E}(gAUFPe^BbBz_IxdTtVL zz@I`rU@hu-6`}*T6W3XDv_e0d3hhYr+Rh~OLkMI$xaJ9Q`zpjaQirH^CUV=6{N4_Q zSARy{(~*qUa8Vd44InQvX6LPTfPYdzT!yMHNMI$plB~l}{N~jS7#H9gNDd?mRbF2d zdw{QlsvoI2n8`oV4L@`b@E0V9v#APnH^gY9Ml9!Y4%^I9q#tBO(HKz(QI0b@;}=CI zuU_!d`D?wE9r6q{N zfIz>G1zl++dHcV{fS&RONkcw>DWY`k#~8r;BQc;Q`tgzJh=X7+ym=z7*kMr2r+#v4Xr-eeV_Y2ai>on@4}`wa<~*WOm4tQ2E5#(Ni}k>KH|arZ9cg(!ZC zL_Y7I%9x^KCYR4(`9G(ltJF#UFGQ3A9sgA3HsATtl@6Gr!xCSLwC&e=un>6xpyU4= zdO*dgFdGK;{{P!`M@J~K3DT}X*Lt^X-hA}`+P2M+RlwXQpfyAm)GRW{lk}$$r$-?@ z=I6$~fCo^Ug!mcmT#7OXh^WBAytvVRS5$>4&jslSMmvf_3f%fK^iEa^sQwhte7z9C z=ysVk+tv`$1nKqg8E^+IbFqCxsQ4U3UH zAXdY4{%M~?>I5|tL~6$puC!qcB!?)yk|M4m`Y1idwGyn!FlMY-SbDteKs*R|4G|>i z(ZFBeI1w@u&~DPlP!@qRoSt-GUB&ushIC~aSFTW9$Gpe2!=Ac@qtc%Afxb}o8j9%h zM&rb`0Cg+Sm#*0snb~x=2hd*dM-Z#y7pY%bM9g>DQu?)&YcnqB4&eWf=!U#$9r_-u zqAOnkN&FE16u9+K@P%+4l@XFQa_J{F^7w}~vXqO#!#hVp+7#HzzW?f2mLq<6b4Nvu zgtDPaD5Cw54ZTIQ_4`g6s=~&bzvLhoxN40MR6`=LFQiK84KMK!jAkUHt>h?}P7xT%PI=3!S_N z{KF1JE9oDod&qh@)?n~1ytO0D6$Mmlz?A~5KTx>&;2qhgfFocl$C${07W|IS7zH_;x&l0}@(>K6OU2={XY%S3|@AxU~$) zlky$#NpC*>5Fx?!@$Ec$97)qPL=*EnO`fP#{G#TD6^T|ty>pBt_F_8ml~}fY&@~&- z&y&L(awwWZ2J9gHBaTX7bna-JQ1yfvN?eugeT3MH!F(%|YhQ3ZZ(Kmy1Lm6#x(4_G zZR5Vc9&xNfuE7NF>VjJ9?sHZnS|Ht&sUXS*PBG(9a9`1@xnAo%1X(YCPs> zbTzex^Jnsecm%&ld%8u8=gOp`7n!_4?e>((pTb^JUttax=C(lxDOstZ6|6n=nDkjk&5!gOq({X9)#!h)LJKSkG#q;A%FFzp z=y&~ptlzwI*)SgR^=mzciY^(aL7oEI3DFOne?4xjn=o?d70=(peGtFkd2Z5M;1~tR zgurK*i>EB00_hIm1KbII?LE4i*BTlS?}4X3UJpy^{!XO(K0ncYga1GW$W<1k{^$Qh z|LgxJ`fr5zf^!7WH^{1}newhO?~23iGH>d)n})0SPxRbGZUnD9Krq3aohsdjc{~7HMchlNe0jd@-v>f z{mvS~@lO10=*5@i`~ZI`Iswguh>u)4XL9L;3hi9<{DmW;${N-8W}*n4^z1NW3YZGs znv7LQe;kgk`hA)QbNz2)uQV(4;(vK_04+p;5b=pivn`?pbeW{t4*iI{CGw&q#SoE! z2XIFBV2Vi_Ki-*Fe(9A4tzK!M#;Wzpu~IyqL`+5We&*6U3(-RQ7g8I*S3$l9c7|yr zYbJk2SB+5n*b{+{QFL|H(P|{wYIGgY$E%^>v6iA0JSJE{(O;~iX-KzZ(713M0)87} z8J@LAo%;FkC;D0U8~VxmuAgRz!ZnCf>pN-esihDD{5ZBD2SC!V@SF7iSXb*%e?4PG zULI=^@!xf|cls%f1?q9+1(6nSMGDqbkugJ8ZGj33`69=q^_%0uwa4D{zYf(^em<^A z=*f!SbQ)_9&D`f;s(1W3Z0!sZK7vKr#E47mC zclZm>>;3y@ZEU&7#~XK(u`TEVwXO=TgK8@Lh4tWA56C)=8X-YUMs*q~gt?o^;dhQ7 zVg;%lBZM3<4+gW%JnaxPF0LO#*BT-0?#sn)yq>kEAL?yxQvoeh6(VG~_u zki++Oc;o>5#B!L0sPr#*Zh64s?dYNe|UL_z6QeIOx3yI>WJj5`mswAPoDWLtCF9l+hWE zKtqC1MQ1oHj0i#voz>BC*_0r((HRbF3xXJn&T!zj z5I%V8c<1-dFPx><{R0Swf&1 zCBcTH0>S%NbZ$jQ+%f{qXA*2U5|<-k@Dpr0I*?(b^%Mjfj%GKsw+%WAqhs9)0zG>| zu;FMx8^cJWb3b(WAW$BF&T{B@y^4;dq~^oW8IITJy%svdu?amuGYXwYqhkzu zm|=k+?fb4g6OoPqvVazPaghW4ZhlN!#;MpLCqT1=&3dGEyHTbzw$ax`V#%K#|!P*E9ZWNDu;gvnS zr{oWs4O#w=>4fik_D1p+ENF4B-;>(%I-EZq*O4nbiMDtYr$Rn`Z0ywQG%Cqym$<& z54uNEzvdK1OIXeNGoAmHwm>U@Ezok;1wIJb(N1b_jG8 z4dwDKNM8XRaeRhne=&B!x?{5L0Qn2x6GV1^X>j0qAiU0s@?pJ6Rf}uAD2hPV{Ac>4 zaJ8onk-ho+p}xB6{g3;LL;d}EtK`M{u9W;(zP(u>L0BOA%#fb}Edk#I`VY|o=^MKu z4y>r+jhz&~%Wz${cJLbFRmfZYs3j$Ce5FL-HgLuP*Uv=#Q`w;jq6o(p%TdjGI6_Sv zmx0#9EAW5NuMX-NV!`gcL*HGgRX{sDz?DjfO42sK`mp|E{CDG830&13=P)PCP{jd! zJyZ!`Pe?o?iCcra@W`$I!@hsqAA#0NfR5rAdKu~$qD~x(K<${cXxsv(18@>zSXdVf zF%+c-pig95xUzxQ;6j|d3|$ZLGmISK9PnyCYRMF}gI8UHPQt2Im^lG23OYiqV5O?v zwx~^@m!ic5wZW^~u)-VGQ&5p#SMA1kntZ=%X*cQirhhD+@hTxscHcXoH{=8DfT(ae zq84;xEusl{LatT@q^6J*BQC_+P%FUNPofz?Y(f2-qx*3sn!1Z>3wVrW`%mcu8NoPs zcQn9sP`uVgbXcQNI3uZH1cCJ}csHo^y;$z{AJ7dN<&APTSjPpPn52CsikM8%|H+lh z;1vNU$QX#19!v+w0$PH0Y!F?bKdXD8gh00YO_J;snjd8jHF4pHO6p=!D8S z*^)nM37!mh>6IS#59q;FSjcPvUBD)=O~TVzpg$z9{+XPyEHUrQ5WRT!Dstr)tjva5 z4%WgMsDG@zy>WEXcU^!;!Ue?>zke^p0=za$`t;tt2x+nS`83FSy5jNAy~p8heL(~R zXiMP(;05fj0FeyF9V+`Exld|jS1h4C0j6;VwJuF2v2QrW84Od(dZJ;&0KBH&8 zM)jhFr6Je{m5XA3`YWx&RL$y)7`l%F zDt|aSq}m0JLaYiaZE$RbcSHiK#u9;9ez32f<1<9r-6L!QjiEQ*;&t@o&W#Q8;E$|qkPdo$uv%b1si`S+DmjFLWAf>r^>FGg1beRwqx_$&OZ7*tkC%g40@UONFwc(IbwTWk+KaSo^@ zt%Rhk>v%|>l6c9>Q%Z(FDees{hO|bQnIP*YWUjTICuaeF3_CJ}5QnotO6@->&gg?|iiFMK>uvg4N*a>F@IJ z=eaQ%QT|?QhCOHl{Q!i6y%%t9k2MqMB(o8h?-I?YVgHBgA)pVq->&#V*4DbNvH`0h zag&TTanwZijAulkc7rhoSNX6tkQfg31G`%faPKx;{x9Xu(c!r? zJkPR<`w6rI{Gjj!;yH5H0+?F|Joz=1zzYgr#vpl;t733Y2iOGrf; z;+E%dWrO4PAGbxt1!PTxALsbE-JVqioWb{ogan$a z>753u1_fimqrkn8UWGt=Sc4A${m1rfiX;WL3$-OcUGPs-#Ro?e&=Oe_1-}Hg(KV;) zioH-l#x)VD{?t|T0H-LLz%TV|6C~NQRtY{8H1l^_2JG+IF2qV*_s@p5KvRKhKcjOD zYHu1BBVaZk=SSTm3UcJ+=-?hpbS-bh^Q(J+?(Y3y6o;7;SQh|1z!jU`;xbsNKAeZY zn!V8QqgbvtdSMM1@CC>I-A9PlfArEm_-At83u^rW$rW@D!a81mj;#%B6{0yDePIsw zR)t-FI6CXQ@4(s&kr2j9z-y>9yP&IZqyuOTPxXT@z*)#V?lpMc0$)pF6WoiVqwbYq zumLOw5*u+O3_aoM0ToL^b)9Sp`yS{UuEQ!(2Sq@)2ap;FdW9$k^DD|+BNZBc8h(3n7OKri@m(#rvdu*_Y!5{FS? zlqaKMye5XkEU3lxqyc(`Hu0U*UjF|)YBxnKc8y6%{&pQ9FLh9p{9n)pzET_YH3QK| zM!b{X}PP7N#`ulPHk0nN0;jiSBfG~l#KPtw@$P5|*x(_*yB9fc}LP>J}4p>!! z5xjSvlez=ahKLq)7OauXZOIxe%prmQ1DwDS4WIzFQqV)7ft=5THXyzM8-tn=tmj(K z)pZID$oXqBwjj0cM^xINIgVM}9>K>_x(T%WxE2CE!{`BXaNdCFfD9Or6|ewM4e0GT zX1EjU1jYx_+Pd=sbVv>9gb|m2c>V=E5w3TwMiGbh_c%cXT?erRv<+wusE4Z+)Kv=H zw!mf~>cF#1ysbda2Qw{@5drN$dGC9(kauH?hrWBKk?MO8x@IPqAEZWg{SIsy@=>6Q zmnOImXo8i(5I0hHVcx)fpcjDkym-ia9kgMK_(470OnnPH>HZWw^)2qhjGJ=-c9C@o z9KrHlCym;{dsI=Mg6zyY`3(93nnKBsx(@i)yPu)1!LsXKo1?xrL~R&xb&j`or-1Ya zIK*2;oXMpE@OL;5H-SfB-7r@Kcu%!VpmAf}f*C`kyo`GdTgg{Y;*7>5gEzsvM!^ZnLwrju- zoV$9VcX-8RSKfnMfKR=B2W=Rm{vexx9p``42IwZ(&{`e}0ulDlLNxV6dJ?$-$9SvK z7{AH&q99>hNu{g`M_N6zaNc|q?}Z7mG+;;f=pW~<&<@Z=J`IHZ0q7?5ioe5h0eRz@ zbfA^jZ*;A1D4_4mklZ1vAu$(w1WZL&t)Xfg&=+_*{G=+}2^7FQC~gUQ1jf5Vvr%s2pFV5Xe(;?O^D zJqc(6y(2v?wB1wIJ@El&3Y4aUm0@oT6#u83BO~{-|5|2XTjRJ@A9yYi_(OVJ%tu&r zfVqJ+6wnQ4MP#*uydF!dtLMRd1?vY)BiFE!vc`TEXGFk7DieiW4|Nb0k~Cr2kroA& zHL$b)J9&{~?&)iLYB?z@A^M-dL0+FV0_iG_0C|1ZkDqe~p9UD}&b2I3ex#=VC<4Q) zM4*m><2n3)jFC{uf%qD<2=KAD{FUmpcWnT@!M6W1eZbxoxC=D|@R;4(HTz9V{2Ar~ zEo4t!^XOlX4)mKBk-_)loDndB)B^N83&LcK#*~+ax)2pDVZ2gB)mNzX>7c))tiTSa zObNanjlRX+LKRst*$d=%R_Jq(80jGZMJW$TwhR;l_i;6|CvIUXu=PUJg)=PNCfOT7 zB%}Ujpq9E<^T|6%)a^s$3v|X#GJzSkY1$g{2@iJ zFQV6$%MafBf!{yoG{^z;9Z(WWh>B2vMyNx>$}${Dg7mR`$X<7IW&HozJDV82uJgWM zN=oZEN^77g>Y`apS+SB+(W1T#*Oe5FXGY>ALux!TqU1z{Oi?3=GCvfOk|kFK)YL#- zR0Xu~!V535@FEK@yzoK`7tq2B7Z3_9yzs&cqA*Y~Fp@M)fuwQMKEL~)|2*g1ulJqd zP*Offyz|by@4e?dpa19MoO^Fqf6Q5PZ3;YBJswnuM|7{T7h`T{BIX$*!EEYGp|)4P zp(w4SX})vAXd2RIdLC8>h027H*=%aD>CHH=w*P~WpR2Cgsz2!Y-j4V%#@}#lEQ`0D zbZyd~m-cfkB>ha-DRJJokEOCQ;M_CMtL?IDb;nzzr2j|#ua2(A#LBvMO-I%JSXrfK z6R8o)T9>D-n^1aui25YV4+=F(t@EG zbkm*Ig!~+DgcN;37rI)xYVRnhE)e?(oZo_KNKlg2&Qo7INFNBz_zu0GkTR#tDOvkK zvMI|kT~1*PHX?7Nx{yZcEggxqKyX~Ye`YtqE7p)BaJ7fyz;G-wRb8no1|`}i zSMZ)lOZ0A%L8~Jd+XBBr7_O|b7#j%MPhWARv^D{1ZbLX2W=Z6hm4b! zwLJo@_w5b|*lzmw%e7QBMq)i#Va5e0OZyc9>kML7()j8q&T=%itW|LQPY5InM1Ir; zSZZ0jLu|2NkQw=1u=RcXQUS9L+u!c7M>v(7u2C2+I>cVfM%M-0^e9zdv zzx6QyYttP9ke+7F>QzSFJd7UG-D7`3bjn1%Q&EsIzD zok*CqaPlea$2vqQS|zdxYli8gs!{|>uOym*k{lSvCWX3);q_~v{ICfP0398gFMD@ zH}@z>2FjD_oh9-FcfGvFdn4@d(YQDEa@-^PvB+J?qFrHuWY{RSsJ(W{7pRf5a!$QB z%lwO9EE7+@(spG^zL$2n_8vAoPrHcd)QO)i3MfXTeTcWT68SjaGk?9FrOr}GL;B_G z;g=gNzlkuC*pSEd^X;xL_&Ovij9IAgY&*C6>*7HBko+vs>`F_&Q&wX6XBUY*0PIfb(G0TEE~-ktJldokCr-iS8%f-Iga!_=l8l_eR8eHE5+ z3&ncLAoE1ci}viLM2;s~Zm$H_*z)EMolyN))O_w9*!EH~-amQoZm#$Cj%|2!br)Fa zg%?Rov0br4nZ;}wo(#%Bb}&>|L3i!inNz7T$8$0TG9ShA^Z<7{R4lQT{R+;LuK7Lo-jM0^$k6}e8x|iV z)q~Bz6{p#iN5sO{WSv8?x3uT^vhb9$>~t*m9*SSHqmuG+$6|1n%FG=`GADIboViZv zuWtp8+9h@k;bgu0(K)pDvVdZugFJuS^Y7ld-V6H$E8!}0SV<4RNr{qoCMr1U3oY?J z>K!6Aeg_Ry7v=R&_}NMvH?)3-EQQ4Q9A^`EjcDUs5YvQrre{oxCt^41T5!1#*UFGsVtb`qg>8@XK!*bjJQC*;$Cg^&eFxL= zsNfF33pddJ;gHpM?gqfFa6A?SEqiZ{0*;oKW9|2B*no~wWT?A@mX@;OVUVce5Zs~o-EOB zO^_5KYxhDJPo1e!JHmEd+vKRz1fMm|v7duoV(GQgrEGpVP<5@*u6uUnapYRdJ4jBR zHygopuct>_E!$=nV;nta4q0-Sqq|%DneK(YeFL$u?oREk8!4o7te%brwhZG4y34&_ z=o<^LHqfgghHZ$atI&C-FG*Cs9t7Rm@!K1n>y-PJW?=+Z5YlBv64?tF>v43uiXWEV zgbHB2ulyQ%?o4;B#&*M5+3kfVcXN?I?o%8`j=u_45Z9O#G4^ko9ptlohONMe-~CqW zyt*s4*F`uM`GLk{H-B9%=Bb_9{mMvXaVjK0kJEXd;>d&Tn)2Ays_wS9*OKe#!kg(G z;pzt{8^UX|KdA`28CogV5)?%fkSw={daU`c%G`M0f_pSXfY-_0gR^6AmwlG&38IsM=+(?KzX*v^-RO`Hmiu`i3 z7#Tq+(Gxr4k!$Hbl&%h&=A|2!6YVXQ_A5saN;xQ%2RJ1?5ZPz@2ztmhpZ&%+YOIqn zuQ5_r&XS z>Yg$umlRYj$%k1H%KGzSQ)>y>lSYMwCcLWWM@Nn41`CdOjUT zaNUEi!Bm$#dxBsqb^}dtm}?vGhyIOv(m3`?y+(fs-W(5V9br-PXo>NMt31XlJ1n(K z%(dgY`tFkZ6QLDJ>5+J+^OT6-xW=`0vQPUP83ot%$mZRtI?~#nIN^6+ll|%lE|xY0 zZBunVciccu8|`sm`jz6YgDbHQAnEqqBNXesCdN655leCYrQmnD)eCDCzQe(#iTd;I z-E|fAC_f%rb)H}ut2*bJJABfgcD)#DaXx-&M^4@AnSEt%_ljP}d->|v-L1=OzUS9p z?nA^wtiQ4KH=;2-CL(JEW%`Qtt}MuBjD$_|rI4$1jKAQ$tbYw@?szLh?by2%*_)y~@7AUA90>)A}>0P*w>wBzM6_3Zx+qJlw z_Nlf9<2A|5FSbU;W<=_b*TXIVA7BU`&71@o`zf#m?1R>&v3(ba_$P-Bd?m$o{(8Vhv-6PHdpQt8RbQyRIJ& zlB%Ym)*7NK?X#X;U1GFVudZ~D$|@jx@1P>lRdzqWH1FA6F-yOCV}$PRsAL!O(&qTv z*Jq!C955X?!-EzmgBZ2#0jMrA2eiy)s5D|6c@0xt{0<$z6UUyYBy|@tV(&bD-Cr!rs9li7oCeG zjnz>ea~35~K+d7vbmX(D$jZ*Su4C8SqH$>d9&+%Wz5cp-87rAD?<8d}#zGPwT!okq zSF;O|cZ}UlYB~I2s4ub=a#nwIJS%ecNGvRKKUG=vdcNvQxL`GO{2gN(lMY?7pj1ys@#!7GmfnXVM~mudRrXwcUDu%orRB^t^L+KxZ0|7 z_Su{@&G$RJQFBlcdDn+BnQ8iy&vB};jU2bTr^58F?5>f`L;HHrUS{ECgkLKS_fb>R_feXMb_CMKWu%<53sF4SulE>RIm!^@p{ zx8n{kX;OL15 z(3#XvKFcyHeKA#&<{PlRi|1b=L&?{eOBH^LD1ui*{4zHUIN>rn!gXkUUMV z>_#z{{X|GtXyyEh$Wcpss?REG--vTSX6#d|4XVND<8oK7QpRX3yNb1y^>=xwW2bt= zKHJr^I-+&GC%Q_@To2md7prt9f^J#U{A@(W;}XHo#_}-6EW4xZ8pvJ<6m;)ThYU*Z z%}#8}NnH$A>UhP4I43)`)`PkOC5q0+t4AL4wsCGvL9JiWvNsv{-kYorU`JS}`^E0y zP@DLA*C#@XvwE^)&kug@)_i{O!~Za!AN*ojmd?TM9eCHGS{NV4!?|y#pXP?aH z2mdgB_w$3l|HXWM@VDZ*zkYu3n@{EQgTMcDK0o;FFBQ)ZKAq1GUO%7D5B_fa?&k-; z`%*qX__LSt`N8MnS-gIJ@H_FlpC7#SNsLweTLUJ*C@X#G=!zJ?&bb>UuZ%-4k&HnIS94{0&;Mmyaq!og39L z+;d!e)xbg`H1{i1r}$L-CI1&r9%)D7CtU#E!H>9+DcE;D zDnv$nI=+P(@-F&x`K;okydFce)7R}%!{6jv=;=(%p}W-S4tdu3-YI?{#+jm*Cy5TP zKKV(z-K(h%LQmI1dSIbr@BXwyUdJf#FwG?v`thjgZT+5qTpP0`jW^CSH`L|Y$@_5* zQQFcaijc1#9X!TQ&8_W|^+@}cv!q!PxIQX4TlM^QiywP^uJN>Xu8+2^Zs$P?(Dd0r z4|eC49)A6Zfo^?o#QovL5MASTdN8O_Q$5?3YJBQy;Q5d&BZAUHNH^QJgt^*>K8qU7 zKII|goLpCMy(7WXK~sjs;r|Cgohurwdmdi(TuE{Z=4Y4n#THNX*X3^e_p%crlcVi5%k_Q+Fb_p z-ft~lv{1Jtp6R|;v*JKLS4^{iA-KW5_xfyT);mASG><)3`puvl|G=)`+xc(G1zWm3 zz5_3*JJf9)iUz77$}9M}FF&Zx3L`si=1W$?EI{UR6s*l7fmJ6W;*xc^ z9u)Gv99=B>9I7kSmrQhKy@&dVyN|HmYdt=t)0Iz) zxze-itE~*=YnNi)Sso_t%5K~SsR;`{7-Cna>$mU}op z;nQou$+h-(JrS6_j`|yH0w;9mOKcxq8=VzPYhM=6=#)M<-5>4AtgD{=Q#(wN?c!J~ z4|M-nXqxUD(Sy~)Gl5fm`LPa$PO%X60({*Xc6l*qR}T*x!=90VpD>*^g0IrCmep~F za35KTvi?)5P6my+TBq)xwW@ymI9*t}DoOU3GizzdrZ26Y*fX2P)y}9>Ps9GvgxrwP zD`7wAKD~^p!$?5|t2>%NIawH5-JeaA@?8lI>YY4`p?B;T489WUwm0he*sq5?hW)_# zS8e~bAKLG={zERX*=p2&zZ4w!{{4L$Mi=$m{Tx8P&#D5wFOV92Fe-9K2Xu}ck8HpC zhU0TTE5~Bj-M(5ETQVw&yNz~~<@kvg;UlMF#&e;Wve$19U|mhP*6$+Z$iwj#edjtq zt7wb_#bq_m-{P-71K-u7=zM4Ys^!S_xM$2fbRt7l3wSEofwO|~PKe~zG+@*`b3mg< zKD%=T3<#roMZBNg<(YZGG-Lq3wMFuYD%xx9w~~=GZ@ONXlP+>1Y>P2c>ZLC9ov67M zn4vpNc^+b1ZQoa8G}-I9pq||v@RCT<*Jx=oB`$JK&3PW_`ClEG-HcrWu0_5J#@ zHqF;!-qQ1#qs}14{T4i(mcld@j_~Xco5%L1N2{|8G^Kc}7zPSmX!8y9O6;yZ)tq3f zX&t6eIS@Esc2<>BUTXfJ^@x0GEs?Y9*(E|yDR~@hwVM~nQ=s_D&Kk6GAncHw)pzw0 zvW!_)=E{9~N@_|UnU|VaLpn6>FYxY2(zho?;cKmL5v|1qQYQxnQ=JLV>qF+I?`7~N zje2a55lbOcA>)*-Yt`O3CatT2pt6{XltWK0&$!an)ex6yH^$a!eR<48xuH?#u_^Wn zoeBwHSLn&QAbJzG`Kg>;I$DoUavy?vYe{{dYOY$G9T&TE%|m(9P$%cy{m6%x3%%Lf zK_=*EsE#?jj_0eYJrzBv{)l*zf#M53XXKVI2*0cwA3g4yLp40tP366isg36M0IK@U zR*$SNs*2#frhredd$i3er=qr3N_5sxEe0jni0^o?&|;pVA$lj%!f%CZ&&8}p&ToaD z>bR=i3fMX*{Ms%R=!5SVwaf&uODq)H`_YC^#}Q;GVx-=_6tqciwUjbW)-eT6)um<) zRJAJ-ITJManm77ip5dh!f4sgz3$FyXWz|>Hp>bSeqTTEV^)vL?FMJ^LxxT6AoAvy6 z_U_eszS`;y{Iy-9*}5hm77Xzm>+~_+UUUNncGsL{YpCo))pU&6fGe$e#UX(gjwfX9&KH^m&0244}JmG5vhc0Ua3 z2Vak_*~<(H5{GOG6EJW9+zu)#Uk}jwe&WH{qMI#i)7Q*k_NHahmJaUA*+Mv5nOjo95J z1lHJ&ShYO42VVEd^|K@@TgEa{MoshVA$*EY>lxG??P{g&V#+Qu;^N6f#o?$=G{Sqa zqH-?&?qa;(|AGReyD##zs{-<-4=bjVDqO_93#M$!P^u$M>AM37FfB5Z|k zyn94+mEHzbN8$*_JL)N9POt4$oX5OnZl(A^Er;|tQZiI06Hg~*F)P1gVb@y!A@cC4 z>`oar^!@3MbuyA^ApabTJB-1Y^#18m!=WRr?3AnMr*fmTJ>^I01lkXY>GXv!#ue$z zu-QtSYm~gx(%l!`lWK}ib|77AX>?}dt|Qzg4lW13Wj~v1l#x8$Y36klr~6LSy|z8@P2nRouDI`j9lsr_=+G@p&JPlj}5J;)3j&5k~m)`b>B<#CqD12fL+TFW!aV1LRz)$_hem4Mw|Q-+@&RAS(zH+1-H5*Y>{-YO zS{xfLR{CeN(H~#ug#QoL;FW$p4yEX=YSlu}r7Q%Wvh(i~_e%rSm*H=^4@_yj#l$@zfq`_&v z|7p;(5D}l67i`*&=N26eBm_UOBWL}ta`1-CG0cT}qM#kQg*sDcn(x&?VO${!*%k

5Nw1ALgze0e-rW3*6$v};jKoWAPJK_isY^=xnshv65 zVSH`Zf8c=JgIF#!qx(ptrb|WTB>KfYe-pQLmb{T!)t^#^tV}&MC246j(&&3f*x+KT zNv+*y3910*2S zsLw*@{jASBmN~@cf*7s$6do?v^VqdF-#mu`A78!DD_#9=8$n!dK=avXxrCCAD!) zPCHh4#W``*_MYQQ?nA2xdCYRl!_YD8^1=RnC$Taoy%rTqj)Cq@w0=ibc&_-S344+J zK>>Ae*b$NEJ{;d~#96hc38j%F7}mA?Ji7_qmh}o5k>`NQzwYW&SE_IwJtL8=Z3oVtiZhfClw+Z{t@xhG1e+3) z)^f!rkHFu(kN4AB!DV$Rf*CI--l7wRR?cvp&jR$8ev>*|8ATNDZc)Z4tDthM|Zt z6#*IX@xYjjh4~5N($h88qb@kGWY8cE}>`G^U_>qU%S>sDi$wmN?y1CT!1 zu_1lxo+dA~eG$b@bbl&n!z)2|Wd_Jov~_!siUY&4P$a1>!qO?%rSKN`(2i$aGl%p_ zbQ;R&6}ibV55hl7&X6EkkPsfLrkb1`0n@KO6IZ<-4ZDU5+w4+E(KY^j+Nf=S3K}hC z=UGzzeQhNbWaP@1 zh@R}Ju$}e4MZa6rZsxuCv+23-;oz&jj4Dj+suBeZB@rAF2g$- z7Rs~o%Jo}k+D{k}Zg{#Vzh8})^MaC=G9xY5-LsFOe&lkSYn*BOqlp3_FckIqB}1s! z+oUyF0LS-vBPu$+!&lv{aK0yc)^}o>xE$_xD@GtW&>J66W+0S?9;~m_swy+$qkB){ z&U4$&N}CbZmlbD;lhWq6l}j^6n)`zqqBQ>XnO)R*CoqTJizLi_qsrA-`JlXbDC?2t zO&`2N85w4I^wG8^)xONs?64K;?%sOhl*R=+GIBrI(tmH8vw|gm8uTyj>ePE?X60Z@Ghn%n8NScN+YhM{8 z4PAd`B_6Rve9-%;u7=*rD1sEzt}F#zD|zs`!6KwFt|?UH2Q~Y;Vs{$Hl2`cYsb%i; zgV57Tiv#Y!I(|EoR@JQ8=#gRRRKI;o??x|Mhp(*CHA#+y<9UjGYbYWqKg3sgDlId( z7l0?84%)O6i*rkQ%nf?XJ!~ZxLPEoekOqAG!CSH_z8 z>Tut&bUj8P3Y6E-GqWU$&%D^u&Rp3)Q_fXWD0>rq_NJF^Hm1OzS7M|gt@5yoOjb8O znl$EUl3D7IwMnXE5a6$~5;7<(5L?n`=Ytx~FJlPbrEMVLyw8O+sg?PzXK@~q(tThT z;y2~|c{bn5E6GzY6g!%P-LrkFm6@`4DZgS zYaJswY+mbW+Pw-+;iv9Z$v#V~)yfLTs5;WFaIT$3D)+UYmeyR_O08X1AizpGe9|qetIq|hT!~F)hAPXZlpb^ilv%0ZSe+$W!$+9ww{LaQtMego7&hdnd z<(j%OYiyUOWdu~;ZQMWXZLxOL%oiJ2N7kCn^YNRWMdjIKr}AUgTh9*U$z6_=U(sDX z2z-QJ#)>uiL)XU-_BfrhALC`l-*q-kgQ?Iid_5bCzhz=ibXxUyQ|r9 z`fR)Qz`W)9iZu$i^?DW(#;c(bzjTILN}{`(iyu0(;9cv`oE} zh)Pu}kR=O1Y;d$XlR`)ePIOog5q zadGocwON4^@-+1~x+l&XKdN!nyEbph4Cm+%3vJ>~-$6 zc8l^=RoTDL(fIy?K8@*`(*=$H&t*rvP+dj*Xo_v^`&HiJ@fTxOZ(j3dA#-@(eb zc5j?LPd@hZL-_FC<8tZIu6xe4)(np4ek5qX&dF6}W5{-#GU?Mjd9 z2WD~JrTF$!LEUWp>po8fJRnjG`-i^a8VYRYT6?4`8Lf@eQBVijnld6NR;f?JQE)_E zX{r5u7ym5_WnhWJ@<4jY#82=;vq3>hYyPMVvNINFgRI>7BJ=25R`&@_vah8$-_`UO z-DyWX->gw${ir89}+i3+?QaVcsJx*vL!HT_#TVcAycBYz_0EL( zx@qKqd$FUeQm~*p@2Gj|XC7V7>^m>qJe@Dv#+7$sAy^r5q3431`Tj;k6EMd83#y6k zN|F39*Ocywqdh}E=Nz7MZ12p;m@RitmtK8uJ4;?A&yE%H0pIaOJKfnmua4YS`@Nzh z-JI=M121`zU5h$6;$A6aji5SySW1bVY(!hz3$asGFmlv6b-Wysn`I<Qm}4$ukDRby7qu&cM;w5*=v}`IKJA-UojzZFZY5g zQ=~&yS6CiPvXq4h$#Rto&6KM+!`C*@+YrygnM+btW?JMS%;(;)bR=xg6&PyfmC$(N z^wl_zh=fO}x}D_QX?}WWe|f6pnE4`-7{Y6L*i>FBJrUcG)!9~Z_?K5k zu#LWDPIV_7J@LugOOmff7NtMyukilr#=x^RXrU#T#LToW_WJu602l?@M|G)#S3=> zyQ?%_p~eQSC+%k#y0YqTZG$@IY#T3y%%LW>Eg;J4bOPK+5sS-57f#`RvOP zx`@z3Sk;x}n)Q9Vl=1cN3#H?=qC9?YF7Mcyl5#CYsLVABqwZ|EC{~v{qnqX@VV_fd z7UC(|$(ZxB+0N)ztLiFi3664C6`b+@O(aL> z>G6m^kHnR5cgK!Z@&`Uw4K7P{PUFm&2#YmlrIG%mTv|F2BFT@jZ`V6q34>SUH}LG0 zQS6IKkgA`fxJHmGu0J0+)QPsMD9*E{kQ`O{8IM)}J`*#!&J#82D=;Es_jg*-S44*J zCkjOk^!NETt=c!owX{0AT<5zd=Hg84!_u5|URbRn>I<=_^c zoab4qTYsm>9ymiKJs6JjFs5n+W}8ejgCJwL|i$6)ayFUDdS z>(Yo#bB_<^K{A)3j**X#cgAed^!io<-H>f7_BqDuwUB6? z%awk%{~4QxYo1nOY}*wYB+pBP(Ml@UXyGCBsuVXLrx6v4=c!%(lw)(A2;f#Z($+g0`B{+3~nf({~~+Yi(c_zA4TO*V3#X;0M$w z&@9VFcKqGX3%vhrdFOll0MEJ@_ZD^SgU1h}1mhAjx=4YRr*VRNMIMQW{l$1~1x}ra zocyu)ztw(zIL2gL(9ze=4C$X^Tzk96Lqtj>^Xd<^UH`rq!rI|jYI!9(ID=}MA5GW~ z=j|1n*vEjsvmW?i;S$#mjQhkmE@fQ<<#&YiOI|(|pM4l8+cax8*Elt$>wbnQ>?UtI z_{>RO-p`?zc5HxbcAktX`_b;AmU$%iN!w_>Y9rRbR^z^Or=o|p7QZdW-|O+a?v=dK z`rk&YCnA}9PE#>n*aAw_@l-ymy_u3l?e#jgvj}zOz}CF`!`P~WR&B94(}rxb3sg!+h>?2_>YQmK^!R9Kth@ir z`xVG*@G#`7Ohowt$aye#h}J#)%a|`;dNk&}(y`R7kd9W`)JdV+LcG#$gKq`KC?_Qo z1>;%cXIDC1HLk4Gnu~ZNC-p8i-KA(Kg88mOi^W(}R(bYx2gCjLs~y^~C8#r9wE?-Oq*s^ZGJBLxl8dK`bRtJI54KP#G1$yH=FJ3H6QNM)D`V~+?Dl!WdD3i3y7hJ zDd|u1ki5c4_K%E~Xc?=ZCeXOu-Gt_0Wmhiq(!U(~DG~CK&&`F493Q0J#(qiJPxA+-8X=uPXJ&g`944(;Q-8b>QfI@8)Dh^Q3^N!=Bf^Az$m z@;2Q&(>-t^Ynes!HyM`hVwlzakXHJ;2wmZqd5^1l-4C{u^HvehhoxgV*vp&dXCdeD zT%vr|jdt-_Mnn|wpv>SvN2Dfa`kmF=l7H#tR{Iondy*?#|2Vqq)6*2h;v+ zf>$GZxhqc`jOF9hIpt~&QK^mxpnbUx-ldst==8d#@>n5EooCFyIvLN(JT$K3S$wl{ z^-St-k+8^o?mZ+qhYj5^Ny-7rJe+&xbVbEqT zg66tE2shw~BXYU&#yK zDtkz(q39K<>&ogFz5Kj(s$7eMTU{SCbf-xpleJF7D03uyQ@t_xsn_%*sf|`n&&KL$ z#)4W(O>^6XoN`@B_O&<5!X7iLg~mQ!xW3+tJh5^iJkcRnHA^`W8I(z6PAT2Eqb>=} zvy#^||01qI_}_<`hyOfYAG@h({vzIg>89r4cW-T)Uyk>u+OJOYzR@_m-OG6Y>5J;`O!V zruk>_&g=E#O>-pPc|E%t>u&M>Yw`NwT08?U-g({eg{HYb-g$j5e*dGJn>TnJ+i05q z8SlK_iQj+WmgbGW6R#g{HqFQ5o!2)WZ<<@<{g>nQ^;1pr_v4+{8((aiPsTg1_1}u; ze8&6XcrE;P)BN*z=XG^EWEAg5#N^A4{!&c|Fg zHSh3xFMhvvbMww0#_NR(P4g%5&g<*H(=>k`?|%`m*Dp2AU&cGHcQ1!BZfV}RDP9j; ziC^QL*ITbN&EJjpUyRpnUul}Z7w^38_-fpHG~WMFyngn2)BNLj|0nUf?TxT~-sAP| zziXP?b2@X zG*)MoeUrjLtX6B9vcXgB`R*^{Cr4WQ<|p{UJDB!(ykBhRC3olQ%7x~EzyK;E{0w9v z`}*zpl)m5d?Kkp+v+Xy!hW1~#qkTLo6UACq#K95!xOPHl&4}H$+^d5wH~xG!eo`yb zm7-c+i=V$0pF9?yt;BaJi?7D9Pz&WBk8kS#$pQ5PScycCCe-UnKICP|YP5%%F6~`f zs0Ll5ch43cv5$cqGx@0{xOa-D=tV>IjstN*1@J-W>O^Q}IrQ^Lyp}>&Wqh4iL+oQ> zO&DIrH6uOSrXCGFU?Z#1MM;g26KG|P+L3I-Hl?o0jnWvk+${Rxu1}wJgy&K1veT{0 zLt15Rc-Pm+r?h7W+N;J8@@h_fm8P3kR$CCVMM{> zLBpxAPWhbsK;%{Q_iI@X`{0eDS{TmWnR#$x%2VWJ*Efo6o2YUy6j5zx zO>{~hvpvrBJ9ap0B;N~B@i=4T>3B|*R&W?aF?kF>iFNJ_s5(@vr@o?kq<`!&wBvzz ztm;J_DxrxomB_r>XU?ly)N^mNOOIZW$N4a5flui9@Fes54f~apQHh$DcMu0#yxyFo zj3r%sNRMW;{!xe=il87xZv!CSW{x{1FO~jROs7)G*mgZ z#213rN8@uatzQB0S$R!bX?IvD(j1$E^%u|LXYMMez}K;B)iK`NOU0zh3*VQ2Q8`hm zk&Uj$JG|$Z&|dox-5vT|HJw;J9;r`te%X&sd%++2KA^oFR9d1+t5Nb&#gfBO)v2>5 z>!M$Z?8@4Ku@A>5Xho4pdQ-Isve5hSXZSuHd}>O%GROq2I;y-CyxS(=?_g=7)}r2< zMNR7BSkSW;^qr2Z4=G&;eX-I;EY&s0>Y~|G-)!lx_be&C+Me)Bp)H?NC$r9M9bn5t z@)1R%1fBSK;wNLYauu=k*l$kIqS%h*fXCR-SFeMv)sPEyVUD&Pe1ScEt81E}{c4tx zygf4sNeUdX_p`c#j~$HNmU3MQF1Q;fc885@N0m(MVx6w(ts_Z z1?32ttM=uVRs&DNev-#n%Irbz`XclB`Sf&uexH zM_#oRqp~^#RnjSFpIKC?JzI3?O`1)k)H5X#+;V8xeuyNlNc#E(AE5)<$RyRV$lS@= zsePBDEg#Xg8h$Z^pVBJ6b*a4rHfxdOLFA!)O5INEQ@oLumO^Gkpy}vA)zYn4z{0Mq zOAbwQRxW#$auZa8>BtI%L1I^fbFHx$WlQ-C&1L)PkCXJ2HiXYi`)P6pO?AgbsEA!5 zX*vOA9;{EO+U4a$iLbQ0J=++Y*n5?79k+!cUXg_(fbysD^QHgIm|c;hKGL;&mjfaO zb7-C4b(G>Aeg=nRo^Y)k3(LdmjSAGcUF_}VL3F{KVdEuTl*BK)7^csd)wyHMQAV4l zc`|6&Y$+VZ1=V-VtPDrh1m3Onf9=#uU$XANJL*2mhdVnt@2$Jx!`jg}u0xW-;p#5# zGj|~KIV-N`yBrci0_uhK{r{;s^oX6~BmHjOaHO^zw`S^Q|x0gNuc^k2VGs-(PKm(EA_xWHpkcZ>>on8rNF&B6t&mZQ6x?9l- zZ$bx9q*@iKpj@`nuP4yDP?mFQ-i>ZZ1{s4wcgF8J2b9)MBz?x!y{gN-A5Xhoi4eow zo9g6`{A2mhn9kTmO=VE#z>J--#I{mr_twj1=X_CIMlj@WJeJHBJf_%GzyYiZoenuKi|^AJ}pH# z<)hxw>28nLNXePzh2 za%A;xOWGxo?rxc~n&yb1QBUtm`o5s|WXNwQzeDa^1;J0GZS8z^;vDl*%FO32@5rXi z6Mf}{X93}mGP6;dLzfEXCCY zt5L_Zx8lK_E3kM*$;xhRAe+%1Xw7jUbPH!w79&2!aU430p;LJ`72?ImR0q1btlcPw0=6&vjt1* zUI@I%m$>HpVrYW@?x#=-e|UT}4farTV|fvayWITxF5TGm7~S*NVN>>CdrTZ4>lm|# zw-1feLw1Wzpov*lx)5H21~WI4Jji|VJTh>+g>m>sR%*0UmV^i6+nURHGG{B^mY67O z%e22>yv*%8(hiu2Ag(cZ3}+2FH1glq1dA$Wf2&+S@`Lx75zVoLIqj^*!TkwYW`Q zjkL5sp5rbTM?Mh8ZN#^kd**RSYlxEV9V*6Yt!aVQtn|fCmlxu;dK^5#rsnmwmRI5! zIIk*^zsgxSW;xEoeo7qg&La=YO1Av|S7VID!j3N%auTnp3%XRqQrLVz%;f^o#Y~5fn^y zgg;eK9eeR(t`l(v_6yRK@4xFWg!c( zTZ{Wk%iHldB1^6aYn^5|zrlKpdu3LxY@Yecod(cG{eoW6BU}niJfG~sxir?=_XI{w zD;SoYC5A{gb>u6ID2kAnctXN-WF&7;5FJ2=;xTwOq#;EJx!C~EESqG_E zJfbhRDSb9Kfzj88spN8XRsmlisPc01T*vW5sXE3YBkQb;ws*!T_=~QRXC^vQvUv6H zxPuWD=IK^HXkUAvhG*|m+O-MjnBOX(KsXP#i4Xk6T}ng?UCb#o?`I5_h9db8n$NSB ziDQOkV{eY@3j@oA-I*Od8J{igGFfsdfTD@TOGPavX=uyP4k_gNxixR^mU1l!nA%J#Hs+ub2Y}`c>We5)jAb+F`jGN%*H|U^>!R6?)Nk^)k%$VAZYBXi4GO3o* z=R~(;s2D=;nn;spWzU+iN>5`%cvq(t-!hlPm@!|Kjc5-xK8}^W60^!TvBFF)G86kRjwG~P^hp>7C%&Zpo7R<{{1t*_#hORWV+L)t-I$M(}ZJJ+!Zp*R>``Zx1{ zh~isSBASJ3#Vu+yXaudmE^N**2wL3VpT92iU7FV8Z0+(TDkFFHQ@DdmEm~TRbf64q zj3!%j(>!~#@l7?RXXy5V;d5*>Y#jMpMp|c5k4$z9u3d{GBvJZe94{Hw-8ylnT1MXI zljZ%7_(1{t)CoyHMU0JEmC38%K!cHAS;8mk0O7ofZU3(j!^X(JyQ}y{* z!U9BF_EF3WbgB!a$O{eicu2r@-_fUxSG@`4-+j)pu5qtOAN%9Ll9iwiFSeC+_t~dEl$KocoY@VQ zNHl}v{xL%qh39=x*m?ZWLGnv4D|;J5I#z@o=h``C-_O9Fv^!85|LE^}NQCvXsBU}& zihIpPjjla!;HPX|7$xhk*II;j;DGX8*|_tF{!U7Jsw|IOnkq2Im%U@LcolN-SjHen zSXrRZo%xMkAROLCOvEE)fAX6Ct~M-M8cG^i&CuErd(*hzbUvqE)i~&ZXAyNdvWz{d z%Y}AHhayb9^OVobomQ{Am_t^u8Pd_6rL}g7TwaUZ!m?vWv3Q)W7)nek5zkS_C|k<@ zaJ$nrbE(5SGf{888Tg_ZkwfV%)1Fb+K(6+wW01NdmgB;FSoeC+vJrZ~`i77Rbkmxd z#^rkND=`~CWx>uO6~C;NDVsCWPVF@-C6m3xdArBfD%Euz)r23;4YBduwfD)uL2Rbp z=cgP9X_H;#u1-g8vTH{PMz8&X{8l!P-*S$Q==RUE3xKCtvUN=_u7;6pBj)u=E*v}& zRAKLr#ArOzikaDwv>iKoo@ri;ckYOb2aILb*&iVu?hV(_Q;8~MAhuUW^Vyc-e7fvT zgZ6zaNDsvi)Ez6cN;)T!eYb0p-e;?9ns+~Be(U{k*o}NIX%*tIvQF<6&Xh^%ejDZ5 z$Di(Ejj+1BYEPC8>f>$Xi1-TDH)W^Rn`#)2o*!|BQLpUm*hKHP#??|K99X z-5f9t+{&H$$lm;pMPwn}^;BbI3yPuipVa$cUR$!zrOao2#uUnee0#6kRPCy-<|=i& ztyW5Zmd2-|y>xLZ_+rGXAxCF9Sd3yk=q6mS4auI#=XC!AWPokr^LMh)M}@ZXYMdOZiPs53tFW;x1FPX>*y3ld?CZuVWx zvDG4#VhbY;KhY^WGQv&8a`4SvNbiWM>q5mB@Iu`UX|;|mM4j@A>5M9A4aylrF@6a0 z&R74k>Wj?eqh%*W)(&pia=DsbvLrw0TREQzAuX@6p21IL9j*s_eD2Xm{itrh&WT)L z?Vq>rwLGCNS4<>5t!s{Vr=Xn}($B^c>%lX0$6hS7j!mbI7vpp0G9S72_4OWJvWF2L zM|MPmg@_N_jT~#cyU|YBr5#AKh9Q6CLz@>x%d zF}~LI$=LO5oKMBvxxxZHbp$5nf9KGC^Sk+RwBGwQmT}_kswK-lhHN0Ulo4r2??Rt) zZ|3bGb3HdrXJVs-7B|ZQ2Z=9Z836y%qU#;mcw0riqcJ!=hQ$Ev@{8?G*_*Fz7Enhwt@nsm-)oESX zK`k|N~k6$tFwhiByinpg7? zv#~!iKE|1vU%8)lZq+-=IG=1n*8(Xc1K-P;`uU2amRXCtQ)R|N9;L`NMVZ$t99ixw zdgZL^kJb11Po4282V+JoN7BJk$btAoGxT*bBx#PHzgDa>8=?PVWHQ;Ed#$|&@pAL! zHj7fH&Xr-T34{&puV*!XPD89#e579}i|O{;up(7K<8jox2g)p0lGhr&Y&F*ag>z%M z&z&)2>9_M#imdL`UEv`?JN8G)9-aSVw^@0VRW`W;xN^p(UGeh3Z0pybxsqz9ta&HDb0!6o)BR&%iUil_db{!%c)7oNEB9y=y zEG?u>44``i(m_Ap)-=d!(N3;cVG;F;jjb$4Vr)edRfpw{Gi)rYuC7(qnw4!RItWY1 ziCGzP4T>}@L+f6sRrYv_mg1>=XS32)>^6K!cv;H&_7cx$o)XjxA3B-gzId{5p;*0 zOFOb&59&&fL%QnsV(IQ&FoSnS!zFVlLUFwE085)45XL`ZJDz21DAzpqZ!@g%r5D>~ zzBVBz{86@#Zty*E0596H0iAp%UvNf&ch4V*o3NTO`;eVE+hJXF$V!g48hSSDg>P+z zHDYF8B5j2J%UG^rUf>nZopUk+au|y&UISeUD?lpR+r(K|3}cP0`3?JRmyYq7>4GoB ze`T7A2~*dH`MOqFA`}|Ai4SG&x-{5vT58YWxZy5mMv1a36?qIs&-SGN;IJv26FiQ~v&FSBAB-ML-ii7S>!>`jP&sw@9xgb1ACaxz*XlZ{Xd+bABls}_NCiH`mzLNKSFMyg*1S!6d}9U z(DWt0e=dxZ)UDEjo(nc+Q?;zD2T-`Ao!C1@{p^Tz&#o6IP3I2nwaz=(y)>&~f2@Qm zGrlvptC6W*1t9j>(^uk5kR0F2?#^aA68`mkc-I}#!#EQEH=8fUYb#>siP&BCSp45= zKR?{=H|ka&Xk8u(#^a;T06;c+WLlTdPAfD=&zZBxu%t<_Kl^MPm=-!mQY44bVqbA=BA7(wz@wSoBiTKjy8& z==CiG6|<^;^&q;`soKf)NNN_)k=+>7)-` zit6yX-{Bp{{oN;yKB|L+RoZb?%BsBE^X&U=J{2)?IFB9b1a5^CRHMyPZT=v}UJ0~u zhHo^450H;KI(N7ep};}oIOvRZI38mSV_hrBe3*~L&>6Uk4N-mMs8c*R2l_9cq{kr1dxT&t)6#vwQS2~<8#V7O;>I!ild3>Qx)`ZRcI@m>IE!E9-eonguha&n)ztRTI*E{(Lv=|p=8OQiud7pGtVPN@8ABh zTQCk?^YRW9-!;o!&NBCASEQ?Ta;TwQ?CyYOFFfNd2rYkLkHwR7>1nITil(%mJ(8*1 zh5J!c$+<^@-Pb*M^X>IMTgG%3gL**w6^1g;dW>?0`*!p9`*&-Kt09-M+@+6du9C=A z=U_8j#Mw2K!}^u>gj?%v^l%M4|4J0~$zJ*MdK*>>ok?xKdaUzo1x?`$m=ThJAgjFb6fwiM+pCIhJxOw4$7?mS-Y|yG~>Y zV3Z{1F2Y8PO085Pd58PvjtTxfdfC(3n*^K*m$m_Q=_T_@hWN=INmfnC#gL4C z*8lz_KH#b;b?)f`SmUd~IXQLH{I~X)V{wn4W6i&e*L`uv)gULZZ04IgexiZrtOuI1{bSbGJ_ z^<2~@kboyxYGsg}nDEV2S{Iss6|?J3$MiB;5m?>%hMpPrhL`F-^B3aVdqRH4qSM6_ zX^us=STuaLxix4I^3#*x3a^uKHnQd4$v1!mH=c zUAX+<;g>I6_}t|)FKnMbefjA5vu7?{ynOMwD@V^~q_f zubkSxbouPX3lAQ?`_sn`KlIBVUs!nH%B7btUs=8I+{HOZyI1GMnU}ZEynN~GmDk#F z7UJInm$rZB<(Osr+4W0jUpaej`}yt5yI+=Hue=(kUI{|hwqMykw{VW14<0^!dG*38 z7r(rH>F~nKXBW?$*}fdSJ$L%t - + @@ -106,5 +106,9 @@ 9.2.0 + + + + \ No newline at end of file diff --git a/src/StatusAggregator/StatusAggregator.nuspec b/src/StatusAggregator/StatusAggregator.nuspec new file mode 100644 index 000000000..814ed5f51 --- /dev/null +++ b/src/StatusAggregator/StatusAggregator.nuspec @@ -0,0 +1,22 @@ + + + + StatusAggregator.$branch$ + $version$ + ArchivePackages + .NET Foundation + .NET Foundation + StatusAggregator + Copyright .NET Foundation + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 368008f17..087ba8ccf 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -34,7 +34,7 @@ public StatusExporter(CloudBlobContainer container, ITableWrapper table) public async Task Export() { - var rootComponent = Components.Root; + var rootComponent = Components.CreateNuGetServiceRootComponent(); var activeEvents = _table.GetActiveEvents(); foreach (var activeEvent in activeEvents) @@ -46,7 +46,7 @@ public async Task Export() continue; } - currentComponent.Status = activeEvent.AffectedComponentStatus; + currentComponent.Status = (ComponentStatus)activeEvent.AffectedComponentStatus; } var recentEvents = _table @@ -63,7 +63,7 @@ public async Task Export() return e.AsEvent(messages); }); - var status = new ServiceStatus(Components.Root, recentEvents); + var status = new ServiceStatus(rootComponent, recentEvents); var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); var blob = _container.GetBlockBlobReference(StatusBlobName); diff --git a/src/StatusAggregator/Table/EventEntity.cs b/src/StatusAggregator/Table/EventEntity.cs index f51d64b24..5d0e0edc2 100644 --- a/src/StatusAggregator/Table/EventEntity.cs +++ b/src/StatusAggregator/Table/EventEntity.cs @@ -17,13 +17,13 @@ public EventEntity(IncidentEntity incidentEntity) : base(DefaultPartitionKey, GetRowKey(incidentEntity)) { AffectedComponentPath = incidentEntity.AffectedComponentPath; - AffectedComponentStatus = incidentEntity.AffectedComponentStatus; + AffectedComponentStatus = (int)incidentEntity.AffectedComponentStatus; StartTime = incidentEntity.CreationTime; incidentEntity.EventRowKey = RowKey; } public string AffectedComponentPath { get; set; } - public ComponentStatus AffectedComponentStatus { get; set; } + public int AffectedComponentStatus { get; set; } public DateTime StartTime { get; set; } public DateTime? EndTime { get; set; } public bool IsActive @@ -34,7 +34,7 @@ public bool IsActive public Event AsEvent(IEnumerable messages) { - return new Event(AffectedComponentPath, AffectedComponentStatus, StartTime, EndTime, messages); + return new Event(AffectedComponentPath, (ComponentStatus)AffectedComponentStatus, StartTime, EndTime, messages); } private static string GetRowKey(IncidentEntity incidentEntity) diff --git a/src/StatusAggregator/Table/IncidentEntity.cs b/src/StatusAggregator/Table/IncidentEntity.cs index c8ead2217..6a839e77f 100644 --- a/src/StatusAggregator/Table/IncidentEntity.cs +++ b/src/StatusAggregator/Table/IncidentEntity.cs @@ -18,7 +18,7 @@ public IncidentEntity(ParsedIncident parsedIncident) { IncidentApiId = parsedIncident.Id; AffectedComponentPath = parsedIncident.AffectedComponentPath; - AffectedComponentStatus = parsedIncident.AffectedComponentStatus; + AffectedComponentStatus = (int)parsedIncident.AffectedComponentStatus; CreationTime = parsedIncident.CreationTime; MitigationTime = parsedIncident.MitigationTime; } @@ -31,7 +31,7 @@ public bool IsLinkedToEvent } public string IncidentApiId { get; set; } public string AffectedComponentPath { get; set; } - public ComponentStatus AffectedComponentStatus { get; set; } + public int AffectedComponentStatus { get; set; } public DateTime CreationTime { get; set; } public DateTime? MitigationTime { get; set; } public bool IsActive diff --git a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs index e6dd48eea..5fde76df1 100644 --- a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs +++ b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs @@ -55,7 +55,7 @@ public async Task CreatesNewEventIfNoPossibleEvents() Assert.Equal(Id, incidentEntity.IncidentApiId); Assert.Equal(CreationTime, incidentEntity.CreationTime); Assert.Equal(AffectedComponentPath, incidentEntity.AffectedComponentPath); - Assert.Equal(AffectedComponentStatus, incidentEntity.AffectedComponentStatus); + Assert.Equal((int)AffectedComponentStatus, incidentEntity.AffectedComponentStatus); Assert.NotNull(eventEntity); Assert.Equal(eventEntity.RowKey, incidentEntity.EventRowKey); Assert.Equal(IncidentEntity.DefaultPartitionKey, incidentEntity.PartitionKey); From 0eaf2ec71b1370abc7c37bde4ebc7f181665a4ee Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 13 Jul 2018 16:03:46 -0700 Subject: [PATCH 08/49] add statusaggregator to build scripts --- build.ps1 | 6 ++++-- test.ps1 | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/build.ps1 b/build.ps1 index 74762a2ab..e66c6fdba 100644 --- a/build.ps1 +++ b/build.ps1 @@ -114,7 +114,8 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { ` "$PSScriptRoot\src\NuGet.Jobs.Common\Properties\AssemblyInfo.g.cs", "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs", "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs" + "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs", + "$PSScriptRoot\src\StatusAggregator\Properties\AssemblyInfo.g.cs" $versionMetadata | ForEach-Object { Set-VersionInfo -Path $_ -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA @@ -173,7 +174,8 @@ Invoke-BuildStep 'Creating artifacts' { "src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", ` "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", ` "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", ` - "src/PackageLagMonitor/Monitoring.PackageLag.csproj" ` + "src/PackageLagMonitor/Monitoring.PackageLag.csproj", ` + "src/StatusAggregator/StatusAggregator.csproj" ` + $ProjectsWithSymbols Foreach ($Project in $Projects) { diff --git a/test.ps1 b/test.ps1 index 0c42d19a2..4b8aaecad 100644 --- a/test.ps1 +++ b/test.ps1 @@ -37,7 +37,8 @@ Function Run-Tests { "tests\Validation.PackageSigning.ValidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.ValidateCertificate.Tests.dll", ` "tests\Validation.PackageSigning.RevalidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.RevalidateCertificate.Tests.dll", ` "tests\Validation.PackageSigning.Core.Tests\bin\$Configuration\Validation.PackageSigning.Core.Tests.dll", ` - "tests\Validation.Common.Job.Tests\bin\$Configuration\Validation.Common.Job.Tests.dll" + "tests\Validation.Common.Job.Tests\bin\$Configuration\Validation.Common.Job.Tests.dll", ` + "tests\StatusAggregator\bin\$Configuration\StatusAggregator.dll" $TestCount = 0 From 04dbf018f55b6caf55dd12d878d079955ff6625c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 13 Jul 2018 16:04:45 -0700 Subject: [PATCH 09/49] sign status aggregator --- src/StatusAggregator/StatusAggregator.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 23468fe56..9fe8e174b 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -111,4 +111,5 @@ + \ No newline at end of file From 8b5e98fed7f0ebf9c9d25288899f8e06071e697b Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 13 Jul 2018 16:20:01 -0700 Subject: [PATCH 10/49] fix containername in script --- src/StatusAggregator/Scripts/StatusAggregator.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StatusAggregator/Scripts/StatusAggregator.cmd b/src/StatusAggregator/Scripts/StatusAggregator.cmd index fe84a08c7..699ccb50b 100644 --- a/src/StatusAggregator/Scripts/StatusAggregator.cmd +++ b/src/StatusAggregator/Scripts/StatusAggregator.cmd @@ -12,7 +12,7 @@ start /w statusaggregator.exe ^ -StatusIncidentApiRoutingId "#{Jobs.statusaggregator.IncidentApiRoutingId}" ^ -StatusIncidentApiCertificateThumbprint "#{Jobs.statusaggregator.IncidentApiCertificateThumbprint}" ^ -StatusStorageAccount "#{Jobs.statusaggregator.StorageAccount}" ^ - -StatusContainerName "#{Jobs.statusaggregator.TableName}" ^ + -StatusContainerName "#{Jobs.statusaggregator.ContainerName}" ^ -StatusTableName "#{Jobs.statusaggregator.TableName}" ^ -StatusEnvironment "#{Jobs.statusaggregator.Environment}" ^ -StatusMaximumSeverity "#{Jobs.statusaggregator.MaximumSeverity}" From 69dab670b75881c8564a32fe586d4af80c4049e0 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 13 Jul 2018 17:06:14 -0700 Subject: [PATCH 11/49] add missing caret --- src/StatusAggregator/Scripts/StatusAggregator.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StatusAggregator/Scripts/StatusAggregator.cmd b/src/StatusAggregator/Scripts/StatusAggregator.cmd index 699ccb50b..989547efe 100644 --- a/src/StatusAggregator/Scripts/StatusAggregator.cmd +++ b/src/StatusAggregator/Scripts/StatusAggregator.cmd @@ -15,7 +15,7 @@ start /w statusaggregator.exe ^ -StatusContainerName "#{Jobs.statusaggregator.ContainerName}" ^ -StatusTableName "#{Jobs.statusaggregator.TableName}" ^ -StatusEnvironment "#{Jobs.statusaggregator.Environment}" ^ - -StatusMaximumSeverity "#{Jobs.statusaggregator.MaximumSeverity}" + -StatusMaximumSeverity "#{Jobs.statusaggregator.MaximumSeverity}" ^ -VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^ -ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^ -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^ From cb04d653bef08b03ce4a52baf821477dffd4489a Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 13 Jul 2018 17:59:17 -0700 Subject: [PATCH 12/49] get certificate from keyvault --- .../Configuration/JobArgumentNames.cs | 4 +-- src/StatusAggregator/Job.cs | 31 +++++++++---------- .../Scripts/StatusAggregator.cmd | 2 +- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs index d679da57a..602236843 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs @@ -126,9 +126,7 @@ public static class JobArgumentNames public const string StatusMaximumSeverity = "StatusMaximumSeverity"; public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri"; public const string StatusIncidentApiRoutingId = "StatusIncidentApiRoutingId"; - public const string StatusIncidentApiCertificateThumbprint = "StatusIncidentApiCertificateThumbprint"; - public const string StatusIncidentApiCertificateStoreName = "StatusIncidentApiCertificateStoreName"; - public const string StatusIncidentApiCertificateStoreLocation = "StatusIncidentApiCertificateStoreLocation"; + public const string StatusIncidentApiCertificate = "StatusIncidentApiCertificate"; // Arguments specific to Stats.AggregateCdnDownloadsInGallery public static string BatchSleepSeconds = "BatchSleepSeconds"; diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index f548194f7..523ac615d 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -8,8 +8,8 @@ using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; using StatusAggregator.Incidents; using StatusAggregator.Incidents.Parse; using StatusAggregator.Table; @@ -40,22 +40,8 @@ public override void Init(IServiceContainer serviceContainer, IDictionary GetIncidentParsers(IEnumerable envi new OutdatedSearchServiceInstanceIncidentParser(environments, maximumSeverity) }; } + + private static X509Certificate2 GetCertificateFromJson(string certJson) + { + var certJObject = JObject.Parse(certJson); + + var certData = certJObject["Data"].Value(); + var certPassword = certJObject["Password"].Value(); + + var certBytes = Convert.FromBase64String(certData); + return new X509Certificate2(certBytes, certPassword); + } } } diff --git a/src/StatusAggregator/Scripts/StatusAggregator.cmd b/src/StatusAggregator/Scripts/StatusAggregator.cmd index 989547efe..1ed312c46 100644 --- a/src/StatusAggregator/Scripts/StatusAggregator.cmd +++ b/src/StatusAggregator/Scripts/StatusAggregator.cmd @@ -10,7 +10,7 @@ title #{Jobs.archivepackages.Title} start /w statusaggregator.exe ^ -StatusIncidentApiBaseUri "#{Jobs.statusaggregator.IncidentApiBaseUri}" ^ -StatusIncidentApiRoutingId "#{Jobs.statusaggregator.IncidentApiRoutingId}" ^ - -StatusIncidentApiCertificateThumbprint "#{Jobs.statusaggregator.IncidentApiCertificateThumbprint}" ^ + -StatusIncidentApiCertificate "#{Jobs.statusaggregator.IncidentApiCertificate}" ^ -StatusStorageAccount "#{Jobs.statusaggregator.StorageAccount}" ^ -StatusContainerName "#{Jobs.statusaggregator.ContainerName}" ^ -StatusTableName "#{Jobs.statusaggregator.TableName}" ^ From ceea509ba3b6d3c9563303ece5cad72fed3c6885 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 16 Jul 2018 09:59:13 -0700 Subject: [PATCH 13/49] exporter fixes --- src/StatusAggregator/Components.cs | 22 ++++++++--------- src/StatusAggregator/StatusExporter.cs | 29 ++++++++++++----------- src/StatusAggregator/Table/EventEntity.cs | 2 +- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/StatusAggregator/Components.cs b/src/StatusAggregator/Components.cs index 766724bd7..751434c02 100644 --- a/src/StatusAggregator/Components.cs +++ b/src/StatusAggregator/Components.cs @@ -16,8 +16,8 @@ public static IComponent CreateNuGetServiceRootComponent() "Browsing the Gallery website", new[] { - new LeafComponent("USNC", "Primary region"), - new LeafComponent("USSC", "Backup region") + new LeafComponent("North Central US", "Primary region"), + new LeafComponent("South Central US", "Backup region") }), new TreeComponent( "Restore", @@ -25,14 +25,14 @@ public static IComponent CreateNuGetServiceRootComponent() new IComponent[] { new TreeComponent( - "V3", + "V3 Protocol", "Restore using the V3 API", new[] { new LeafComponent("Global", "V3 restore for users outside of China"), new LeafComponent("China", "V3 restore for users inside China") }), - new LeafComponent("V2", "Restore using the V2 API") + new LeafComponent("V2 Protocol", "Restore using the V2 API") }), new TreeComponent( "Search", @@ -41,19 +41,19 @@ public static IComponent CreateNuGetServiceRootComponent() { new PrimarySecondaryComponent( "Global", - "Search for packages outside Asia", + "Search for packages outside China", new[] { - new LeafComponent("USNC", "Primary region"), - new LeafComponent("USSC", "Backup region") + new LeafComponent("North Central US", "Primary region"), + new LeafComponent("South Central US", "Backup region") }), new PrimarySecondaryComponent( - "Asia", - "Search for packages inside Asia", + "China", + "Search for packages inside China", new[] { - new LeafComponent("EA", "Primary region"), - new LeafComponent("SEA", "Backup region") + new LeafComponent("East Asia", "Primary region"), + new LeafComponent("Southeast Asia", "Backup region") }) }), new LeafComponent("Package Publishing", "Uploading new packages to NuGet.org") diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 087ba8ccf..0049f7c30 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -36,19 +36,6 @@ public async Task Export() { var rootComponent = Components.CreateNuGetServiceRootComponent(); - var activeEvents = _table.GetActiveEvents(); - foreach (var activeEvent in activeEvents) - { - var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); - - if (currentComponent == null) - { - continue; - } - - currentComponent.Status = (ComponentStatus)activeEvent.AffectedComponentStatus; - } - var recentEvents = _table .CreateQuery() .Where(e => @@ -61,7 +48,21 @@ public async Task Export() .ToList() .Select(m => m.AsMessage()); return e.AsEvent(messages); - }); + }) + .Where(e => e.Messages != null && e.Messages.Any()); + + foreach (var activeEvent in recentEvents.Where(e => e.EndTime == null || e.EndTime >= DateTime.Now)) + { + var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); + + if (currentComponent == null) + { + continue; + } + + currentComponent.Status = activeEvent.AffectedComponentStatus; + } + var status = new ServiceStatus(rootComponent, recentEvents); var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); diff --git a/src/StatusAggregator/Table/EventEntity.cs b/src/StatusAggregator/Table/EventEntity.cs index 5d0e0edc2..d6c2ccea6 100644 --- a/src/StatusAggregator/Table/EventEntity.cs +++ b/src/StatusAggregator/Table/EventEntity.cs @@ -32,7 +32,7 @@ public bool IsActive set { } } - public Event AsEvent(IEnumerable messages) + public Event AsEvent(IEnumerable messages) { return new Event(AffectedComponentPath, (ComponentStatus)AffectedComponentStatus, StartTime, EndTime, messages); } From 8de4d990b6cfbb94b571c27cda38adddc9f9c1fc Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 16 Jul 2018 10:05:17 -0700 Subject: [PATCH 14/49] disable flaky test --- .../SignatureValidatorIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs index ab55f228f..3d20fedf2 100644 --- a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs +++ b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs @@ -210,7 +210,7 @@ public async Task AcceptsValidSignedPackage() Assert.Empty(result.Issues); } - [Fact] + [Fact(Skip = "This test is flaky, disabling for now.")] public async Task RejectsUntrustedSigningCertificate() { // Arrange From ec75f3792ad9427a1205d7faaa1ad4a0b038e4f4 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 16 Jul 2018 10:34:07 -0700 Subject: [PATCH 15/49] fix messagings and export logic --- src/StatusAggregator/MessageUpdater.cs | 4 ++-- src/StatusAggregator/StatusExporter.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 143e03e06..5ce3f52ce 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -21,7 +21,7 @@ public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime n // Enough time must have passed before we create a start message for an event. if (nextCreationTime > eventEntity.StartTime + EventStartDelay) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "WE ARE IMPACTED"); + var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "Package publishing is degraded. New packages will likely take longer than usual before becoming available for download."); await _table.InsertOrReplaceAsync(messageEntity); } } @@ -36,7 +36,7 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) // Only create a message if the event already has messages associated with it. if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, "NO LONGER IMPACTED"); + var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, "Package publishing is no longer degraded. New packages should become available for download as quickly as usual."); await _table.InsertOrReplaceAsync(messageEntity); } } diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 0049f7c30..0be6b6415 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -40,7 +40,7 @@ public async Task Export() .CreateQuery() .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && - (e.IsActive || (e.EndTime >= DateTime.Now - EventVisibilityPeriod))) + (e.IsActive || (e.EndTime >= DateTime.UtcNow - EventVisibilityPeriod))) .ToList() .Select(e => { @@ -51,7 +51,7 @@ public async Task Export() }) .Where(e => e.Messages != null && e.Messages.Any()); - foreach (var activeEvent in recentEvents.Where(e => e.EndTime == null || e.EndTime >= DateTime.Now)) + foreach (var activeEvent in recentEvents.Where(e => e.EndTime == null || e.EndTime >= DateTime.UtcNow)) { var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); From aaadf9357f7681ece9c794dd1d398e4805bef7d0 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 16 Jul 2018 13:55:46 -0700 Subject: [PATCH 16/49] "thank you for your patience" --- src/StatusAggregator/MessageUpdater.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 5ce3f52ce..6df23c66a 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -36,7 +36,7 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) // Only create a message if the event already has messages associated with it. if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, "Package publishing is no longer degraded. New packages should become available for download as quickly as usual."); + var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, "Package publishing is no longer degraded. New packages should become available for download as quickly as usual. Thank you for your patience."); await _table.InsertOrReplaceAsync(messageEntity); } } From a13521dbb9b0501c0aa3fae09715ebeb3899d4e0 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 16 Jul 2018 15:43:56 -0700 Subject: [PATCH 17/49] revert applicationhost.config changes --- .vs/config/applicationhost.config | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.vs/config/applicationhost.config b/.vs/config/applicationhost.config index 030cffcb2..582b1211a 100644 --- a/.vs/config/applicationhost.config +++ b/.vs/config/applicationhost.config @@ -163,20 +163,12 @@ - + - - - - - - - - From 71e6ea4ba610a012c045642786033a0e8410ae85 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Tue, 17 Jul 2018 15:59:24 -0700 Subject: [PATCH 18/49] integrate with pingdom --- .../Configuration/JobArgumentNames.cs | 2 +- src/StatusAggregator/Components.cs | 58 ++++++++---- src/StatusAggregator/Incidents/Incident.cs | 2 +- .../Incidents/IncidentCollector.cs | 10 +- .../Incidents/Parse/DefaultIncidentParser.cs | 60 ------------ .../Incidents/Parse/EnvironmentFilter.cs | 25 +++++ .../Parse/EnvironmentPrefixIncidentParser.cs | 18 ++++ .../Incidents/Parse/IIncidentParsingFilter.cs | 9 ++ .../Incidents/Parse/IncidentParser.cs | 39 +++++++- ...onalSearchServiceInstanceIncidentParser.cs | 26 +++--- ...atedSearchServiceInstanceIncidentParser.cs | 8 +- .../Incidents/Parse/PingdomIncidentParser.cs | 92 +++++++++++++++++++ .../Incidents/Parse/SeverityFilter.cs | 19 ++++ .../Parse/ValidationDurationIncidentParser.cs | 8 +- src/StatusAggregator/Job.cs | 21 ++++- src/StatusAggregator/StatusAggregator.csproj | 6 +- 16 files changed, 291 insertions(+), 112 deletions(-) delete mode 100644 src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs create mode 100644 src/StatusAggregator/Incidents/Parse/EnvironmentFilter.cs create mode 100644 src/StatusAggregator/Incidents/Parse/EnvironmentPrefixIncidentParser.cs create mode 100644 src/StatusAggregator/Incidents/Parse/IIncidentParsingFilter.cs create mode 100644 src/StatusAggregator/Incidents/Parse/PingdomIncidentParser.cs create mode 100644 src/StatusAggregator/Incidents/Parse/SeverityFilter.cs diff --git a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs index 602236843..eace904e7 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs @@ -125,7 +125,7 @@ public static class JobArgumentNames public const string StatusEnvironment = "StatusEnvironment"; public const string StatusMaximumSeverity = "StatusMaximumSeverity"; public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri"; - public const string StatusIncidentApiRoutingId = "StatusIncidentApiRoutingId"; + public const string StatusIncidentApiTeamId = "StatusIncidentApiTeamId"; public const string StatusIncidentApiCertificate = "StatusIncidentApiCertificate"; // Arguments specific to Stats.AggregateCdnDownloadsInGallery diff --git a/src/StatusAggregator/Components.cs b/src/StatusAggregator/Components.cs index 751434c02..f32d70fbc 100644 --- a/src/StatusAggregator/Components.cs +++ b/src/StatusAggregator/Components.cs @@ -4,59 +4,83 @@ namespace StatusAggregator { public static class Components { + public const string RootName = "NuGet"; + public const string GalleryName = "NuGet.org"; + public const string RestoreName = "Restore"; + public const string SearchName = "Search"; + public const string UploadName = "Package Publishing"; + + public const string V2ProtocolName = "V2 Protocol"; + public const string V3ProtocolName = "V3 Protocol"; + + public const string GlobalRegionName = "Global"; + public const string ChinaRegionName = "China"; + + public const string UsncInstanceName = "North Central US"; + public const string UsscInstanceName = "South Central US"; + public const string EaInstanceName = "East Asia"; + public const string SeaInstanceName = "Southeast Asia"; + public static IComponent CreateNuGetServiceRootComponent() { return new TreeComponent( - "NuGet", + RootName, "", new IComponent[] { new PrimarySecondaryComponent( - "NuGet.org", + GalleryName, "Browsing the Gallery website", new[] { - new LeafComponent("North Central US", "Primary region"), - new LeafComponent("South Central US", "Backup region") + new LeafComponent(UsncInstanceName, "Primary region"), + new LeafComponent(UsscInstanceName, "Backup region") }), new TreeComponent( - "Restore", + RestoreName, "Downloading and installing packages from NuGet", new IComponent[] { new TreeComponent( - "V3 Protocol", + V3ProtocolName, "Restore using the V3 API", new[] { - new LeafComponent("Global", "V3 restore for users outside of China"), - new LeafComponent("China", "V3 restore for users inside China") + new LeafComponent(GlobalRegionName, "V3 restore for users outside of China"), + new LeafComponent(ChinaRegionName, "V3 restore for users inside China") }), - new LeafComponent("V2 Protocol", "Restore using the V2 API") + new PrimarySecondaryComponent( + V2ProtocolName, + "Restore using the V2 API", + new[] + { + new LeafComponent(UsncInstanceName, "Primary region"), + new LeafComponent(UsscInstanceName, "Backup region") + }) }), new TreeComponent( - "Search", + SearchName, "Searching for new and existing packages in Visual Studio or the Gallery website", new[] { new PrimarySecondaryComponent( - "Global", + GlobalRegionName, "Search for packages outside China", new[] { - new LeafComponent("North Central US", "Primary region"), - new LeafComponent("South Central US", "Backup region") + new LeafComponent(UsncInstanceName, "Primary region"), + new LeafComponent(UsscInstanceName, "Backup region") }), new PrimarySecondaryComponent( - "China", + ChinaRegionName, "Search for packages inside China", new[] { - new LeafComponent("East Asia", "Primary region"), - new LeafComponent("Southeast Asia", "Backup region") + new LeafComponent(EaInstanceName, "Primary region"), + new LeafComponent(SeaInstanceName, "Backup region") }) }), - new LeafComponent("Package Publishing", "Uploading new packages to NuGet.org") + new LeafComponent(UploadName, "Uploading new packages to NuGet.org") }); } } diff --git a/src/StatusAggregator/Incidents/Incident.cs b/src/StatusAggregator/Incidents/Incident.cs index dd3369707..fe2003121 100644 --- a/src/StatusAggregator/Incidents/Incident.cs +++ b/src/StatusAggregator/Incidents/Incident.cs @@ -12,7 +12,7 @@ public class Incident public IncidentStatus Status { get; set; } public DateTime CreateDate { get; set; } public string Title { get; set; } - public string RoutingId { get; set; } + public string OwningTeamId { get; set; } public IncidentSourceData Source { get; set; } public IncidentStateChangeEventData MitigationData { get; set; } public IncidentStateChangeEventData ResolutionData { get; set; } diff --git a/src/StatusAggregator/Incidents/IncidentCollector.cs b/src/StatusAggregator/Incidents/IncidentCollector.cs index bf0041b39..7e936dd6f 100644 --- a/src/StatusAggregator/Incidents/IncidentCollector.cs +++ b/src/StatusAggregator/Incidents/IncidentCollector.cs @@ -21,11 +21,11 @@ public class IncidentCollector : IIncidentCollector public IncidentCollector( Uri incidentApiBaseUri, X509Certificate2 incidentApiCertificate, - string incidentApiRoutingId) + string incidentApiTeamId) { _incidentApiBaseUri = incidentApiBaseUri; _incidentApiCertificate = incidentApiCertificate; - _incidentApiRoutingId = incidentApiRoutingId; + _incidentApiTeamId = incidentApiTeamId; } public Task GetIncident(string id) @@ -65,7 +65,7 @@ public async Task> GetRecentIncidents(DateTime since) private readonly Uri _incidentApiBaseUri; private readonly X509Certificate2 _incidentApiCertificate; - private readonly string _incidentApiRoutingId; + private readonly string _incidentApiTeamId; private string GetIncidentApiIncidentList(string oDataQueryParameters) { @@ -79,12 +79,12 @@ private string GetIncidentApiGetIncidentQuery(string id) private string GetIncidentApiIncidentListAllIncidentsQuery() { - return $"$filter=RoutingId eq '{_incidentApiRoutingId}'"; + return $"$filter=OwningTeamId eq '{_incidentApiTeamId}'"; } private string GetIncidentApiIncidentListRecentIncidentsQuery(DateTime cursor) { - return $"$filter=RoutingId eq '{_incidentApiRoutingId}' and CreateDate gt datetime'{cursor.ToString("o")}'"; + return $"$filter=OwningTeamId eq '{_incidentApiTeamId}' and CreateDate gt datetime'{cursor.ToString("o")}'"; } private Uri GetIncidentApiUri(string query) diff --git a/src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs deleted file mode 100644 index daf37e596..000000000 --- a/src/StatusAggregator/Incidents/Parse/DefaultIncidentParser.cs +++ /dev/null @@ -1,60 +0,0 @@ -using NuGet.Services.Status; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; - -namespace StatusAggregator.Incidents.Parse -{ - public abstract class DefaultIncidentParser : IncidentParser - { - private const string EnvironmentGroupName = "Environment"; - - private readonly IEnumerable _environments; - private readonly int _maximumSeverity; - - public DefaultIncidentParser(string subtitleRegEx, IEnumerable environments, int maximumSeverity) - : base(GetRegEx(subtitleRegEx)) - { - _environments = environments; - _maximumSeverity = maximumSeverity; - } - - protected override bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident) - { - parsedIncident = null; - - if (incident.Severity > _maximumSeverity) - { - return false; - } - - if (!_environments.Any(e => string.Equals(groups[EnvironmentGroupName].Value, e, StringComparison.OrdinalIgnoreCase))) - { - return false; - } - - if (!TryParseAffectedComponentPath(incident, groups, out var affectedComponentPath)) - { - return false; - } - - if (!TryParseAffectedComponentStatus(incident, groups, out var affectedComponentStatus)) - { - return false; - } - - parsedIncident = new ParsedIncident(incident, affectedComponentPath, affectedComponentStatus); - return true; - } - - protected abstract bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath); - - protected abstract bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus); - - private static string GetRegEx(string subtitleRegEx) - { - return $@"\[(?<{EnvironmentGroupName}>.*)\] {subtitleRegEx}"; - } - } -} diff --git a/src/StatusAggregator/Incidents/Parse/EnvironmentFilter.cs b/src/StatusAggregator/Incidents/Parse/EnvironmentFilter.cs new file mode 100644 index 000000000..15743002e --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/EnvironmentFilter.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StatusAggregator.Incidents.Parse +{ + public class EnvironmentFilter : IIncidentParsingFilter + { + public const string EnvironmentGroupName = "Environment"; + + public IEnumerable Environments { get; } + + public EnvironmentFilter(IEnumerable environments) + { + Environments = environments; + } + + public bool ShouldParse(Incident incident, GroupCollection groups) + { + return Environments.Any( + e => string.Equals(groups[EnvironmentGroupName].Value, e, StringComparison.OrdinalIgnoreCase)); + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/EnvironmentPrefixIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/EnvironmentPrefixIncidentParser.cs new file mode 100644 index 000000000..379b5c73b --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/EnvironmentPrefixIncidentParser.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StatusAggregator.Incidents.Parse +{ + public abstract class EnvironmentPrefixIncidentParser : IncidentParser + { + public EnvironmentPrefixIncidentParser(string subtitleRegEx, IEnumerable environments, IEnumerable filters) + : base(GetRegEx(subtitleRegEx), filters.Concat(new[] { new EnvironmentFilter(environments) })) + { + } + + private static string GetRegEx(string subtitleRegEx) + { + return $@"\[(?<{EnvironmentFilter.EnvironmentGroupName}>.*)\] {subtitleRegEx}"; + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/IIncidentParsingFilter.cs b/src/StatusAggregator/Incidents/Parse/IIncidentParsingFilter.cs new file mode 100644 index 000000000..9ba20c6a5 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/IIncidentParsingFilter.cs @@ -0,0 +1,9 @@ +using System.Text.RegularExpressions; + +namespace StatusAggregator.Incidents.Parse +{ + public interface IIncidentParsingFilter + { + bool ShouldParse(Incident incident, GroupCollection groups); + } +} diff --git a/src/StatusAggregator/Incidents/Parse/IncidentParser.cs b/src/StatusAggregator/Incidents/Parse/IncidentParser.cs index 4c240834e..d8dc0c1f3 100644 --- a/src/StatusAggregator/Incidents/Parse/IncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/IncidentParser.cs @@ -1,6 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using NuGet.Services.Status; +using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; namespace StatusAggregator.Incidents.Parse @@ -9,9 +12,18 @@ public abstract class IncidentParser : IIncidentParser { private readonly string _regExPattern; + private readonly IEnumerable _filters; + public IncidentParser(string regExPattern) { _regExPattern = regExPattern; + _filters = Enumerable.Empty(); + } + + public IncidentParser(string regExPattern, IEnumerable filters) + : this(regExPattern) + { + _filters = filters.ToList(); } public bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident) @@ -21,6 +33,31 @@ public bool TryParseIncident(Incident incident, out ParsedIncident parsedInciden return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); } - protected abstract bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident); + protected bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident) + { + parsedIncident = null; + + if (_filters.Any(f => !f.ShouldParse(incident, groups))) + { + return false; + } + + if (!TryParseAffectedComponentPath(incident, groups, out var affectedComponentPath)) + { + return false; + } + + if (!TryParseAffectedComponentStatus(incident, groups, out var affectedComponentStatus)) + { + return false; + } + + parsedIncident = new ParsedIncident(incident, affectedComponentPath, affectedComponentStatus); + return true; + } + + protected abstract bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath); + + protected abstract bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus); } } diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs index 075a4536b..6c32ace25 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs @@ -6,16 +6,16 @@ namespace StatusAggregator.Incidents.Parse { - public class OutdatedRegionalSearchServiceInstanceIncidentParser : DefaultIncidentParser + public class OutdatedRegionalSearchServiceInstanceIncidentParser : EnvironmentPrefixIncidentParser { - private const string ServiceEnvironmentGroupName = "SearchServiceName"; - private const string ServiceRegionGroupName = "SearchServiceName"; - private static string SubtitleRegEx = $@"Search service 'nuget-\[(?<{ServiceEnvironmentGroupName}>.*)\]-\[(?<{ServiceRegionGroupName}>.*)\]-search' is using an outdated index!"; + private const string ServiceEnvironmentGroupName = "SearchEnvironment"; + private const string ServiceRegionGroupName = "SearchRegion"; + private static string SubtitleRegEx = $@"Search service 'nuget-(?<{ServiceEnvironmentGroupName}>.*)-(?<{ServiceRegionGroupName}>.*)-search' is using an outdated index!"; private readonly IEnumerable _environments; - public OutdatedRegionalSearchServiceInstanceIncidentParser(IEnumerable environments, int maximumSeverity) - : base(SubtitleRegEx, environments, maximumSeverity) + public OutdatedRegionalSearchServiceInstanceIncidentParser(IEnumerable environments, IEnumerable filters) + : base(SubtitleRegEx, environments, filters) { _environments = environments; } @@ -40,11 +40,11 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo case "0": case "usnc": case "ussc": - region = "Global"; + region = Components.GlobalRegionName; break; case "eastasia": case "southeastasia": - region = "Asia"; + region = Components.ChinaRegionName; break; default: return false; @@ -54,22 +54,22 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo { case "0": case "usnc": - subRegion = "USNC"; + subRegion = Components.UsncInstanceName; break; case "ussc": - subRegion = "USSC"; + subRegion = Components.UsscInstanceName; break; case "eastasia": - subRegion = "EA"; + subRegion = Components.EaInstanceName; break; case "southeastasia": - subRegion = "SEA"; + subRegion = Components.SeaInstanceName; break; default: return false; } - affectedComponentPath = ComponentUtility.GetPath("NuGet", "Search", region, subRegion); + affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.SearchName, region, subRegion); return true; } diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index fe2ce935d..527d1c5fe 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -4,18 +4,18 @@ namespace StatusAggregator.Incidents.Parse { - public class OutdatedSearchServiceInstanceIncidentParser : DefaultIncidentParser + public class OutdatedSearchServiceInstanceIncidentParser : EnvironmentPrefixIncidentParser { private const string SubtitleRegEx = "A search service instance is using an outdated index!"; - public OutdatedSearchServiceInstanceIncidentParser(IEnumerable environments, int maximumSeverity) - : base(SubtitleRegEx, environments, maximumSeverity) + public OutdatedSearchServiceInstanceIncidentParser(IEnumerable environments, IEnumerable filters) + : base(SubtitleRegEx, environments, filters) { } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = ComponentUtility.GetPath("NuGet", "Package Publishing"); + affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.UploadName); return true; } diff --git a/src/StatusAggregator/Incidents/Parse/PingdomIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/PingdomIncidentParser.cs new file mode 100644 index 000000000..87087c112 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/PingdomIncidentParser.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using System.Text.RegularExpressions; +using NuGet.Services.Status; + +namespace StatusAggregator.Incidents.Parse +{ + public class PingdomIncidentParser : IncidentParser + { + private const string CheckNameGroupName = "CheckName"; + private const string CheckUrlGroupName = "CheckUrl"; + private static string SubtitleRegEx = $@"Pingdom check '(?<{CheckNameGroupName}>.*)' is failing! '(?<{CheckUrlGroupName}>.*)' is DOWN!"; + + public PingdomIncidentParser(IEnumerable filters) + : base(SubtitleRegEx, filters) + { + } + + protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) + { + affectedComponentPath = null; + + var checkName = groups[CheckNameGroupName].Value; + + switch (checkName) + { + case "CDN DNS": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.RestoreName, Components.V3ProtocolName); + break; + case "CDN Global": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.GlobalRegionName); + break; + case "CDN China": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.ChinaRegionName); + break; + case "Gallery DNS": + case "Gallery Home": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.GalleryName); + break; + case "Gallery USNC /": + case "Gallery USNC /Packages": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.GalleryName, Components.UsncInstanceName); + break; + case "Gallery USSC /": + case "Gallery USSC /Packages": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.GalleryName, Components.UsscInstanceName); + break; + case "Gallery USNC /api/v2/Packages()": + case "Gallery USNC /api/v2/package/NuGet.GalleryUptime/1.0.0": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.RestoreName, Components.V2ProtocolName, Components.UsncInstanceName); + break; + case "Gallery USSC /api/v2/Packages()": + case "Gallery USSC /api/v2/package/NuGet.GalleryUptime/1.0.0": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.RestoreName, Components.V2ProtocolName, Components.UsscInstanceName); + break; + case "Search USNC /query": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.SearchName, Components.GlobalRegionName, Components.UsncInstanceName); + break; + case "Search USSC /query": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.SearchName, Components.GlobalRegionName, Components.UsscInstanceName); + break; + case "Search EA /query": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.SearchName, Components.ChinaRegionName, Components.EaInstanceName); + break; + case "Search SEA /query": + affectedComponentPath = ComponentUtility.GetPath( + Components.RootName, Components.SearchName, Components.ChinaRegionName, Components.SeaInstanceName); + break; + default: + return false; + } + + return true; + } + + protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) + { + affectedComponentStatus = ComponentStatus.Degraded; + return true; + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/SeverityFilter.cs b/src/StatusAggregator/Incidents/Parse/SeverityFilter.cs new file mode 100644 index 000000000..468f32145 --- /dev/null +++ b/src/StatusAggregator/Incidents/Parse/SeverityFilter.cs @@ -0,0 +1,19 @@ +using System.Text.RegularExpressions; + +namespace StatusAggregator.Incidents.Parse +{ + public class SeverityFilter : IIncidentParsingFilter + { + private readonly int _maximumSeverity; + + public SeverityFilter(int maximumSeverity) + { + _maximumSeverity = maximumSeverity; + } + + public bool ShouldParse(Incident incident, GroupCollection groups) + { + return incident.Severity <= _maximumSeverity; + } + } +} diff --git a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs index 0ae17dc2d..54ba1b359 100644 --- a/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/ValidationDurationIncidentParser.cs @@ -4,18 +4,18 @@ namespace StatusAggregator.Incidents.Parse { - public class ValidationDurationIncidentParser : DefaultIncidentParser + public class ValidationDurationIncidentParser : EnvironmentPrefixIncidentParser { private const string SubtitleRegEx = "Too many packages are stuck in the \"Validating\" state!"; - public ValidationDurationIncidentParser(IEnumerable environments, int maximumSeverity) - : base(SubtitleRegEx, environments, maximumSeverity) + public ValidationDurationIncidentParser(IEnumerable environments, IEnumerable filters) + : base(SubtitleRegEx, environments, filters) { } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = ComponentUtility.GetPath("NuGet", "Package Publishing"); + affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.UploadName); return true; } diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 523ac615d..f4699864b 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -39,10 +39,10 @@ public override void Init(IServiceContainer serviceContainer, IDictionary GetIncidentParsers(IEnumerable environments, int maximumSeverity) { + var filters = GetIncidentParsingFilters(maximumSeverity); + return new IIncidentParser[] { - new ValidationDurationIncidentParser(environments, maximumSeverity), - new OutdatedRegionalSearchServiceInstanceIncidentParser(environments, maximumSeverity), - new OutdatedSearchServiceInstanceIncidentParser(environments, maximumSeverity) + new ValidationDurationIncidentParser(environments, filters), + new OutdatedRegionalSearchServiceInstanceIncidentParser(environments, filters), + new OutdatedSearchServiceInstanceIncidentParser(environments, filters), + new PingdomIncidentParser(filters) + }; + } + + private IEnumerable GetIncidentParsingFilters(int maximumSeverity) + { + return new IIncidentParsingFilter[] + { + new SeverityFilter(maximumSeverity) }; } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 9fe8e174b..ef2ee45b0 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -54,6 +54,10 @@ + + + + @@ -68,7 +72,7 @@ - + From c60ffbf9df6b6d2775e41ec79a976c19308d4436 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 18 Jul 2018 13:22:10 -0700 Subject: [PATCH 19/49] add messaging for other types of events --- src/StatusAggregator/MessageUpdater.cs | 150 ++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 4 deletions(-) diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 6df23c66a..66802665f 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using NuGet.Services.Status; using StatusAggregator.Table; namespace StatusAggregator @@ -21,11 +23,55 @@ public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime n // Enough time must have passed before we create a start message for an event. if (nextCreationTime > eventEntity.StartTime + EventStartDelay) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, "Package publishing is degraded. New packages will likely take longer than usual before becoming available for download."); - await _table.InsertOrReplaceAsync(messageEntity); + if (TryGetMessageForEventStartForEvent(eventEntity, out var message)) + { + var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, message); + await _table.InsertOrReplaceAsync(messageEntity); + } } } + private static bool TryGetMessageForEventStartForEvent(EventEntity eventEntity, out string message) + { + return TryGetMessageForEventHelper(eventEntity, _innerMessageMapForEventStart, "is", "", out message); + } + + private const string _youMayEncounterIssues = "You may encounter issues "; + private static readonly IEnumerable _innerMessageMapForEventStart = new InnerMessageForComponentPathPrefix[] + { + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.GalleryName), + $"{_youMayEncounterIssues}browsing the NuGet Gallery."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.ChinaRegionName), + $"{_youMayEncounterIssues}restoring packages from NuGet.org's V3 feed from China."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName), + $"{_youMayEncounterIssues}restoring packages from NuGet.org's V3 feed."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V2ProtocolName), + $"{_youMayEncounterIssues}restoring packages from NuGet.org's V2 feed."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName), + $"{_youMayEncounterIssues}restoring packages."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.SearchName, Components.ChinaRegionName), + $"{_youMayEncounterIssues}searching for packages from China."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.SearchName), + $"{_youMayEncounterIssues}searching for packages."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.UploadName), + "New packages will likely take longer than usual before becoming available for download."), + }; + public async Task CreateMessageForEventEnd(EventEntity eventEntity) { if (!eventEntity.EndTime.HasValue) @@ -36,8 +82,104 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) // Only create a message if the event already has messages associated with it. if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, "Package publishing is no longer degraded. New packages should become available for download as quickly as usual. Thank you for your patience."); - await _table.InsertOrReplaceAsync(messageEntity); + if (TryGetMessageForEventEndForEvent(eventEntity, out var message)) + { + var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, message); + await _table.InsertOrReplaceAsync(messageEntity); + } + } + } + + private static bool TryGetMessageForEventEndForEvent(EventEntity eventEntity, out string message) + { + return TryGetMessageForEventHelper(eventEntity, _innerMessageMapForEventEnd, "is no longer", " Thank you for your patience.", out message); + } + + private const string _youShouldNoLongerEncounterAnyIssues = "You should no longer encounter any issues "; + private static readonly IEnumerable _innerMessageMapForEventEnd = new InnerMessageForComponentPathPrefix[] + { + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.GalleryName), + $"{_youShouldNoLongerEncounterAnyIssues}browsing the NuGet Gallery."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.ChinaRegionName), + $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V3 feed from China."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName), + $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V3 feed."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V2ProtocolName), + $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V2 feed."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.RestoreName), + $"{_youShouldNoLongerEncounterAnyIssues}restoring packages."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.SearchName, Components.ChinaRegionName), + $"{_youShouldNoLongerEncounterAnyIssues}searching for packages from China."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.SearchName), + $"{_youShouldNoLongerEncounterAnyIssues}searching for packages."), + + new InnerMessageForComponentPathPrefix( + ComponentUtility.GetPath(Components.RootName, Components.UploadName), + "New packages should become available for download as quickly as usual."), + }; + + private static string[] _componentStatusNames = Enum.GetNames(typeof(ComponentStatus)); + private static bool TryGetMessageForEventHelper( + EventEntity eventEntity, + IEnumerable innerMessageMap, + string boldedPartInnerString, + string messageSuffix, + out string message) + { + message = null; + + var path = eventEntity.AffectedComponentPath; + var component = Components.CreateNuGetServiceRootComponent().GetByPath(path); + if (component == null) + { + return false; + } + + var componentNames = path.Split(Constants.ComponentPathDivider); + var name = string.Join(" ", componentNames.Skip(1).Reverse()); + var boldedPart = $"{name} {boldedPartInnerString} {_componentStatusNames[eventEntity.AffectedComponentStatus].ToLowerInvariant()}."; + + string innerMessage = innerMessageMap + .FirstOrDefault(m => m.Matches(path))? + .InnerMessage; + + if (innerMessage == null) + { + return false; + } + + message = $"{boldedPart} {innerMessage}{messageSuffix}"; + + return !string.IsNullOrEmpty(message); + } + + private class InnerMessageForComponentPathPrefix + { + public string ComponentPathPrefix { get; } + public string InnerMessage { get; } + + public InnerMessageForComponentPathPrefix(string componentPathPrefix, string innerWarning) + { + ComponentPathPrefix = componentPathPrefix; + InnerMessage = innerWarning; + } + + public bool Matches(string componentPath) + { + return componentPath.StartsWith(ComponentPathPrefix, StringComparison.OrdinalIgnoreCase); } } } From b3b296e6a5fade228eaf6475495fe3640e984659 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 18 Jul 2018 14:38:45 -0700 Subject: [PATCH 20/49] check that event has no messages before writing first message --- src/StatusAggregator/MessageUpdater.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 66802665f..a90bf6455 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -21,13 +21,13 @@ public MessageUpdater(ITableWrapper table) public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime nextCreationTime) { // Enough time must have passed before we create a start message for an event. - if (nextCreationTime > eventEntity.StartTime + EventStartDelay) + // Only create a message if the event doesn't have messages associated with it. + if (nextCreationTime > eventEntity.StartTime + EventStartDelay && + !_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any() && + TryGetMessageForEventStartForEvent(eventEntity, out var message)) { - if (TryGetMessageForEventStartForEvent(eventEntity, out var message)) - { - var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, message); - await _table.InsertOrReplaceAsync(messageEntity); - } + var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, message); + await _table.InsertOrReplaceAsync(messageEntity); } } From 1e9ec04ac1e62629d305026fa1d3c3083e4b5484 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 20 Jul 2018 10:39:21 -0700 Subject: [PATCH 21/49] minor fixes --- src/StatusAggregator/IncidentFactory.cs | 7 +++++++ src/StatusAggregator/StatusExporter.cs | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index 3213e04d1..4527a2abd 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -53,6 +53,13 @@ public async Task CreateIncident(ParsedIncident parsedIncident) } Console.WriteLine($"Linking {parsedIncident.Id} to {possibleEvent.RowKey}"); + if ((int)parsedIncident.AffectedComponentStatus > possibleEvent.AffectedComponentStatus) + { + Console.WriteLine($"{parsedIncident.Id} is a more severe than {possibleEvent.RowKey}, upgrading severity of event"); + possibleEvent.AffectedComponentStatus = (int)parsedIncident.AffectedComponentStatus; + await _table.InsertOrReplaceAsync(possibleEvent); + } + incidentEntity.EventRowKey = possibleEvents.First().RowKey; break; } diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 0be6b6415..8dfe39a0b 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -51,7 +51,13 @@ public async Task Export() }) .Where(e => e.Messages != null && e.Messages.Any()); - foreach (var activeEvent in recentEvents.Where(e => e.EndTime == null || e.EndTime >= DateTime.UtcNow)) + // If multiple events are affecting a single region, the event with the highest severity should affect the component. + var activeEvents = recentEvents + .Where(e => e.EndTime == null || e.EndTime >= DateTime.UtcNow) + .GroupBy(e => e.AffectedComponentPath) + .Select(g => g.OrderByDescending(e => e.AffectedComponentStatus).First()); + + foreach (var activeEvent in activeEvents) { var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); From de5f18966c688dfab7c1d09a090b6f3d7ae494bf Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 20 Jul 2018 11:34:08 -0700 Subject: [PATCH 22/49] fix script to mirror variable name change --- src/StatusAggregator/Scripts/StatusAggregator.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StatusAggregator/Scripts/StatusAggregator.cmd b/src/StatusAggregator/Scripts/StatusAggregator.cmd index 1ed312c46..f3754c5c8 100644 --- a/src/StatusAggregator/Scripts/StatusAggregator.cmd +++ b/src/StatusAggregator/Scripts/StatusAggregator.cmd @@ -9,7 +9,7 @@ title #{Jobs.archivepackages.Title} start /w statusaggregator.exe ^ -StatusIncidentApiBaseUri "#{Jobs.statusaggregator.IncidentApiBaseUri}" ^ - -StatusIncidentApiRoutingId "#{Jobs.statusaggregator.IncidentApiRoutingId}" ^ + -StatusIncidentApiTeamId "#{Jobs.statusaggregator.IncidentApiTeamId}" ^ -StatusIncidentApiCertificate "#{Jobs.statusaggregator.IncidentApiCertificate}" ^ -StatusStorageAccount "#{Jobs.statusaggregator.StorageAccount}" ^ -StatusContainerName "#{Jobs.statusaggregator.ContainerName}" ^ From 910b151188b63cd463d5ea8bc99c2f2e79e77276 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 20 Jul 2018 16:42:32 -0700 Subject: [PATCH 23/49] fix OutdatedRegionalSearchServiceInstanceIncidentParser --- ...onalSearchServiceInstanceIncidentParser.cs | 41 +------------------ 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs index 6c32ace25..099ab28b4 100644 --- a/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Incidents/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs @@ -10,7 +10,7 @@ public class OutdatedRegionalSearchServiceInstanceIncidentParser : EnvironmentPr { private const string ServiceEnvironmentGroupName = "SearchEnvironment"; private const string ServiceRegionGroupName = "SearchRegion"; - private static string SubtitleRegEx = $@"Search service 'nuget-(?<{ServiceEnvironmentGroupName}>.*)-(?<{ServiceRegionGroupName}>.*)-search' is using an outdated index!"; + private static string SubtitleRegEx = $@"Search service 'nuget-(?<{ServiceEnvironmentGroupName}>.*)-(?<{ServiceRegionGroupName}>.*)-(v2v3)?search' is using an outdated index!"; private readonly IEnumerable _environments; @@ -32,44 +32,7 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo return false; } - string region; - string subRegion; - - switch (searchRegion) - { - case "0": - case "usnc": - case "ussc": - region = Components.GlobalRegionName; - break; - case "eastasia": - case "southeastasia": - region = Components.ChinaRegionName; - break; - default: - return false; - } - - switch (searchRegion) - { - case "0": - case "usnc": - subRegion = Components.UsncInstanceName; - break; - case "ussc": - subRegion = Components.UsscInstanceName; - break; - case "eastasia": - subRegion = Components.EaInstanceName; - break; - case "southeastasia": - subRegion = Components.SeaInstanceName; - break; - default: - return false; - } - - affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.SearchName, region, subRegion); + affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.UploadName); return true; } From 5fa8140afa28285cb6e175037e0c74f4f7069ec5 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 23 Jul 2018 15:00:43 -0700 Subject: [PATCH 24/49] use shared table classes --- src/StatusAggregator/Cursor.cs | 1 + src/StatusAggregator/EventUpdater.cs | 1 + src/StatusAggregator/IEventUpdater.cs | 1 + src/StatusAggregator/IIncidentFactory.cs | 1 + src/StatusAggregator/IMessageUpdater.cs | 1 + src/StatusAggregator/IncidentFactory.cs | 8 +++- src/StatusAggregator/IncidentUpdater.cs | 3 +- src/StatusAggregator/MessageUpdater.cs | 1 + src/StatusAggregator/StatusAggregator.csproj | 8 ++-- src/StatusAggregator/StatusExporter.cs | 1 + src/StatusAggregator/Table/CursorEntity.cs | 26 ---------- src/StatusAggregator/Table/EventEntity.cs | 45 ----------------- src/StatusAggregator/Table/IncidentEntity.cs | 48 ------------------- src/StatusAggregator/Table/MessageEntity.cs | 37 -------------- src/StatusAggregator/Table/TableUtility.cs | 12 ----- .../Table/TableWrapperExtensions.cs | 3 +- 16 files changed, 21 insertions(+), 176 deletions(-) delete mode 100644 src/StatusAggregator/Table/CursorEntity.cs delete mode 100644 src/StatusAggregator/Table/EventEntity.cs delete mode 100644 src/StatusAggregator/Table/IncidentEntity.cs delete mode 100644 src/StatusAggregator/Table/MessageEntity.cs delete mode 100644 src/StatusAggregator/Table/TableUtility.cs diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index e35af4c14..9f017bd14 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using NuGet.Services.Status.Table; using StatusAggregator.Table; diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index 00ad64671..26c4c5054 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using NuGet.Services.Status.Table; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/IEventUpdater.cs b/src/StatusAggregator/IEventUpdater.cs index b4920121f..54e9bb437 100644 --- a/src/StatusAggregator/IEventUpdater.cs +++ b/src/StatusAggregator/IEventUpdater.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using NuGet.Services.Status.Table; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/IIncidentFactory.cs b/src/StatusAggregator/IIncidentFactory.cs index 77d8b0159..1e7b98b3b 100644 --- a/src/StatusAggregator/IIncidentFactory.cs +++ b/src/StatusAggregator/IIncidentFactory.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using NuGet.Services.Status.Table; using StatusAggregator.Incidents.Parse; using StatusAggregator.Table; diff --git a/src/StatusAggregator/IMessageUpdater.cs b/src/StatusAggregator/IMessageUpdater.cs index 2db6d4c5f..589348749 100644 --- a/src/StatusAggregator/IMessageUpdater.cs +++ b/src/StatusAggregator/IMessageUpdater.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using NuGet.Services.Status.Table; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index 4527a2abd..59c2f2f07 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using NuGet.Services.Status.Table; using StatusAggregator.Incidents.Parse; using StatusAggregator.Table; @@ -21,7 +22,12 @@ public IncidentFactory(ITableWrapper table, IEventUpdater eventUpdater) public async Task CreateIncident(ParsedIncident parsedIncident) { Console.WriteLine($"Attempting to save {parsedIncident.Id}"); - var incidentEntity = new IncidentEntity(parsedIncident); + var incidentEntity = new IncidentEntity( + parsedIncident.Id, + parsedIncident.AffectedComponentPath, + parsedIncident.AffectedComponentStatus, + parsedIncident.CreationTime, + parsedIncident.MitigationTime); // Find an event to attach this incident to var possibleEvents = _table diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index 383579ac1..ad5034a1a 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -1,4 +1,5 @@ -using StatusAggregator.Incidents; +using NuGet.Services.Status.Table; +using StatusAggregator.Incidents; using StatusAggregator.Incidents.Parse; using StatusAggregator.Table; using System; diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index a90bf6455..8a1de5ab2 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using NuGet.Services.Status; +using NuGet.Services.Status.Table; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index ef2ee45b0..3a9d38db7 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -71,25 +71,20 @@ - - - - - @@ -106,6 +101,9 @@ 2.26.0-sb-status-34842 + + 2.28.0-sb-morestatus-35606 + 9.2.0 diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 8dfe39a0b..a504e0303 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -6,6 +6,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Converters; using NuGet.Services.Status; +using NuGet.Services.Status.Table; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/Table/CursorEntity.cs b/src/StatusAggregator/Table/CursorEntity.cs deleted file mode 100644 index d5b229f46..000000000 --- a/src/StatusAggregator/Table/CursorEntity.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using Microsoft.WindowsAzure.Storage.Table; - -namespace StatusAggregator.Table -{ - public class CursorEntity : TableEntity - { - public const string DefaultPartitionKey = "cursors"; - public const string DefaultRowKey = "1"; - - public CursorEntity() - { - } - - public CursorEntity(DateTime value) - : base(DefaultPartitionKey, DefaultRowKey) - { - Value = value; - } - - public DateTime Value { get; set; } - } -} diff --git a/src/StatusAggregator/Table/EventEntity.cs b/src/StatusAggregator/Table/EventEntity.cs deleted file mode 100644 index d6c2ccea6..000000000 --- a/src/StatusAggregator/Table/EventEntity.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.Generic; -using Microsoft.WindowsAzure.Storage.Table; -using NuGet.Services.Status; - -namespace StatusAggregator.Table -{ - public class EventEntity : TableEntity - { - public const string DefaultPartitionKey = "events"; - - public EventEntity() - { - } - - public EventEntity(IncidentEntity incidentEntity) - : base(DefaultPartitionKey, GetRowKey(incidentEntity)) - { - AffectedComponentPath = incidentEntity.AffectedComponentPath; - AffectedComponentStatus = (int)incidentEntity.AffectedComponentStatus; - StartTime = incidentEntity.CreationTime; - incidentEntity.EventRowKey = RowKey; - } - - public string AffectedComponentPath { get; set; } - public int AffectedComponentStatus { get; set; } - public DateTime StartTime { get; set; } - public DateTime? EndTime { get; set; } - public bool IsActive - { - get { return EndTime == null; } - set { } - } - - public Event AsEvent(IEnumerable messages) - { - return new Event(AffectedComponentPath, (ComponentStatus)AffectedComponentStatus, StartTime, EndTime, messages); - } - - private static string GetRowKey(IncidentEntity incidentEntity) - { - return $"{TableUtility.ToRowKeySafeComponentPath(incidentEntity.AffectedComponentPath)}_{incidentEntity.CreationTime.ToString("o")}"; - } - } -} diff --git a/src/StatusAggregator/Table/IncidentEntity.cs b/src/StatusAggregator/Table/IncidentEntity.cs deleted file mode 100644 index 6a839e77f..000000000 --- a/src/StatusAggregator/Table/IncidentEntity.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using Microsoft.WindowsAzure.Storage.Table; -using NuGet.Services.Status; -using StatusAggregator.Incidents.Parse; - -namespace StatusAggregator.Table -{ - public class IncidentEntity : TableEntity - { - public const string DefaultPartitionKey = "incidents"; - - public IncidentEntity() - { - } - - public IncidentEntity(ParsedIncident parsedIncident) - : base(DefaultPartitionKey, GetRowKey(parsedIncident)) - { - IncidentApiId = parsedIncident.Id; - AffectedComponentPath = parsedIncident.AffectedComponentPath; - AffectedComponentStatus = (int)parsedIncident.AffectedComponentStatus; - CreationTime = parsedIncident.CreationTime; - MitigationTime = parsedIncident.MitigationTime; - } - - public string EventRowKey { get; set; } - public bool IsLinkedToEvent - { - get { return !string.IsNullOrEmpty(EventRowKey); } - set { } - } - public string IncidentApiId { get; set; } - public string AffectedComponentPath { get; set; } - public int AffectedComponentStatus { get; set; } - public DateTime CreationTime { get; set; } - public DateTime? MitigationTime { get; set; } - public bool IsActive - { - get { return MitigationTime == null; } - set { } - } - - private static string GetRowKey(ParsedIncident parsedIncident) - { - return $"{parsedIncident.Id}_{TableUtility.ToRowKeySafeComponentPath(parsedIncident.AffectedComponentPath)}_{parsedIncident.AffectedComponentStatus}"; - } - } -} diff --git a/src/StatusAggregator/Table/MessageEntity.cs b/src/StatusAggregator/Table/MessageEntity.cs deleted file mode 100644 index 8913d9c7d..000000000 --- a/src/StatusAggregator/Table/MessageEntity.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using Microsoft.WindowsAzure.Storage.Table; -using NuGet.Services.Status; - -namespace StatusAggregator.Table -{ - public class MessageEntity : TableEntity - { - public const string DefaultPartitionKey = "messages"; - - public MessageEntity() - { - } - - public MessageEntity(EventEntity eventEntity, DateTime time, string contents) - : base(DefaultPartitionKey, GetRowKey(eventEntity, time)) - { - EventRowKey = eventEntity.RowKey; - Time = time; - Contents = contents; - } - - public string EventRowKey { get; set; } - public DateTime Time { get; set; } - public string Contents { get; set; } - - public Message AsMessage() - { - return new Message(Time, Contents); - } - - private static string GetRowKey(EventEntity eventEntity, DateTime time) - { - return $"{eventEntity.RowKey}_{time.ToString("o")}"; - } - } -} diff --git a/src/StatusAggregator/Table/TableUtility.cs b/src/StatusAggregator/Table/TableUtility.cs deleted file mode 100644 index a50bd3b72..000000000 --- a/src/StatusAggregator/Table/TableUtility.cs +++ /dev/null @@ -1,12 +0,0 @@ -using NuGet.Services.Status; - -namespace StatusAggregator.Table -{ - public static class TableUtility - { - public static string ToRowKeySafeComponentPath(string componentPath) - { - return componentPath.Replace(Constants.ComponentPathDivider, '_'); - } - } -} diff --git a/src/StatusAggregator/Table/TableWrapperExtensions.cs b/src/StatusAggregator/Table/TableWrapperExtensions.cs index 887f5ebd5..cfbc0c239 100644 --- a/src/StatusAggregator/Table/TableWrapperExtensions.cs +++ b/src/StatusAggregator/Table/TableWrapperExtensions.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using NuGet.Services.Status.Table; +using System.Collections.Generic; using System.Linq; namespace StatusAggregator.Table From f647d8260eef36f742b6b94c8dc3a4bd8f733fe1 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 23 Jul 2018 15:11:07 -0700 Subject: [PATCH 25/49] increase nuget.services.status reference --- src/StatusAggregator/StatusAggregator.csproj | 2 +- tests/StatusAggregator.Tests/EventUpdaterTests.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 3a9d38db7..c003cdddc 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -99,7 +99,7 @@ - 2.26.0-sb-status-34842 + 2.28.0-sb-morestatus-35606 2.28.0-sb-morestatus-35606 diff --git a/tests/StatusAggregator.Tests/EventUpdaterTests.cs b/tests/StatusAggregator.Tests/EventUpdaterTests.cs index 7ba29bccb..2c86a5645 100644 --- a/tests/StatusAggregator.Tests/EventUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/EventUpdaterTests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Moq; +using NuGet.Services.Status.Table; using StatusAggregator.Table; using Xunit; From 5328acf61c3a8d63a0ebbd5e971e125f3652f850 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Tue, 24 Jul 2018 10:46:25 -0700 Subject: [PATCH 26/49] use shared incident api library --- src/StatusAggregator/Cursor.cs | 13 +- src/StatusAggregator/ICursor.cs | 5 +- src/StatusAggregator/IIncidentFactory.cs | 2 +- src/StatusAggregator/IncidentFactory.cs | 2 +- src/StatusAggregator/IncidentUpdater.cs | 35 ++++-- .../Incidents/IIncidentCollector.cs | 13 -- src/StatusAggregator/Incidents/Incident.cs | 30 ----- .../Incidents/IncidentCollector.cs | 112 ------------------ .../Incidents/IncidentList.cs | 15 --- .../Incidents/IncidentStatus.cs | 17 --- src/StatusAggregator/Job.cs | 9 +- .../Parse/AggregateIncidentParser.cs | 4 +- .../Parse/EnvironmentFilter.cs | 5 +- .../Parse/EnvironmentPrefixIncidentParser.cs | 2 +- .../Parse/IAggregateIncidentParser.cs | 5 +- .../{Incidents => }/Parse/IIncidentParser.cs | 4 +- .../Parse/IIncidentParsingFilter.cs | 5 +- .../{Incidents => }/Parse/IncidentParser.cs | 3 +- ...onalSearchServiceInstanceIncidentParser.cs | 3 +- ...atedSearchServiceInstanceIncidentParser.cs | 5 +- .../{Incidents => }/Parse/ParsedIncident.cs | 5 +- .../Parse/PingdomIncidentParser.cs | 3 +- .../{Incidents => }/Parse/SeverityFilter.cs | 5 +- .../Parse/ValidationDurationIncidentParser.cs | 3 +- src/StatusAggregator/StatusAggregator.csproj | 34 +++--- src/StatusAggregator/StatusUpdater.cs | 6 +- src/StatusAggregator/Table/ITableWrapper.cs | 3 + src/StatusAggregator/Table/TableWrapper.cs | 7 ++ .../IncidentFactoryTests.cs | 5 +- .../StatusAggregator.Tests.csproj | 9 ++ 30 files changed, 110 insertions(+), 259 deletions(-) delete mode 100644 src/StatusAggregator/Incidents/IIncidentCollector.cs delete mode 100644 src/StatusAggregator/Incidents/Incident.cs delete mode 100644 src/StatusAggregator/Incidents/IncidentCollector.cs delete mode 100644 src/StatusAggregator/Incidents/IncidentList.cs delete mode 100644 src/StatusAggregator/Incidents/IncidentStatus.cs rename src/StatusAggregator/{Incidents => }/Parse/AggregateIncidentParser.cs (92%) rename src/StatusAggregator/{Incidents => }/Parse/EnvironmentFilter.cs (89%) rename src/StatusAggregator/{Incidents => }/Parse/EnvironmentPrefixIncidentParser.cs (93%) rename src/StatusAggregator/{Incidents => }/Parse/IAggregateIncidentParser.cs (56%) rename src/StatusAggregator/{Incidents => }/Parse/IIncidentParser.cs (82%) rename src/StatusAggregator/{Incidents => }/Parse/IIncidentParsingFilter.cs (55%) rename src/StatusAggregator/{Incidents => }/Parse/IncidentParser.cs (97%) rename src/StatusAggregator/{Incidents => }/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs (96%) rename src/StatusAggregator/{Incidents => }/Parse/OutdatedSearchServiceInstanceIncidentParser.cs (91%) rename src/StatusAggregator/{Incidents => }/Parse/ParsedIncident.cs (88%) rename src/StatusAggregator/{Incidents => }/Parse/PingdomIncidentParser.cs (98%) rename src/StatusAggregator/{Incidents => }/Parse/SeverityFilter.cs (78%) rename src/StatusAggregator/{Incidents => }/Parse/ValidationDurationIncidentParser.cs (94%) diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index 9f017bd14..23b4a56fc 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -1,5 +1,4 @@ using System; -using System.Linq; using System.Threading.Tasks; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -16,15 +15,13 @@ public Cursor(ITableWrapper table) private ITableWrapper _table; - public DateTime Get() + public async Task Get() { - var cursors = _table - .CreateQuery() - .Where(e => e.PartitionKey == CursorEntity.DefaultPartitionKey) - .ToList(); + var cursor = await _table.Retrieve( + CursorEntity.DefaultPartitionKey, CursorEntity.DefaultRowKey); - return cursors.Any() - ? cursors.Max(c => c.Value) + return cursor != null + ? cursor.Value : DateTime.MinValue; } diff --git a/src/StatusAggregator/ICursor.cs b/src/StatusAggregator/ICursor.cs index 474c7a516..e059935e1 100644 --- a/src/StatusAggregator/ICursor.cs +++ b/src/StatusAggregator/ICursor.cs @@ -1,14 +1,11 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading.Tasks; namespace StatusAggregator { public interface ICursor { - DateTime Get(); + Task Get(); Task Set(DateTime value); } } diff --git a/src/StatusAggregator/IIncidentFactory.cs b/src/StatusAggregator/IIncidentFactory.cs index 1e7b98b3b..ed53e0d94 100644 --- a/src/StatusAggregator/IIncidentFactory.cs +++ b/src/StatusAggregator/IIncidentFactory.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; using NuGet.Services.Status.Table; -using StatusAggregator.Incidents.Parse; +using StatusAggregator.Parse; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index 59c2f2f07..7ea672ce3 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -2,7 +2,7 @@ using System.Linq; using System.Threading.Tasks; using NuGet.Services.Status.Table; -using StatusAggregator.Incidents.Parse; +using StatusAggregator.Parse; using StatusAggregator.Table; namespace StatusAggregator diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index ad5034a1a..a62afc349 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -1,6 +1,6 @@ -using NuGet.Services.Status.Table; -using StatusAggregator.Incidents; -using StatusAggregator.Incidents.Parse; +using NuGet.Services.Incidents; +using NuGet.Services.Status.Table; +using StatusAggregator.Parse; using StatusAggregator.Table; using System; using System.Linq; @@ -15,21 +15,25 @@ public class IncidentUpdater : IIncidentUpdater private readonly IEventUpdater _eventUpdater; private readonly IAggregateIncidentParser _aggregateIncidentParser; - private readonly IIncidentCollector _incidentCollector; + private readonly IIncidentApiClient _incidentClient; private readonly IIncidentFactory _incidentFactory; + private readonly string _incidentTeamId; + public IncidentUpdater( ITableWrapper table, IEventUpdater eventUpdater, - IIncidentCollector incidentCollector, + IIncidentApiClient incidentClient, IAggregateIncidentParser aggregateIncidentParser, - IIncidentFactory incidentFactory) + IIncidentFactory incidentFactory, + string incidentTeamId) { _table = table; _eventUpdater = eventUpdater; - _incidentCollector = incidentCollector; + _incidentClient = incidentClient; _aggregateIncidentParser = aggregateIncidentParser; _incidentFactory = incidentFactory; + _incidentTeamId = incidentTeamId; } public async Task RefreshExistingIncidents() @@ -40,7 +44,7 @@ public async Task RefreshExistingIncidents() foreach (var activeIncidentEntity in activeIncidentEntities) { - var activeIncident = await _incidentCollector.GetIncident(activeIncidentEntity.IncidentApiId); + var activeIncident = await _incidentClient.GetIncident(activeIncidentEntity.IncidentApiId); activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; await _table.InsertOrReplaceAsync(activeIncidentEntity); } @@ -48,7 +52,11 @@ public async Task RefreshExistingIncidents() public async Task FetchNewIncidents(DateTime cursor) { - var incidents = await _incidentCollector.GetRecentIncidents(cursor); + var incidents = (await _incidentClient.GetIncidents(GetRecentIncidentsQuery(cursor))) + // The incident API trims the milliseconds from any filter. + // Therefore, a query asking for incidents newer than '2018-06-29T00:00:00.5Z' will return an incident from '2018-06-29T00:00:00.25Z' + // We must perform a check on the CreateDate ourselves to verify that no old incidents are returned. + .Where(i => i.CreateDate > cursor); var parsedIncidents = incidents.SelectMany(i => _aggregateIncidentParser.ParseIncident(i)); foreach (var parsedIncident in parsedIncidents.OrderBy(i => i.CreationTime)) @@ -58,6 +66,15 @@ public async Task RefreshExistingIncidents() return incidents.Any() ? incidents.Max(i => i.CreateDate) : (DateTime?)null; } + + private string GetRecentIncidentsQuery(DateTime cursor) + { + var cursorPart = cursor == DateTime.MinValue + ? "" + : $" and CreateDate gt datetime'{cursor.ToString("o")}'"; + + return $"$filter=OwningTeamId eq '{_incidentTeamId}'{cursorPart}"; + } public async Task UpdateActiveEvents(DateTime cursor) { diff --git a/src/StatusAggregator/Incidents/IIncidentCollector.cs b/src/StatusAggregator/Incidents/IIncidentCollector.cs deleted file mode 100644 index ca66d0a25..000000000 --- a/src/StatusAggregator/Incidents/IIncidentCollector.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace StatusAggregator.Incidents -{ - public interface IIncidentCollector - { - Task GetIncident(string incidentApiId); - - Task> GetRecentIncidents(DateTime since); - } -} diff --git a/src/StatusAggregator/Incidents/Incident.cs b/src/StatusAggregator/Incidents/Incident.cs deleted file mode 100644 index fe2003121..000000000 --- a/src/StatusAggregator/Incidents/Incident.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; - -namespace StatusAggregator.Incidents -{ - public class Incident - { - public string Id { get; set; } - public int Severity { get; set; } - public IncidentStatus Status { get; set; } - public DateTime CreateDate { get; set; } - public string Title { get; set; } - public string OwningTeamId { get; set; } - public IncidentSourceData Source { get; set; } - public IncidentStateChangeEventData MitigationData { get; set; } - public IncidentStateChangeEventData ResolutionData { get; set; } - } - - public class IncidentSourceData - { - public DateTime CreateDate { get; set; } - } - - public class IncidentStateChangeEventData - { - public DateTime Date { get; set; } - } -} diff --git a/src/StatusAggregator/Incidents/IncidentCollector.cs b/src/StatusAggregator/Incidents/IncidentCollector.cs deleted file mode 100644 index 7e936dd6f..000000000 --- a/src/StatusAggregator/Incidents/IncidentCollector.cs +++ /dev/null @@ -1,112 +0,0 @@ -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Security.Cryptography.X509Certificates; -using System.Threading.Tasks; - -namespace StatusAggregator.Incidents -{ - public class IncidentCollector : IIncidentCollector - { - - private const string IncidentApiIncidentsEndpoint = "incidents"; - private static readonly string IncidentApiIndividualIncidentQueryFormatString = $"{IncidentApiIncidentsEndpoint}({{0}})"; - private static readonly string IncidentApiIncidentListQueryFormatString = $"{IncidentApiIncidentsEndpoint}?{{0}}"; - - private static readonly JsonSerializerSettings _incidentApiJsonSerializerSettings = - new JsonSerializerSettings() { DateTimeZoneHandling = DateTimeZoneHandling.Utc }; - - public IncidentCollector( - Uri incidentApiBaseUri, - X509Certificate2 incidentApiCertificate, - string incidentApiTeamId) - { - _incidentApiBaseUri = incidentApiBaseUri; - _incidentApiCertificate = incidentApiCertificate; - _incidentApiTeamId = incidentApiTeamId; - } - - public Task GetIncident(string id) - { - return GetIncidentApiResponse(GetIncidentApiGetIncidentQuery(id)); - } - - public async Task> GetRecentIncidents(DateTime since) - { - var incidents = new List(); - - string query = since == DateTime.MinValue - ? GetIncidentApiIncidentListAllIncidentsQuery() - : GetIncidentApiIncidentListRecentIncidentsQuery(since); - var nextLink = GetIncidentApiUri(GetIncidentApiIncidentList(query)); - do - { - var incidentList = await GetIncidentApiResponse(nextLink); - foreach (var incident in incidentList.Incidents) - { - // The incident API trims the milliseconds from any filter. - // Therefore, a query asking for incidents newer than '2018-06-29T00:00:00.5Z' will return an incident from '2018-06-29T00:00:00.25Z' - // We must perform a check on the CreateDate ourselves to verify that no old incidents are returned. - if (incident.CreateDate <= since) - { - continue; - } - - incidents.Add(incident); - } - - nextLink = incidentList.NextLink; - } while (nextLink != null); - - return incidents; - } - - private readonly Uri _incidentApiBaseUri; - private readonly X509Certificate2 _incidentApiCertificate; - private readonly string _incidentApiTeamId; - - private string GetIncidentApiIncidentList(string oDataQueryParameters) - { - return string.Format(IncidentApiIncidentListQueryFormatString, oDataQueryParameters); - } - - private string GetIncidentApiGetIncidentQuery(string id) - { - return string.Format(IncidentApiIndividualIncidentQueryFormatString, id); - } - - private string GetIncidentApiIncidentListAllIncidentsQuery() - { - return $"$filter=OwningTeamId eq '{_incidentApiTeamId}'"; - } - - private string GetIncidentApiIncidentListRecentIncidentsQuery(DateTime cursor) - { - return $"$filter=OwningTeamId eq '{_incidentApiTeamId}' and CreateDate gt datetime'{cursor.ToString("o")}'"; - } - - private Uri GetIncidentApiUri(string query) - { - return new Uri(_incidentApiBaseUri, query); - } - - private Task GetIncidentApiResponse(string query) - { - return GetIncidentApiResponse(GetIncidentApiUri(query)); - } - - private async Task GetIncidentApiResponse(Uri uri) - { - var request = WebRequest.CreateHttp(uri); - request.ClientCertificates.Add(_incidentApiCertificate); - var response = await request.GetResponseAsync(); - using (var reader = new StreamReader(response.GetResponseStream())) - { - var content = await reader.ReadToEndAsync(); - return JsonConvert.DeserializeObject(content, _incidentApiJsonSerializerSettings); - } - } - } -} diff --git a/src/StatusAggregator/Incidents/IncidentList.cs b/src/StatusAggregator/Incidents/IncidentList.cs deleted file mode 100644 index 64250a511..000000000 --- a/src/StatusAggregator/Incidents/IncidentList.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using Newtonsoft.Json; - -namespace StatusAggregator.Incidents -{ - public class IncidentList - { - [JsonProperty(PropertyName = "value")] - public IEnumerable Incidents { get; set; } - - [JsonProperty(PropertyName = "odata.nextLink")] - public Uri NextLink { get; set; } - } -} diff --git a/src/StatusAggregator/Incidents/IncidentStatus.cs b/src/StatusAggregator/Incidents/IncidentStatus.cs deleted file mode 100644 index db7824b0b..000000000 --- a/src/StatusAggregator/Incidents/IncidentStatus.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace StatusAggregator.Incidents -{ - public enum IncidentStatus - { - Holding, - Active, - Mitigated, - Resolved, - Suppressed, - New, - Correlating, - Mitigating - } -} diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index f4699864b..6d8e1737a 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -10,8 +10,8 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using StatusAggregator.Incidents; -using StatusAggregator.Incidents.Parse; +using NuGet.Services.Incidents; +using StatusAggregator.Parse; using StatusAggregator.Table; namespace StatusAggregator @@ -42,12 +42,13 @@ public override void Init(IServiceContainer serviceContainer, IDictionary - - - - + + + + - - - - + + - - - - - - + + + - - - + + + - + @@ -98,6 +93,9 @@ + + 2.28.0-sb-incidents-35615 + 2.28.0-sb-morestatus-35606 diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs index 62b77b122..adb984a1a 100644 --- a/src/StatusAggregator/StatusUpdater.cs +++ b/src/StatusAggregator/StatusUpdater.cs @@ -1,9 +1,5 @@ using System; -using System.Linq; using System.Threading.Tasks; -using StatusAggregator.Incidents; -using StatusAggregator.Incidents.Parse; -using StatusAggregator.Table; namespace StatusAggregator { @@ -23,7 +19,7 @@ public StatusUpdater( public async Task Update() { - var lastCursor = _cursor.Get(); + var lastCursor = await _cursor.Get(); await _incidentUpdater.RefreshExistingIncidents(); var nextCursor = await _incidentUpdater.FetchNewIncidents(lastCursor); diff --git a/src/StatusAggregator/Table/ITableWrapper.cs b/src/StatusAggregator/Table/ITableWrapper.cs index 7ffe369c2..d319ca1db 100644 --- a/src/StatusAggregator/Table/ITableWrapper.cs +++ b/src/StatusAggregator/Table/ITableWrapper.cs @@ -9,6 +9,9 @@ public interface ITableWrapper { Task CreateIfNotExistsAsync(); + Task Retrieve(string partitionKey, string rowKey) + where T : class, ITableEntity; + Task InsertOrReplaceAsync(ITableEntity tableEntity); IQueryable CreateQuery() where T : ITableEntity, new(); diff --git a/src/StatusAggregator/Table/TableWrapper.cs b/src/StatusAggregator/Table/TableWrapper.cs index 951b37254..ba4c86c9a 100644 --- a/src/StatusAggregator/Table/TableWrapper.cs +++ b/src/StatusAggregator/Table/TableWrapper.cs @@ -21,6 +21,13 @@ public Task CreateIfNotExistsAsync() return _table.CreateIfNotExistsAsync(); } + public async Task Retrieve(string partitionKey, string rowKey) + where T : class, ITableEntity + { + var operation = TableOperation.Retrieve(partitionKey, rowKey); + return (await _table.ExecuteAsync(operation)).Result as T; + } + public Task InsertOrReplaceAsync(ITableEntity tableEntity) { var operation = TableOperation.InsertOrReplace(tableEntity); diff --git a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs index 5fde76df1..01eb2a253 100644 --- a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs +++ b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs @@ -3,9 +3,10 @@ using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage.Table; using Moq; +using NuGet.Services.Incidents; using NuGet.Services.Status; -using StatusAggregator.Incidents; -using StatusAggregator.Incidents.Parse; +using NuGet.Services.Status.Table; +using StatusAggregator.Parse; using StatusAggregator.Table; using Xunit; diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj index efb72f93d..c87ceb60a 100644 --- a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj +++ b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj @@ -62,6 +62,15 @@ 4.7.145 + + 2.28.0-sb-incidents-35615 + + + 2.28.0-sb-morestatus-35606 + + + 2.28.0-sb-morestatus-35606 + 4.3.0 From a884640bffbd80d6900ae0ee35a9118e714300fc Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Tue, 24 Jul 2018 11:38:09 -0700 Subject: [PATCH 27/49] increase to correct version of incident api --- src/StatusAggregator/StatusAggregator.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index eeb9e9fb9..172242aac 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -94,7 +94,7 @@ - 2.28.0-sb-incidents-35615 + 2.28.0-sb-incidents-35734 2.28.0-sb-morestatus-35606 From f8e32405f38f3b56f8c2a4051d8bf22497f3e051 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 10:31:13 -0700 Subject: [PATCH 28/49] logging, comments, di --- src/ArchivePackages/ArchivePackages.Job.cs | 3 +- src/Gallery.CredentialExpiration/Job.cs | 3 +- src/Gallery.Maintenance/Job.cs | 3 +- .../Configuration/JobConfigurationManager.cs | 3 +- .../Extensions/LoggerExtensions.cs | 57 ++++++++ .../Extensions/ServiceProviderExtensions.cs | 12 ++ .../NuGet.Jobs.Common.csproj | 2 + ...upportRequestsNotificationScheduledTask.cs | 3 +- src/Search.GenerateAuxiliaryData/Job.cs | 3 +- .../Job.cs | 3 +- .../Job.cs | 3 +- src/Stats.ImportAzureCdnStatistics/Job.cs | 3 +- .../RefreshClientDimensionJob.cs | 3 +- src/Stats.RollUpDownloadFacts/Job.cs | 3 +- .../{Components.cs => ComponentFactory.cs} | 5 +- src/StatusAggregator/Cursor.cs | 26 +++- src/StatusAggregator/EventUpdater.cs | 96 ++++++++----- src/StatusAggregator/ICursor.cs | 3 + src/StatusAggregator/IEventUpdater.cs | 18 ++- src/StatusAggregator/IIncidentFactory.cs | 7 +- src/StatusAggregator/IIncidentUpdater.cs | 17 ++- src/StatusAggregator/IMessageUpdater.cs | 13 +- src/StatusAggregator/IStatusExporter.cs | 6 +- src/StatusAggregator/IStatusUpdater.cs | 6 +- src/StatusAggregator/IncidentFactory.cs | 106 ++++++++------ src/StatusAggregator/IncidentUpdater.cs | 90 +++++++----- src/StatusAggregator/Job.cs | 128 ++++++++++------- src/StatusAggregator/MessageUpdater.cs | 135 ++++++++++++------ .../Parse/AggregateIncidentParser.cs | 32 +++-- .../Parse/EnvironmentFilter.cs | 35 ++++- .../Parse/EnvironmentPrefixIncidentParser.cs | 19 ++- .../Parse/IAggregateIncidentParser.cs | 6 + src/StatusAggregator/Parse/IIncidentParser.cs | 13 ++ .../Parse/IIncidentParsingFilter.cs | 6 + src/StatusAggregator/Parse/IncidentParser.cs | 73 ++++++++-- ...onalSearchServiceInstanceIncidentParser.cs | 46 ------ ...atedSearchServiceInstanceIncidentParser.cs | 11 +- src/StatusAggregator/Parse/ParsedIncident.cs | 3 + .../Parse/PingdomIncidentParser.cs | 35 +++-- src/StatusAggregator/Parse/SeverityFilter.cs | 23 ++- .../Parse/ValidationDurationIncidentParser.cs | 9 +- src/StatusAggregator/StatusAggregator.cs | 36 +++++ src/StatusAggregator/StatusAggregator.csproj | 9 +- .../StatusAggregatorConfiguration.cs | 44 ++++++ .../StatusContractResolver.cs | 6 + src/StatusAggregator/StatusExporter.cs | 100 ++++++++----- src/StatusAggregator/StatusUpdater.cs | 37 +++-- src/StatusAggregator/Table/TableWrapper.cs | 7 +- .../UpdateLicenseReports.Job.cs | 3 +- .../EventUpdaterTests.cs | 26 ++-- .../IncidentFactoryTests.cs | 9 +- .../StatusAggregator.Tests.csproj | 3 +- 52 files changed, 954 insertions(+), 397 deletions(-) create mode 100644 src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs create mode 100644 src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs rename src/StatusAggregator/{Components.cs => ComponentFactory.cs} (96%) delete mode 100644 src/StatusAggregator/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs create mode 100644 src/StatusAggregator/StatusAggregator.cs create mode 100644 src/StatusAggregator/StatusAggregatorConfiguration.cs diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 97f7528e6..b8c53c999 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -12,6 +12,7 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; @@ -66,7 +67,7 @@ public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index 4290dba1b..f0a72d645 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -17,6 +17,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; using NuGet.Services.Storage; @@ -47,7 +48,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); _galleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); diff --git a/src/Gallery.Maintenance/Job.cs b/src/Gallery.Maintenance/Job.cs index 488a49b92..10c6ba235 100644 --- a/src/Gallery.Maintenance/Job.cs +++ b/src/Gallery.Maintenance/Job.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; @@ -24,7 +25,7 @@ public class Job : JobBase public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); diff --git a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs index 808cb42ea..998635cd5 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs @@ -6,6 +6,7 @@ using System.ComponentModel.Design; using System.Linq; using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.Configuration; using NuGet.Services.KeyVault; @@ -197,7 +198,7 @@ private static Dictionary ReadCommandLineArguments(ILogger logge private static IDictionary InjectSecrets(IServiceContainer serviceContainer, Dictionary argsDictionary) { - var secretReaderFactory = (ISecretReaderFactory)serviceContainer.GetService(typeof(ISecretReaderFactory)); + var secretReaderFactory = serviceContainer.GetService(); var secretReader = secretReaderFactory.CreateSecretReader(argsDictionary); if (secretReader == null) diff --git a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs new file mode 100644 index 000000000..0b3d0e911 --- /dev/null +++ b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Logging; +using System; + +namespace NuGet.Jobs.Extensions +{ + public static class LoggerExtensions + { + public static IDisposable Scope( + this ILogger logger, + string beginMessage, + string finishMessage, + string scopeMessage, + params object[] scopeMessageArgs) + { + return new LoggerScopeHelper( + logger, + beginMessage, + finishMessage, + scopeMessage, + scopeMessageArgs); + } + + private class LoggerScopeHelper : IDisposable + { + private readonly ILogger _logger; + private readonly IDisposable _scope; + + private readonly string _finishMessage; + + private bool _isDisposed = false; + + public LoggerScopeHelper( + ILogger logger, + string beginMessage, + string finishMessage, + string scopeMessage, + object[] scopeMessageArgs) + { + _logger = logger; + _scope = logger.BeginScope(scopeMessage, scopeMessageArgs); + _finishMessage = finishMessage; + + _logger.LogInformation(beginMessage); + } + + public void Dispose() + { + if (!_isDisposed) + { + _logger.LogInformation(_finishMessage); + _scope.Dispose(); + _isDisposed = true; + } + } + } + } +} diff --git a/src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs new file mode 100644 index 000000000..ee3a37f70 --- /dev/null +++ b/src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs @@ -0,0 +1,12 @@ +using System; + +namespace NuGet.Jobs.Extensions +{ + public static class ServiceProviderExtensions + { + public static T GetService(this IServiceProvider provider) + { + return (T)provider.GetService(typeof(T)); + } + } +} diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 4e1ca11ca..923eaa7aa 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -49,6 +49,8 @@ + + diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs index 6976ed0bb..187bc7ddd 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs @@ -6,6 +6,7 @@ using System.ComponentModel.Design; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Notifications; @@ -39,7 +40,7 @@ protected SupportRequestsNotificationScheduledTask( var smtpUri = jobArgsDictionary[JobArgumentNames.SmtpUri]; _messagingService = new MessagingService(loggerFactory, smtpUri); - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var supportDbConnectionString = jobArgsDictionary[JobArgumentNames.SourceDatabase]; var supportDbConnectionFactory = new AzureSqlConnectionFactory(supportDbConnectionString, secretInjector); diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 4d0971f80..a80fe3524 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -11,6 +11,7 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; @@ -45,7 +46,7 @@ internal class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs index 976b2e208..1278f6f0e 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; using IPackageIdGroup = System.Linq.IGrouping; @@ -62,7 +63,7 @@ GROUP BY Stats.[PackageRegistrationKey] public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs index 611390958..aea267cb2 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs @@ -10,6 +10,7 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; using Stopwatch = System.Diagnostics.Stopwatch; @@ -50,7 +51,7 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs index 6b0b18119..5faae61bf 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/Job.cs @@ -10,6 +10,7 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; @@ -30,7 +31,7 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index a10b412c3..7f5ccbd68 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; using Stats.ImportAzureCdnStatistics; @@ -22,7 +23,7 @@ public class RefreshClientDimensionJob : JobBase public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/Job.cs index 76cd61979..38fb75370 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/Job.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; @@ -25,7 +26,7 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); diff --git a/src/StatusAggregator/Components.cs b/src/StatusAggregator/ComponentFactory.cs similarity index 96% rename from src/StatusAggregator/Components.cs rename to src/StatusAggregator/ComponentFactory.cs index f32d70fbc..1b2333f5e 100644 --- a/src/StatusAggregator/Components.cs +++ b/src/StatusAggregator/ComponentFactory.cs @@ -2,7 +2,7 @@ namespace StatusAggregator { - public static class Components + public static class ComponentFactory { public const string RootName = "NuGet"; public const string GalleryName = "NuGet.org"; @@ -21,6 +21,9 @@ public static class Components public const string EaInstanceName = "East Asia"; public const string SeaInstanceName = "Southeast Asia"; + ///

+ /// Creates the NuGet service root component. + /// public static IComponent CreateNuGetServiceRootComponent() { return new TreeComponent( diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index 23b4a56fc..c7a213819 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -8,21 +9,36 @@ namespace StatusAggregator { public class Cursor : ICursor { - public Cursor(ITableWrapper table) + public Cursor( + ITableWrapper table, + ILogger logger) { _table = table; + _logger = logger; } - private ITableWrapper _table; + private readonly ITableWrapper _table; + + private readonly ILogger _logger; public async Task Get() { var cursor = await _table.Retrieve( CursorEntity.DefaultPartitionKey, CursorEntity.DefaultRowKey); - return cursor != null - ? cursor.Value - : DateTime.MinValue; + DateTime value; + if (cursor == null) + { + value = DateTime.MinValue; + _logger.LogInformation("Could not fetch cursor."); + } + else + { + value = cursor.Value; + _logger.LogInformation("Fetched cursor with value {Cursor}.", value); + } + + return value; } public Task Set(DateTime value) diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index 26c4c5054..a23755b47 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -1,6 +1,8 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -13,55 +15,83 @@ public class EventUpdater : IEventUpdater private readonly ITableWrapper _table; private readonly IMessageUpdater _messageUpdater; - public EventUpdater(ITableWrapper table, IMessageUpdater messageUpdater) + private readonly ILogger _logger; + + public EventUpdater( + ITableWrapper table, + IMessageUpdater messageUpdater, + ILogger logger) { _table = table; _messageUpdater = messageUpdater; + _logger = logger; } - public async Task UpdateEvent(EventEntity eventEntity, DateTime nextCreationTime) + public async Task UpdateAllActiveEvents(DateTime cursor) { - eventEntity = eventEntity ?? throw new ArgumentNullException(nameof(eventEntity)); - - if (!eventEntity.IsActive) + using (_logger.Scope( + "Beginning to update all active events.", + "Finished updating all active events.", + "Updating all active events.")) { - // Inactive events cannot be updated. - return false; + var activeEvents = _table.GetActiveEvents(); + foreach (var activeEvent in activeEvents) + { + await UpdateEvent(activeEvent, cursor); + } } + } - var incidentsLinkedToEventToCloseQuery = _table.GetIncidentsLinkedToEvent(eventEntity); + public async Task UpdateEvent(EventEntity eventEntity, DateTime cursor) + { + eventEntity = eventEntity ?? throw new ArgumentNullException(nameof(eventEntity)); - var incidentsLinkedToEventToClose = incidentsLinkedToEventToCloseQuery.ToList(); - if (!incidentsLinkedToEventToClose.Any()) + using (_logger.Scope( + "Beginning to update event.", + "Finished updating event.", + "Updating event '{EventRowKey}' given cursor {Cursor}.", eventEntity.RowKey, cursor)) { - // If an event has no linked incidents it must have been created manually and should not be closed automatically. - return false; - } + if (!eventEntity.IsActive) + { + _logger.LogInformation("Event is inactive, cannot update."); + return false; + } - var shouldClose = !incidentsLinkedToEventToCloseQuery - .Where(i => i.IsActive || i.MitigationTime > nextCreationTime - EventEndDelay) - .ToList() - .Any(); + var incidentsLinkedToEventQuery = _table.GetIncidentsLinkedToEvent(eventEntity); - if (shouldClose) - { - Console.WriteLine($"Closing {eventEntity.RowKey} because its incidents are inactive and too old"); - var mitigationTime = incidentsLinkedToEventToClose - .Max(i => i.MitigationTime ?? DateTime.MinValue); - eventEntity.EndTime = mitigationTime; + var incidentsLinkedToEvent = incidentsLinkedToEventQuery.ToList(); + if (!incidentsLinkedToEvent.Any()) + { + _logger.LogInformation("Event has no linked incidents and must have been created manually, cannot update."); + return false; + } - await _messageUpdater.CreateMessageForEventStart(eventEntity, mitigationTime); - await _messageUpdater.CreateMessageForEventEnd(eventEntity); + var shouldDeactivate = !incidentsLinkedToEventQuery + .Where(i => i.IsActive || i.MitigationTime > cursor - EventEndDelay) + .ToList() + .Any(); - // Update the event - await _table.InsertOrReplaceAsync(eventEntity); - } - else - { - await _messageUpdater.CreateMessageForEventStart(eventEntity, nextCreationTime); - } + if (shouldDeactivate) + { + _logger.LogInformation("Deactivating event because its incidents are inactive and too old."); + var mitigationTime = incidentsLinkedToEvent + .Max(i => i.MitigationTime ?? DateTime.MinValue); + eventEntity.EndTime = mitigationTime; - return shouldClose; + await _messageUpdater.CreateMessageForEventStart(eventEntity, mitigationTime); + await _messageUpdater.CreateMessageForEventEnd(eventEntity); + + // Update the event + await _table.InsertOrReplaceAsync(eventEntity); + } + else + { + _logger.LogInformation("Event has active or recent incidents so it will not be deactivated."); + await _messageUpdater.CreateMessageForEventStart(eventEntity, cursor); + } + + return shouldDeactivate; + } } } } diff --git a/src/StatusAggregator/ICursor.cs b/src/StatusAggregator/ICursor.cs index e059935e1..77e958e8b 100644 --- a/src/StatusAggregator/ICursor.cs +++ b/src/StatusAggregator/ICursor.cs @@ -3,6 +3,9 @@ namespace StatusAggregator { + /// + /// Maintains the current progress of the job. + /// public interface ICursor { Task Get(); diff --git a/src/StatusAggregator/IEventUpdater.cs b/src/StatusAggregator/IEventUpdater.cs index 54e9bb437..971d0d0ad 100644 --- a/src/StatusAggregator/IEventUpdater.cs +++ b/src/StatusAggregator/IEventUpdater.cs @@ -1,12 +1,26 @@ using System; using System.Threading.Tasks; using NuGet.Services.Status.Table; -using StatusAggregator.Table; namespace StatusAggregator { + /// + /// Handles updating any active s. + /// public interface IEventUpdater { - Task UpdateEvent(EventEntity eventEntity, DateTime nextCreationTime); + /// + /// Updates all active s. + /// + /// The current timestamp processed by the job. + Task UpdateAllActiveEvents(DateTime cursor); + + /// + /// Update given . + /// Determines whether or not to deactivate and updates any messages associated with the event. + /// + /// The current timestamp processed by the job. + /// Whether or not was deactivated. + Task UpdateEvent(EventEntity eventEntity, DateTime cursor); } } diff --git a/src/StatusAggregator/IIncidentFactory.cs b/src/StatusAggregator/IIncidentFactory.cs index ed53e0d94..76c5b75c2 100644 --- a/src/StatusAggregator/IIncidentFactory.cs +++ b/src/StatusAggregator/IIncidentFactory.cs @@ -1,12 +1,17 @@ using System.Threading.Tasks; using NuGet.Services.Status.Table; using StatusAggregator.Parse; -using StatusAggregator.Table; namespace StatusAggregator { + /// + /// Handles creation of s. + /// public interface IIncidentFactory { + /// + /// Creates a from and persists it in storage. + /// Task CreateIncident(ParsedIncident parsedIncident); } } diff --git a/src/StatusAggregator/IIncidentUpdater.cs b/src/StatusAggregator/IIncidentUpdater.cs index b405ffaf2..03c4f54ef 100644 --- a/src/StatusAggregator/IIncidentUpdater.cs +++ b/src/StatusAggregator/IIncidentUpdater.cs @@ -1,12 +1,25 @@ -using System; +using NuGet.Services.Incidents; +using NuGet.Services.Status.Table; +using System; using System.Threading.Tasks; namespace StatusAggregator { + /// + /// Handles updating any active s. + /// public interface IIncidentUpdater { + /// + /// Update the status of any active s. + /// Task RefreshExistingIncidents(); + + /// + /// Fetches any new s and processes them. + /// + /// The current timestamp processed by the job. + /// The most recent processed by the job or null if no s were processed. Task FetchNewIncidents(DateTime cursor); - Task UpdateActiveEvents(DateTime cursor); } } diff --git a/src/StatusAggregator/IMessageUpdater.cs b/src/StatusAggregator/IMessageUpdater.cs index 589348749..d8677549d 100644 --- a/src/StatusAggregator/IMessageUpdater.cs +++ b/src/StatusAggregator/IMessageUpdater.cs @@ -1,14 +1,23 @@ using System; using System.Threading.Tasks; using NuGet.Services.Status.Table; -using StatusAggregator.Table; namespace StatusAggregator { + /// + /// Handles updating s for an . + /// public interface IMessageUpdater { - Task CreateMessageForEventStart(EventEntity eventEntity, DateTime nextCreationTime); + /// + /// Posts a for the start of . + /// + /// Used to determine whether or not the message should be posted. + Task CreateMessageForEventStart(EventEntity eventEntity, DateTime cursor); + /// + /// Posts a for the end of . + /// Task CreateMessageForEventEnd(EventEntity eventEntity); } } diff --git a/src/StatusAggregator/IStatusExporter.cs b/src/StatusAggregator/IStatusExporter.cs index 24fb36f67..7bac4f008 100644 --- a/src/StatusAggregator/IStatusExporter.cs +++ b/src/StatusAggregator/IStatusExporter.cs @@ -1,9 +1,13 @@ -using System.Threading.Tasks; +using NuGet.Services.Status; +using System.Threading.Tasks; namespace StatusAggregator { public interface IStatusExporter { + /// + /// Builds a and exports it to public storage so that it can be consumed by other services. + /// Task Export(); } } diff --git a/src/StatusAggregator/IStatusUpdater.cs b/src/StatusAggregator/IStatusUpdater.cs index 0a4c227c8..f14363c12 100644 --- a/src/StatusAggregator/IStatusUpdater.cs +++ b/src/StatusAggregator/IStatusUpdater.cs @@ -1,9 +1,13 @@ -using System.Threading.Tasks; +using NuGet.Services.Status; +using System.Threading.Tasks; namespace StatusAggregator { public interface IStatusUpdater { + /// + /// Aggregates the information necessary to build a that describes the NuGet service. + /// Task Update(); } } diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index 7ea672ce3..1d6f8ab64 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -1,6 +1,8 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.Status.Table; using StatusAggregator.Parse; using StatusAggregator.Table; @@ -10,75 +12,87 @@ namespace StatusAggregator public class IncidentFactory : IIncidentFactory { private readonly ITableWrapper _table; - private readonly IEventUpdater _eventUpdater; - public IncidentFactory(ITableWrapper table, IEventUpdater eventUpdater) + private readonly ILogger _logger; + + public IncidentFactory( + ITableWrapper table, + IEventUpdater eventUpdater, + ILogger logger) { _table = table; _eventUpdater = eventUpdater; + _logger = logger; } public async Task CreateIncident(ParsedIncident parsedIncident) { - Console.WriteLine($"Attempting to save {parsedIncident.Id}"); var incidentEntity = new IncidentEntity( - parsedIncident.Id, - parsedIncident.AffectedComponentPath, - parsedIncident.AffectedComponentStatus, - parsedIncident.CreationTime, + parsedIncident.Id, + parsedIncident.AffectedComponentPath, + parsedIncident.AffectedComponentStatus, + parsedIncident.CreationTime, parsedIncident.MitigationTime); - // Find an event to attach this incident to - var possibleEvents = _table - .CreateQuery() - .Where(e => - e.PartitionKey == EventEntity.DefaultPartitionKey && - // The incident and the event must affect the same component - e.AffectedComponentPath == parsedIncident.AffectedComponentPath && - // The event must begin before or at the same time as the incident - e.StartTime <= parsedIncident.CreationTime && - // The event must be active or the event must end after this incident begins - (e.IsActive || (e.EndTime >= parsedIncident.CreationTime))) - .ToList(); - - Console.WriteLine($"Found {possibleEvents.Count()} possible events to link {parsedIncident.Id} to"); - - foreach (var possibleEvent in possibleEvents) + using (_logger.Scope( + "Beginning to create incident.", + "Finished creating incident.", + "Creating incident '{IncidentRowKey}'.", incidentEntity.RowKey)) { - if (!_table.GetIncidentsLinkedToEvent(possibleEvent).ToList().Any()) + // Find an event to attach this incident to + var possibleEvents = _table + .CreateQuery() + .Where(e => + e.PartitionKey == EventEntity.DefaultPartitionKey && + // The incident and the event must affect the same component + e.AffectedComponentPath == incidentEntity.AffectedComponentPath && + // The event must begin before or at the same time as the incident + e.StartTime <= incidentEntity.CreationTime && + // The event must be active or the event must end after this incident begins + (e.IsActive || (e.EndTime >= incidentEntity.CreationTime))) + .ToList(); + + _logger.LogInformation("Found {EventCount} possible events to link incident to.", possibleEvents.Count()); + EventEntity eventToLinkTo = null; + foreach (var possibleEventToLinkTo in possibleEvents) { - Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because it is not linked to any incidents"); - continue; + if (!_table.GetIncidentsLinkedToEvent(possibleEventToLinkTo).ToList().Any()) + { + _logger.LogInformation("Cannot link incident to event '{EventRowKey}' because it is not linked to any incidents.", possibleEventToLinkTo.RowKey); + continue; + } + + if (await _eventUpdater.UpdateEvent(possibleEventToLinkTo, incidentEntity.CreationTime)) + { + _logger.LogInformation("Cannot link incident to event '{EventRowKey}' because it has been deactivated.", possibleEventToLinkTo.RowKey); + continue; + } + + _logger.LogInformation("Linking incident to event '{EventRowKey}'.", possibleEventToLinkTo.RowKey); + eventToLinkTo = possibleEventToLinkTo; + break; } - if (await _eventUpdater.UpdateEvent(possibleEvent, parsedIncident.CreationTime)) + if (eventToLinkTo == null) { - Console.WriteLine($"Cannot link {parsedIncident.Id} to {possibleEvent.RowKey} because its incidents are inactive and too old"); - continue; + eventToLinkTo = new EventEntity(incidentEntity); + _logger.LogInformation("Could not find existing event to link to, creating new event '{EventRowKey}' to link incident to.", eventToLinkTo.RowKey); + await _table.InsertOrReplaceAsync(eventToLinkTo); } - Console.WriteLine($"Linking {parsedIncident.Id} to {possibleEvent.RowKey}"); - if ((int)parsedIncident.AffectedComponentStatus > possibleEvent.AffectedComponentStatus) + incidentEntity.EventRowKey = eventToLinkTo.RowKey; + await _table.InsertOrReplaceAsync(incidentEntity); + + if ((int)parsedIncident.AffectedComponentStatus > eventToLinkTo.AffectedComponentStatus) { - Console.WriteLine($"{parsedIncident.Id} is a more severe than {possibleEvent.RowKey}, upgrading severity of event"); - possibleEvent.AffectedComponentStatus = (int)parsedIncident.AffectedComponentStatus; - await _table.InsertOrReplaceAsync(possibleEvent); + _logger.LogInformation("Increasing severity of event '{EventRowKey}' because newly linked incident is more severe than the event.", eventToLinkTo.RowKey); + eventToLinkTo.AffectedComponentStatus = (int)parsedIncident.AffectedComponentStatus; + await _table.InsertOrReplaceAsync(eventToLinkTo); } - incidentEntity.EventRowKey = possibleEvents.First().RowKey; - break; - } - - if (string.IsNullOrEmpty(incidentEntity.EventRowKey)) - { - var eventEntity = new EventEntity(incidentEntity); - Console.WriteLine($"Could not find existing event to attach {parsedIncident.Id} to, creating new event {eventEntity.RowKey}"); - await _table.InsertOrReplaceAsync(eventEntity); + return incidentEntity; } - - await _table.InsertOrReplaceAsync(incidentEntity); - return incidentEntity; } } } diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index a62afc349..66e7d53ab 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -1,4 +1,6 @@ -using NuGet.Services.Incidents; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; +using NuGet.Services.Incidents; using NuGet.Services.Status.Table; using StatusAggregator.Parse; using StatusAggregator.Table; @@ -15,56 +17,81 @@ public class IncidentUpdater : IIncidentUpdater private readonly IEventUpdater _eventUpdater; private readonly IAggregateIncidentParser _aggregateIncidentParser; - private readonly IIncidentApiClient _incidentClient; + private readonly IIncidentApiClient _incidentApiClient; private readonly IIncidentFactory _incidentFactory; - private readonly string _incidentTeamId; + private readonly string _incidentApiTeamId; + + private readonly ILogger _logger; public IncidentUpdater( ITableWrapper table, IEventUpdater eventUpdater, - IIncidentApiClient incidentClient, + IIncidentApiClient incidentApiClient, IAggregateIncidentParser aggregateIncidentParser, IIncidentFactory incidentFactory, - string incidentTeamId) + StatusAggregatorConfiguration configuration, + ILogger logger) { _table = table; _eventUpdater = eventUpdater; - _incidentClient = incidentClient; + _incidentApiClient = incidentApiClient; _aggregateIncidentParser = aggregateIncidentParser; _incidentFactory = incidentFactory; - _incidentTeamId = incidentTeamId; + _incidentApiTeamId = configuration.TeamId; + _logger = logger; } public async Task RefreshExistingIncidents() { - var activeIncidentEntities = _table - .CreateQuery() - .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); - - foreach (var activeIncidentEntity in activeIncidentEntities) + using (_logger.Scope( + "Beginning to refresh existing incidents.", + "Finished refreshing existing incidents.", + "Refreshing existing incidents.")) { - var activeIncident = await _incidentClient.GetIncident(activeIncidentEntity.IncidentApiId); - activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; - await _table.InsertOrReplaceAsync(activeIncidentEntity); + var activeIncidentEntities = _table + .CreateQuery() + .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); + + foreach (var activeIncidentEntity in activeIncidentEntities) + { + using (_logger.Scope( + "Beginning to refresh incident.", + "Finished refreshing incident.", + "Refreshing incident '{IncidentRowKey}'.", activeIncidentEntity.RowKey)) + { + var activeIncident = await _incidentApiClient.GetIncident(activeIncidentEntity.IncidentApiId); + activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; + _logger.LogInformation("Updated mitigation time of incident to {MitigationTime}", activeIncidentEntity.MitigationTime); + await _table.InsertOrReplaceAsync(activeIncidentEntity); + } + } } } public async Task FetchNewIncidents(DateTime cursor) { - var incidents = (await _incidentClient.GetIncidents(GetRecentIncidentsQuery(cursor))) - // The incident API trims the milliseconds from any filter. - // Therefore, a query asking for incidents newer than '2018-06-29T00:00:00.5Z' will return an incident from '2018-06-29T00:00:00.25Z' - // We must perform a check on the CreateDate ourselves to verify that no old incidents are returned. - .Where(i => i.CreateDate > cursor); - - var parsedIncidents = incidents.SelectMany(i => _aggregateIncidentParser.ParseIncident(i)); - foreach (var parsedIncident in parsedIncidents.OrderBy(i => i.CreationTime)) + using (_logger.Scope( + "Beginning to fetch new incidents.", + "Finished fetching new incidents.", + "Fetching all new incidents since {Cursor}.", cursor)) { - await _incidentFactory.CreateIncident(parsedIncident); - } + var incidents = (await _incidentApiClient.GetIncidents(GetRecentIncidentsQuery(cursor))) + // The incident API trims the milliseconds from any filter. + // Therefore, a query asking for incidents newer than '2018-06-29T00:00:00.5Z' will return an incident from '2018-06-29T00:00:00.25Z' + // We must perform a check on the CreateDate ourselves to verify that no old incidents are returned. + .Where(i => i.CreateDate > cursor); + + var parsedIncidents = incidents + .SelectMany(i => _aggregateIncidentParser.ParseIncident(i)) + .ToList(); + foreach (var parsedIncident in parsedIncidents.OrderBy(i => i.CreationTime)) + { + await _incidentFactory.CreateIncident(parsedIncident); + } - return incidents.Any() ? incidents.Max(i => i.CreateDate) : (DateTime?)null; + return incidents.Any() ? incidents.Max(i => i.CreateDate) : (DateTime?)null; + } } private string GetRecentIncidentsQuery(DateTime cursor) @@ -73,16 +100,7 @@ private string GetRecentIncidentsQuery(DateTime cursor) ? "" : $" and CreateDate gt datetime'{cursor.ToString("o")}'"; - return $"$filter=OwningTeamId eq '{_incidentTeamId}'{cursorPart}"; - } - - public async Task UpdateActiveEvents(DateTime cursor) - { - var eventsToCheckClosure = _table.GetActiveEvents(); - foreach (var eventToCheckClosure in eventsToCheckClosure) - { - await _eventUpdater.UpdateEvent(eventToCheckClosure, cursor); - } + return $"$filter=OwningTeamId eq '{_incidentApiTeamId}'{cursorPart}"; } } } diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 6d8e1737a..6ffbcbe53 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -6,8 +6,8 @@ using System.ComponentModel.Design; using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; using NuGet.Services.Incidents; @@ -18,75 +18,107 @@ namespace StatusAggregator { public class Job : JobBase { - private CloudBlobContainer _container; - private ITableWrapper _table; - - private IStatusUpdater _statusUpdater; - private IStatusExporter _statusExporter; + public IServiceProvider _serviceProvider; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount); - var storageAccount = CloudStorageAccount.Parse(storageConnectionString); - - var tableName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName); - _table = new TableWrapper(storageAccount, tableName); + var serviceCollection = new ServiceCollection(); - var cursor = new Cursor(_table); + AddLogging(serviceCollection); + AddConfiguration(serviceCollection, jobArgsDictionary); + AddStorage(serviceCollection); + AddServices(serviceCollection); - var messageUpdater = new MessageUpdater(_table); - var eventUpdater = new EventUpdater(_table, messageUpdater); - var incidentFactory = new IncidentFactory(_table, eventUpdater); + _serviceProvider = serviceCollection.BuildServiceProvider(); + } - var incidentApiBaseUri = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)); - var incidentApiTeamId = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiTeamId); - var incidentApiCertificate = GetCertificateFromJson( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificate)); - var incidentConfiguration = new IncidentApiConfiguration() { BaseUri = incidentApiBaseUri, Certificate = incidentApiCertificate }; - var incidentClient = new IncidentApiClient(incidentConfiguration); + public override Task Run() + { + return _serviceProvider + .GetService() + .Run(); + } - var environments = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusEnvironment).Split(';'); - var maximumSeverity = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusMaximumSeverity) ?? int.MaxValue; - var aggregateIncidentParser = new AggregateIncidentParser(GetIncidentParsers(environments, maximumSeverity)); - var incidentUpdater = new IncidentUpdater(_table, eventUpdater, incidentClient, aggregateIncidentParser, incidentFactory, incidentApiTeamId); + private static void AddServices(IServiceCollection serviceCollection) + { + serviceCollection.AddTransient(); + serviceCollection.AddSingleton(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + AddParsing(serviceCollection); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + } - _statusUpdater = new StatusUpdater(cursor, incidentUpdater); + private static void AddParsing(IServiceCollection serviceCollection) + { + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); - var blobClient = storageAccount.CreateCloudBlobClient(); - var containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName); - _container = blobClient.GetContainerReference(containerName); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); + serviceCollection.AddTransient(); - _statusExporter = new StatusExporter(_container, _table); + serviceCollection.AddTransient(); } - public override async Task Run() + private static void AddStorage(IServiceCollection serviceCollection) { - await _table.CreateIfNotExistsAsync(); - await _container.CreateIfNotExistsAsync(); - - await _statusUpdater.Update(); - await _statusExporter.Export(); + serviceCollection.AddSingleton( + serviceProvider => + { + var configuration = serviceProvider.GetRequiredService(); + return CloudStorageAccount.Parse(configuration.StorageAccount); + }); + + serviceCollection.AddSingleton( + serviceProvider => + { + var storageAccount = serviceProvider.GetRequiredService(); + var configuration = serviceProvider.GetRequiredService(); + return new TableWrapper(storageAccount, configuration.TableName); + }); + + serviceCollection.AddSingleton( + serviceProvider => + { + var storageAccount = serviceProvider.GetRequiredService(); + var blobClient = storageAccount.CreateCloudBlobClient(); + var configuration = serviceProvider.GetRequiredService(); + return blobClient.GetContainerReference(configuration.ContainerName); + }); } - private IEnumerable GetIncidentParsers(IEnumerable environments, int maximumSeverity) + private static void AddConfiguration(IServiceCollection serviceCollection, IDictionary jobArgsDictionary) { - var filters = GetIncidentParsingFilters(maximumSeverity); + var configuration = new StatusAggregatorConfiguration() + { + StorageAccount = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount), + ContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName), + TableName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName), + Environments = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusEnvironment).Split(';'), + MaximumSeverity = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusMaximumSeverity) ?? int.MaxValue, + TeamId = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiTeamId) + }; + + serviceCollection.AddSingleton(configuration); - return new IIncidentParser[] + var incidentApiConfiguration = new IncidentApiConfiguration() { - new ValidationDurationIncidentParser(environments, filters), - new OutdatedRegionalSearchServiceInstanceIncidentParser(environments, filters), - new OutdatedSearchServiceInstanceIncidentParser(environments, filters), - new PingdomIncidentParser(filters) + BaseUri = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)), + Certificate = GetCertificateFromJson(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificate)) }; + + serviceCollection.AddSingleton(incidentApiConfiguration); } - private IEnumerable GetIncidentParsingFilters(int maximumSeverity) + private void AddLogging(IServiceCollection serviceCollection) { - return new IIncidentParsingFilter[] - { - new SeverityFilter(maximumSeverity) - }; + serviceCollection.AddSingleton(LoggerFactory); + serviceCollection.AddLogging(); } private static X509Certificate2 GetCertificateFromJson(string certJson) diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 8a1de5ab2..19031fa2f 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.Status; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -12,64 +14,88 @@ public class MessageUpdater : IMessageUpdater { private static TimeSpan EventStartDelay = TimeSpan.FromMinutes(15); - private ITableWrapper _table; + private readonly ITableWrapper _table; - public MessageUpdater(ITableWrapper table) + private readonly ILogger _logger; + + public MessageUpdater(ITableWrapper table, ILogger logger) { _table = table; + _logger = logger; } - public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime nextCreationTime) + public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime cursor) { - // Enough time must have passed before we create a start message for an event. - // Only create a message if the event doesn't have messages associated with it. - if (nextCreationTime > eventEntity.StartTime + EventStartDelay && - !_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any() && - TryGetMessageForEventStartForEvent(eventEntity, out var message)) + using (_logger.Scope( + "Beginning to create message for start of event.", + "Finished creating message for start of event.", + "Creating message for start of event.")) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.StartTime, message); - await _table.InsertOrReplaceAsync(messageEntity); + if (cursor > eventEntity.StartTime + EventStartDelay) + { + if (!_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) + { + if (TryGetContentsForEventStartForEvent(eventEntity, out var contents)) + { + await CreateMessage(eventEntity, eventEntity.StartTime, contents); + } + else + { + _logger.LogWarning("Failed to create a message for start of event!"); + } + } + else + { + // If we've already told customers about an event, we don't need to tell them it's now impacting them. + _logger.LogInformation("Event has messages associated with it, cannot create message for its start."); + } + } + else + { + // We don't want to show events that are too recent to avoid noisy events. + _logger.LogInformation("Event is too recent, cannot create message for its start."); + } } } - private static bool TryGetMessageForEventStartForEvent(EventEntity eventEntity, out string message) + private bool TryGetContentsForEventStartForEvent(EventEntity eventEntity, out string contents) { - return TryGetMessageForEventHelper(eventEntity, _innerMessageMapForEventStart, "is", "", out message); + return TryGetContentsForEventHelper(eventEntity, _innerMessageMapForEventStart, "is", "", out contents); } private const string _youMayEncounterIssues = "You may encounter issues "; private static readonly IEnumerable _innerMessageMapForEventStart = new InnerMessageForComponentPathPrefix[] { new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.GalleryName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.GalleryName), $"{_youMayEncounterIssues}browsing the NuGet Gallery."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.ChinaRegionName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName), $"{_youMayEncounterIssues}restoring packages from NuGet.org's V3 feed from China."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName), $"{_youMayEncounterIssues}restoring packages from NuGet.org's V3 feed."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V2ProtocolName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName), $"{_youMayEncounterIssues}restoring packages from NuGet.org's V2 feed."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName), $"{_youMayEncounterIssues}restoring packages."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.SearchName, Components.ChinaRegionName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName), $"{_youMayEncounterIssues}searching for packages from China."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.SearchName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName), $"{_youMayEncounterIssues}searching for packages."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.UploadName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName), "New packages will likely take longer than usual before becoming available for download."), }; @@ -80,72 +106,94 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) throw new ArgumentException("Must pass in an event with an end time!", nameof(eventEntity)); } - // Only create a message if the event already has messages associated with it. - if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) + using (_logger.Scope( + "Beginning to create message for end of event.", + "Finished creating message for end of event.", + "Creating message for end of event.")) { - if (TryGetMessageForEventEndForEvent(eventEntity, out var message)) + if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { - var messageEntity = new MessageEntity(eventEntity, eventEntity.EndTime.Value, message); - await _table.InsertOrReplaceAsync(messageEntity); + if (TryGetContentsForEventEndForEvent(eventEntity, out var contents)) + { + await CreateMessage(eventEntity, eventEntity.EndTime.Value, contents); + } + else + { + _logger.LogWarning("Failed to create a message for start of event!"); + } + } + else + { + // If we've never told customers about an event, we don't need to tell them it's no longer impacting them. + _logger.LogInformation("Event has no messages associated with it, cannot create message for its end."); } } } - private static bool TryGetMessageForEventEndForEvent(EventEntity eventEntity, out string message) + private Task CreateMessage(EventEntity eventEntity, DateTime time, string contents) + { + var messageEntity = new MessageEntity(eventEntity, time, contents); + _logger.LogInformation("Creating message for start of event with time {MessageTimestamp} and contents {MessageContents}", + messageEntity.Time, messageEntity.Contents); + return _table.InsertOrReplaceAsync(messageEntity); + } + + private bool TryGetContentsForEventEndForEvent(EventEntity eventEntity, out string contents) { - return TryGetMessageForEventHelper(eventEntity, _innerMessageMapForEventEnd, "is no longer", " Thank you for your patience.", out message); + return TryGetContentsForEventHelper(eventEntity, _innerMessageMapForEventEnd, "is no longer", " Thank you for your patience.", out contents); } private const string _youShouldNoLongerEncounterAnyIssues = "You should no longer encounter any issues "; private static readonly IEnumerable _innerMessageMapForEventEnd = new InnerMessageForComponentPathPrefix[] { new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.GalleryName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.GalleryName), $"{_youShouldNoLongerEncounterAnyIssues}browsing the NuGet Gallery."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.ChinaRegionName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName), $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V3 feed from China."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V3ProtocolName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName), $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V3 feed."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName, Components.V2ProtocolName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName), $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V2 feed."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.RestoreName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName), $"{_youShouldNoLongerEncounterAnyIssues}restoring packages."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.SearchName, Components.ChinaRegionName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName), $"{_youShouldNoLongerEncounterAnyIssues}searching for packages from China."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.SearchName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName), $"{_youShouldNoLongerEncounterAnyIssues}searching for packages."), new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(Components.RootName, Components.UploadName), + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName), "New packages should become available for download as quickly as usual."), }; private static string[] _componentStatusNames = Enum.GetNames(typeof(ComponentStatus)); - private static bool TryGetMessageForEventHelper( + private bool TryGetContentsForEventHelper( EventEntity eventEntity, IEnumerable innerMessageMap, string boldedPartInnerString, string messageSuffix, - out string message) + out string contents) { - message = null; + contents = null; var path = eventEntity.AffectedComponentPath; - var component = Components.CreateNuGetServiceRootComponent().GetByPath(path); + var component = ComponentFactory.CreateNuGetServiceRootComponent().GetByPath(path); if (component == null) { + _logger.LogWarning("Could not find a component with path {ComponentPath}.", path); return false; } @@ -153,18 +201,19 @@ private static bool TryGetMessageForEventHelper( var name = string.Join(" ", componentNames.Skip(1).Reverse()); var boldedPart = $"{name} {boldedPartInnerString} {_componentStatusNames[eventEntity.AffectedComponentStatus].ToLowerInvariant()}."; - string innerMessage = innerMessageMap + string innerContents = innerMessageMap .FirstOrDefault(m => m.Matches(path))? .InnerMessage; - if (innerMessage == null) + if (innerContents == null) { + _logger.LogWarning("Could not find an inner message for path {ComponentPath}.", path); return false; } - message = $"{boldedPart} {innerMessage}{messageSuffix}"; + contents = $"{boldedPart} {innerContents}{messageSuffix}"; - return !string.IsNullOrEmpty(message); + return !string.IsNullOrEmpty(contents); } private class InnerMessageForComponentPathPrefix diff --git a/src/StatusAggregator/Parse/AggregateIncidentParser.cs b/src/StatusAggregator/Parse/AggregateIncidentParser.cs index 32054513a..8b393d5a6 100644 --- a/src/StatusAggregator/Parse/AggregateIncidentParser.cs +++ b/src/StatusAggregator/Parse/AggregateIncidentParser.cs @@ -1,29 +1,45 @@ -using NuGet.Services.Incidents; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; +using NuGet.Services.Incidents; using System.Collections.Generic; namespace StatusAggregator.Parse { + /// + /// Default implementation of that returns all s returned by its s. + /// public class AggregateIncidentParser : IAggregateIncidentParser { private readonly IEnumerable _incidentParsers; - public AggregateIncidentParser(IEnumerable incidentParsers) + private readonly ILogger _logger; + + public AggregateIncidentParser( + IEnumerable incidentParsers, + ILogger logger) { _incidentParsers = incidentParsers; + _logger = logger; } public IEnumerable ParseIncident(Incident incident) { - var parsedIncidents = new List(); - foreach (var incidentParser in _incidentParsers) + using (_logger.Scope( + "Beginning to parse incident.", + "Finished parsing incident.", + "Parsing incident {IncidentId}", incident.Id)) { - if (incidentParser.TryParseIncident(incident, out var parsedIncident)) + var parsedIncidents = new List(); + foreach (var incidentParser in _incidentParsers) { - parsedIncidents.Add(parsedIncident); + if (incidentParser.TryParseIncident(incident, out var parsedIncident)) + { + parsedIncidents.Add(parsedIncident); + } } - } - return parsedIncidents; + return parsedIncidents; + } } } } diff --git a/src/StatusAggregator/Parse/EnvironmentFilter.cs b/src/StatusAggregator/Parse/EnvironmentFilter.cs index 28ad97636..cd21e2195 100644 --- a/src/StatusAggregator/Parse/EnvironmentFilter.cs +++ b/src/StatusAggregator/Parse/EnvironmentFilter.cs @@ -1,4 +1,6 @@ -using NuGet.Services.Incidents; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; +using NuGet.Services.Incidents; using System; using System.Collections.Generic; using System.Linq; @@ -6,21 +8,42 @@ namespace StatusAggregator.Parse { + /// + /// Expects that the contains a named with a whitelisted value. + /// public class EnvironmentFilter : IIncidentParsingFilter { public const string EnvironmentGroupName = "Environment"; - public IEnumerable Environments { get; } + private IEnumerable _environments { get; } - public EnvironmentFilter(IEnumerable environments) + private readonly ILogger _logger; + + public EnvironmentFilter( + StatusAggregatorConfiguration configuration, + ILogger logger) { - Environments = environments; + _environments = configuration.Environments; + _logger = logger; } public bool ShouldParse(Incident incident, GroupCollection groups) { - return Environments.Any( - e => string.Equals(groups[EnvironmentGroupName].Value, e, StringComparison.OrdinalIgnoreCase)); + var group = groups[EnvironmentGroupName]; + + if (group.Success) + { + var groupValue = group.Value; + _logger.LogInformation("Incident has environment of {Environment}, expecting one of {Environments}.", + groupValue, string.Join(";", _environments)); + return _environments.Any( + e => string.Equals(groups[EnvironmentGroupName].Value, e, StringComparison.OrdinalIgnoreCase)); + } + else + { + _logger.LogInformation("Incident does not have an enviroment group, will not filter by environment."); + return true; + } } } } diff --git a/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs b/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs index 24ad5c35b..89868c550 100644 --- a/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs +++ b/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs @@ -1,13 +1,26 @@ -using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using NuGet.Services.Incidents; +using System; +using System.Collections.Generic; using System.Linq; namespace StatusAggregator.Parse { + /// + /// Subclass of that expects s are prefixed with "[ENVIRONMENT]". + /// public abstract class EnvironmentPrefixIncidentParser : IncidentParser { - public EnvironmentPrefixIncidentParser(string subtitleRegEx, IEnumerable environments, IEnumerable filters) - : base(GetRegEx(subtitleRegEx), filters.Concat(new[] { new EnvironmentFilter(environments) })) + public EnvironmentPrefixIncidentParser( + string subtitleRegEx, + IEnumerable filters, + ILogger logger) + : base(GetRegEx(subtitleRegEx), filters, logger) { + if (!filters.Any(f => f is EnvironmentFilter)) + { + throw new ArgumentException($"A {nameof(EnvironmentPrefixIncidentParser)} must be run with an {nameof(EnvironmentFilter)}!", nameof(filters)); + } } private static string GetRegEx(string subtitleRegEx) diff --git a/src/StatusAggregator/Parse/IAggregateIncidentParser.cs b/src/StatusAggregator/Parse/IAggregateIncidentParser.cs index 098f04990..ee25db1ec 100644 --- a/src/StatusAggregator/Parse/IAggregateIncidentParser.cs +++ b/src/StatusAggregator/Parse/IAggregateIncidentParser.cs @@ -3,8 +3,14 @@ namespace StatusAggregator.Parse { + /// + /// Aggregates the result of multiple s on . + /// public interface IAggregateIncidentParser { + /// + /// Runs multiple 's with and aggregates their results. + /// IEnumerable ParseIncident(Incident incident); } } diff --git a/src/StatusAggregator/Parse/IIncidentParser.cs b/src/StatusAggregator/Parse/IIncidentParser.cs index 0559ac069..c28354f6a 100644 --- a/src/StatusAggregator/Parse/IIncidentParser.cs +++ b/src/StatusAggregator/Parse/IIncidentParser.cs @@ -1,12 +1,25 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using NuGet.Services.Status; using NuGet.Services.Incidents; namespace StatusAggregator.Parse { + /// + /// Parses s and determines whether or not they should be used to determine the . + /// public interface IIncidentParser { + /// + /// Attempts to parse into . + /// + /// + /// A that describes or null if could not be parsed. + /// + /// + /// true if this can be parse and false otherwise. + /// bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident); } } diff --git a/src/StatusAggregator/Parse/IIncidentParsingFilter.cs b/src/StatusAggregator/Parse/IIncidentParsingFilter.cs index 8c23b4329..e471637db 100644 --- a/src/StatusAggregator/Parse/IIncidentParsingFilter.cs +++ b/src/StatusAggregator/Parse/IIncidentParsingFilter.cs @@ -3,8 +3,14 @@ namespace StatusAggregator.Parse { + /// + /// An additional filter that can be applied to a + /// public interface IIncidentParsingFilter { + /// + /// Returns whether or not an should parse . + /// bool ShouldParse(Incident incident, GroupCollection groups); } } diff --git a/src/StatusAggregator/Parse/IncidentParser.cs b/src/StatusAggregator/Parse/IncidentParser.cs index ffb6b6525..c3ebe68b1 100644 --- a/src/StatusAggregator/Parse/IncidentParser.cs +++ b/src/StatusAggregator/Parse/IncidentParser.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.Incidents; using NuGet.Services.Status; using System.Collections.Generic; @@ -9,56 +11,111 @@ namespace StatusAggregator.Parse { + /// + /// Abstract implementation of that allows specifying a to analyze s with. + /// public abstract class IncidentParser : IIncidentParser { private readonly string _regExPattern; private readonly IEnumerable _filters; - public IncidentParser(string regExPattern) + private readonly ILogger _logger; + + public IncidentParser( + string regExPattern, + ILogger logger) { _regExPattern = regExPattern; _filters = Enumerable.Empty(); + _logger = logger; } - public IncidentParser(string regExPattern, IEnumerable filters) - : this(regExPattern) + public IncidentParser( + string regExPattern, + IEnumerable filters, + ILogger logger) + : this(regExPattern, logger) { _filters = filters.ToList(); } public bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident) { - parsedIncident = null; - var match = Regex.Match(incident.Title, _regExPattern); - return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); + using (_logger.Scope( + "Beginning to parse incident with parser.", + "Finished parsing incident with parser.", + "Parsing incident with parser {IncidentParserType} using {RegExPattern}", + GetType(), _regExPattern)) + { + parsedIncident = null; + var title = incident.Title; + var match = Regex.Match(title, _regExPattern); + _logger.LogInformation("Incident title is {IncidentTitle}, RegEx match result: {MatchResult}", title, match.Success); + return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); + } } - protected bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident) + private bool TryParseIncident(Incident incident, GroupCollection groups, out ParsedIncident parsedIncident) { parsedIncident = null; - if (_filters.Any(f => !f.ShouldParse(incident, groups))) + if (_filters.Any(f => + { + using (_logger.Scope( + "Beginning filtering incident.", + "Finished filtering incident.", + "Filtering incident using filter {IncidentFilterType}", f.GetType())) + { + var shouldParse = f.ShouldParse(incident, groups); + _logger.LogInformation("Filter returned {FilterResult}.", shouldParse); + return !shouldParse; + } + })) { + _logger.LogInformation("Incident failed at least one filter!"); return false; } if (!TryParseAffectedComponentPath(incident, groups, out var affectedComponentPath)) { + _logger.LogInformation("Could not parse incident component path!"); return false; } + _logger.LogInformation("Parsed affected component path {AffectedComponentPath}.", affectedComponentPath); + if (!TryParseAffectedComponentStatus(incident, groups, out var affectedComponentStatus)) { + _logger.LogInformation("Could not parse incident component status!"); return false; } + _logger.LogInformation("Parsed affected component status {AffectedComponentPath}.", affectedComponentStatus); + parsedIncident = new ParsedIncident(incident, affectedComponentPath, affectedComponentStatus); return true; } + /// + /// Attempts to parse a from . + /// + /// + /// The parsed from or null if could not be parsed. + /// + /// + /// true if a can be parsed from and false otherwise. + /// protected abstract bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath); + /// + /// Attempts to parse a from . + /// + /// + /// The parsed from or if could not be parsed. + /// + /// true if a can be parsed from and false otherwise. + /// protected abstract bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus); } } diff --git a/src/StatusAggregator/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs deleted file mode 100644 index 685cc342f..000000000 --- a/src/StatusAggregator/Parse/OutdatedRegionalSearchServiceInstanceIncidentParser.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using NuGet.Services.Incidents; -using NuGet.Services.Status; - -namespace StatusAggregator.Parse -{ - public class OutdatedRegionalSearchServiceInstanceIncidentParser : EnvironmentPrefixIncidentParser - { - private const string ServiceEnvironmentGroupName = "SearchEnvironment"; - private const string ServiceRegionGroupName = "SearchRegion"; - private static string SubtitleRegEx = $@"Search service 'nuget-(?<{ServiceEnvironmentGroupName}>.*)-(?<{ServiceRegionGroupName}>.*)-(v2v3)?search' is using an outdated index!"; - - private readonly IEnumerable _environments; - - public OutdatedRegionalSearchServiceInstanceIncidentParser(IEnumerable environments, IEnumerable filters) - : base(SubtitleRegEx, environments, filters) - { - _environments = environments; - } - - protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) - { - affectedComponentPath = null; - - var searchEnvironment = groups[ServiceEnvironmentGroupName].Value; - var searchRegion = groups[ServiceRegionGroupName].Value; - - if (!_environments.Any(e => string.Equals(searchEnvironment, e, StringComparison.OrdinalIgnoreCase))) - { - return false; - } - - affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.UploadName); - return true; - } - - protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) - { - affectedComponentStatus = ComponentStatus.Degraded; - return true; - } - } -} diff --git a/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index 4566a567e..6fe2b0c9a 100644 --- a/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -1,4 +1,5 @@ -using NuGet.Services.Incidents; +using Microsoft.Extensions.Logging; +using NuGet.Services.Incidents; using NuGet.Services.Status; using System.Collections.Generic; using System.Text.RegularExpressions; @@ -9,14 +10,16 @@ public class OutdatedSearchServiceInstanceIncidentParser : EnvironmentPrefixInci { private const string SubtitleRegEx = "A search service instance is using an outdated index!"; - public OutdatedSearchServiceInstanceIncidentParser(IEnumerable environments, IEnumerable filters) - : base(SubtitleRegEx, environments, filters) + public OutdatedSearchServiceInstanceIncidentParser( + IEnumerable filters, + ILogger logger) + : base(SubtitleRegEx, filters, logger) { } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.UploadName); + affectedComponentPath = ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName); return true; } diff --git a/src/StatusAggregator/Parse/ParsedIncident.cs b/src/StatusAggregator/Parse/ParsedIncident.cs index a7197130c..e41562df2 100644 --- a/src/StatusAggregator/Parse/ParsedIncident.cs +++ b/src/StatusAggregator/Parse/ParsedIncident.cs @@ -4,6 +4,9 @@ namespace StatusAggregator.Parse { + /// + /// Describes how a affects a . + /// public class ParsedIncident { public ParsedIncident( diff --git a/src/StatusAggregator/Parse/PingdomIncidentParser.cs b/src/StatusAggregator/Parse/PingdomIncidentParser.cs index ddbf6a0ec..21b7666fa 100644 --- a/src/StatusAggregator/Parse/PingdomIncidentParser.cs +++ b/src/StatusAggregator/Parse/PingdomIncidentParser.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; using NuGet.Services.Status; @@ -11,9 +12,14 @@ public class PingdomIncidentParser : IncidentParser private const string CheckUrlGroupName = "CheckUrl"; private static string SubtitleRegEx = $@"Pingdom check '(?<{CheckNameGroupName}>.*)' is failing! '(?<{CheckUrlGroupName}>.*)' is DOWN!"; - public PingdomIncidentParser(IEnumerable filters) - : base(SubtitleRegEx, filters) + private readonly ILogger _logger; + + public PingdomIncidentParser( + IEnumerable filters, + ILogger logger) + : base(SubtitleRegEx, filters, logger) { + _logger = logger; } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) @@ -21,61 +27,62 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo affectedComponentPath = null; var checkName = groups[CheckNameGroupName].Value; + _logger.LogInformation("Check name is {CheckName}.", checkName); switch (checkName) { case "CDN DNS": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.RestoreName, Components.V3ProtocolName); + ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName); break; case "CDN Global": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.GlobalRegionName); + ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.GlobalRegionName); break; case "CDN China": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.RestoreName, Components.V3ProtocolName, Components.ChinaRegionName); + ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName); break; case "Gallery DNS": case "Gallery Home": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.GalleryName); + ComponentFactory.RootName, ComponentFactory.GalleryName); break; case "Gallery USNC /": case "Gallery USNC /Packages": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.GalleryName, Components.UsncInstanceName); + ComponentFactory.RootName, ComponentFactory.GalleryName, ComponentFactory.UsncInstanceName); break; case "Gallery USSC /": case "Gallery USSC /Packages": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.GalleryName, Components.UsscInstanceName); + ComponentFactory.RootName, ComponentFactory.GalleryName, ComponentFactory.UsscInstanceName); break; case "Gallery USNC /api/v2/Packages()": case "Gallery USNC /api/v2/package/NuGet.GalleryUptime/1.0.0": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.RestoreName, Components.V2ProtocolName, Components.UsncInstanceName); + ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName, ComponentFactory.UsncInstanceName); break; case "Gallery USSC /api/v2/Packages()": case "Gallery USSC /api/v2/package/NuGet.GalleryUptime/1.0.0": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.RestoreName, Components.V2ProtocolName, Components.UsscInstanceName); + ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName, ComponentFactory.UsscInstanceName); break; case "Search USNC /query": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.SearchName, Components.GlobalRegionName, Components.UsncInstanceName); + ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.GlobalRegionName, ComponentFactory.UsncInstanceName); break; case "Search USSC /query": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.SearchName, Components.GlobalRegionName, Components.UsscInstanceName); + ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.GlobalRegionName, ComponentFactory.UsscInstanceName); break; case "Search EA /query": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.SearchName, Components.ChinaRegionName, Components.EaInstanceName); + ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName, ComponentFactory.EaInstanceName); break; case "Search SEA /query": affectedComponentPath = ComponentUtility.GetPath( - Components.RootName, Components.SearchName, Components.ChinaRegionName, Components.SeaInstanceName); + ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName, ComponentFactory.SeaInstanceName); break; default: return false; diff --git a/src/StatusAggregator/Parse/SeverityFilter.cs b/src/StatusAggregator/Parse/SeverityFilter.cs index 6da8846c1..20aa873e2 100644 --- a/src/StatusAggregator/Parse/SeverityFilter.cs +++ b/src/StatusAggregator/Parse/SeverityFilter.cs @@ -1,20 +1,33 @@ -using NuGet.Services.Incidents; +using Microsoft.Extensions.Logging; +using NuGet.Services.Incidents; using System.Text.RegularExpressions; namespace StatusAggregator.Parse { + /// + /// Expects that the severity of an must be lower than a threshold. + /// public class SeverityFilter : IIncidentParsingFilter { private readonly int _maximumSeverity; - public SeverityFilter(int maximumSeverity) + private readonly ILogger _logger; + + public SeverityFilter( + StatusAggregatorConfiguration configuration, + ILogger logger) { - _maximumSeverity = maximumSeverity; + _maximumSeverity = configuration.MaximumSeverity; + _logger = logger; } - + public bool ShouldParse(Incident incident, GroupCollection groups) { - return incident.Severity <= _maximumSeverity; + var actualSeverity = incident.Severity; + _logger.LogInformation( + "Filtering incident severity: severity is {IncidentSeverity}, must be less than or equal to {MaximumSeverity}", + actualSeverity, _maximumSeverity); + return actualSeverity <= _maximumSeverity; } } } diff --git a/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs index 84d13ea32..f687d92fa 100644 --- a/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; using NuGet.Services.Status; @@ -9,14 +10,16 @@ public class ValidationDurationIncidentParser : EnvironmentPrefixIncidentParser { private const string SubtitleRegEx = "Too many packages are stuck in the \"Validating\" state!"; - public ValidationDurationIncidentParser(IEnumerable environments, IEnumerable filters) - : base(SubtitleRegEx, environments, filters) + public ValidationDurationIncidentParser( + IEnumerable filters, + ILogger logger) + : base(SubtitleRegEx, filters, logger) { } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = ComponentUtility.GetPath(Components.RootName, Components.UploadName); + affectedComponentPath = ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName); return true; } diff --git a/src/StatusAggregator/StatusAggregator.cs b/src/StatusAggregator/StatusAggregator.cs new file mode 100644 index 000000000..11c235ac4 --- /dev/null +++ b/src/StatusAggregator/StatusAggregator.cs @@ -0,0 +1,36 @@ +using Microsoft.WindowsAzure.Storage.Blob; +using StatusAggregator.Table; +using System.Threading.Tasks; + +namespace StatusAggregator +{ + public class StatusAggregator + { + private readonly CloudBlobContainer _container; + private readonly ITableWrapper _table; + + private readonly IStatusUpdater _statusUpdater; + private readonly IStatusExporter _statusExporter; + + public StatusAggregator( + CloudBlobContainer container, + ITableWrapper table, + IStatusUpdater statusUpdater, + IStatusExporter statusExporter) + { + _container = container; + _table = table; + _statusUpdater = statusUpdater; + _statusExporter = statusExporter; + } + + public async Task Run() + { + await _table.CreateIfNotExistsAsync(); + await _container.CreateIfNotExistsAsync(); + + await _statusUpdater.Update(); + await _statusExporter.Export(); + } + } +} diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 172242aac..5b14cae0d 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -43,11 +43,11 @@ - - + + @@ -61,6 +61,7 @@ + @@ -74,7 +75,6 @@ - @@ -93,6 +93,9 @@ + + 1.1.1 + 2.28.0-sb-incidents-35734 diff --git a/src/StatusAggregator/StatusAggregatorConfiguration.cs b/src/StatusAggregator/StatusAggregatorConfiguration.cs new file mode 100644 index 000000000..aee6a89d0 --- /dev/null +++ b/src/StatusAggregator/StatusAggregatorConfiguration.cs @@ -0,0 +1,44 @@ +using NuGet.Jobs; +using NuGet.Services.Incidents; +using NuGet.Services.Status; +using StatusAggregator.Parse; +using System; +using System.Collections.Generic; + +namespace StatusAggregator +{ + public class StatusAggregatorConfiguration + { + /// + /// A connection string for the storage account to use. + /// + public string StorageAccount { get; set; } + + /// + /// The container name to export the to. + /// + public string ContainerName { get; set; } + + /// + /// The table name to persist the in. + /// + public string TableName { get; set; } + + /// + /// A list of environments to filter incidents by. + /// See . + /// + public IEnumerable Environments { get; set; } + + /// + /// The maximum severity of any incidents to process. + /// See . + /// + public int MaximumSeverity { get; set; } + + /// + /// A team ID to use to query the incident API. + /// + public string TeamId { get; set; } + } +} diff --git a/src/StatusAggregator/StatusContractResolver.cs b/src/StatusAggregator/StatusContractResolver.cs index 124ef791a..94bdc59f1 100644 --- a/src/StatusAggregator/StatusContractResolver.cs +++ b/src/StatusAggregator/StatusContractResolver.cs @@ -6,6 +6,9 @@ namespace StatusAggregator { + /// + /// Implementation of used by such that empty fields and arrays are not serialized. + /// public class StatusContractResolver : DefaultContractResolver { protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) @@ -16,6 +19,7 @@ protected override JsonProperty CreateProperty(MemberInfo member, MemberSerializ if (propertyType == typeof(string)) { + // Do not serialize strings if they are null or empty. property.ShouldSerialize = instance => !string.IsNullOrEmpty((string)instance); } @@ -31,6 +35,7 @@ private void SetShouldSerializeForIEnumerable(JsonProperty property, MemberInfo { Func getValue; + // Create a function to get the value of the member using its type. switch (member.MemberType) { case MemberTypes.Field: @@ -43,6 +48,7 @@ private void SetShouldSerializeForIEnumerable(JsonProperty property, MemberInfo return; } + // Do not serialize an IEnumerable if it is null or empty property.ShouldSerialize = instance => { var value = (IEnumerable)getValue(instance); diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index a504e0303..ba87160ca 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -2,9 +2,11 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using NuGet.Jobs.Extensions; using NuGet.Services.Status; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -19,6 +21,8 @@ public class StatusExporter : IStatusExporter private readonly CloudBlobContainer _container; private readonly ITableWrapper _table; + private readonly ILogger _logger; + private static readonly JsonSerializerSettings _statusBlobJsonSerializerSettings = new JsonSerializerSettings() { ContractResolver = new StatusContractResolver(), @@ -27,55 +31,85 @@ public class StatusExporter : IStatusExporter NullValueHandling = NullValueHandling.Ignore }; - public StatusExporter(CloudBlobContainer container, ITableWrapper table) + public StatusExporter( + CloudBlobContainer container, + ITableWrapper table, + ILogger logger) { _container = container; _table = table; + _logger = logger; } public async Task Export() { - var rootComponent = Components.CreateNuGetServiceRootComponent(); + using (_logger.Scope( + "Beginning to export service status.", + "Finished exporting service status.", + "Exporting service status.")) + { + var rootComponent = ComponentFactory.CreateNuGetServiceRootComponent(); + + var recentEvents = _table + .CreateQuery() + .Where(e => + e.PartitionKey == EventEntity.DefaultPartitionKey && + (e.IsActive || (e.EndTime >= DateTime.UtcNow - EventVisibilityPeriod))) + .ToList() + .Select(e => + { + var messages = _table.GetMessagesLinkedToEvent(e) + .ToList() + .Select(m => m.AsMessage()); + return e.AsEvent(messages); + }) + .Where(e => e.Messages != null && e.Messages.Any()); - var recentEvents = _table - .CreateQuery() - .Where(e => - e.PartitionKey == EventEntity.DefaultPartitionKey && - (e.IsActive || (e.EndTime >= DateTime.UtcNow - EventVisibilityPeriod))) - .ToList() - .Select(e => + // If multiple events are affecting a single region, the event with the highest severity should affect the component. + var activeEvents = recentEvents + .Where(e => e.EndTime == null || e.EndTime >= DateTime.UtcNow) + .GroupBy(e => e.AffectedComponentPath) + .Select(g => g.OrderByDescending(e => e.AffectedComponentStatus).First()); + + foreach (var activeEvent in activeEvents) { - var messages = _table.GetMessagesLinkedToEvent(e) - .ToList() - .Select(m => m.AsMessage()); - return e.AsEvent(messages); - }) - .Where(e => e.Messages != null && e.Messages.Any()); + using (_logger.Scope( + "Beginning to apply active event to root component.", + "Finished applying active event to root component.", + "Applying active event affecting '{AffectedComponentPath}' of severity {AffectedComponentStatus} at {StartTime} to root component", + activeEvent.AffectedComponentPath, activeEvent.AffectedComponentStatus, activeEvent.StartTime)) + { + var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); - // If multiple events are affecting a single region, the event with the highest severity should affect the component. - var activeEvents = recentEvents - .Where(e => e.EndTime == null || e.EndTime >= DateTime.UtcNow) - .GroupBy(e => e.AffectedComponentPath) - .Select(g => g.OrderByDescending(e => e.AffectedComponentStatus).First()); + if (currentComponent == null) + { + _logger.LogWarning("Couldn't find component corresponding to active event."); + continue; + } - foreach (var activeEvent in activeEvents) - { - var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); + currentComponent.Status = activeEvent.AffectedComponentStatus; + } + } - if (currentComponent == null) + string statusJson; + using (_logger.Scope( + "Beginning to serialize service status.", + "Finished serializing service status.", + "Serializing service status.")) { - continue; + var status = new ServiceStatus(rootComponent, recentEvents); + statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); } - currentComponent.Status = activeEvent.AffectedComponentStatus; + using (_logger.Scope( + "Beginning to save service status to blob storage.", + "Finished saving service status to blob storage.", + "Saving service status to blob storage.")) + { + var blob = _container.GetBlockBlobReference(StatusBlobName); + await blob.UploadTextAsync(statusJson); + } } - - - var status = new ServiceStatus(rootComponent, recentEvents); - var statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); - - var blob = _container.GetBlockBlobReference(StatusBlobName); - await blob.UploadTextAsync(statusJson); } } } diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs index adb984a1a..2e9666f9e 100644 --- a/src/StatusAggregator/StatusUpdater.cs +++ b/src/StatusAggregator/StatusUpdater.cs @@ -1,33 +1,48 @@ -using System; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; +using System; using System.Threading.Tasks; namespace StatusAggregator { public class StatusUpdater : IStatusUpdater { + private readonly ICursor _cursor; private readonly IIncidentUpdater _incidentUpdater; + private readonly IEventUpdater _eventUpdater; - private ICursor _cursor; + private readonly ILogger _logger; public StatusUpdater( ICursor cursor, - IIncidentUpdater incidentUpdater) + IIncidentUpdater incidentUpdater, + IEventUpdater eventUpdater, + ILogger logger) { _cursor = cursor; _incidentUpdater = incidentUpdater; + _eventUpdater = eventUpdater; + _logger = logger; } public async Task Update() { - var lastCursor = await _cursor.Get(); - - await _incidentUpdater.RefreshExistingIncidents(); - var nextCursor = await _incidentUpdater.FetchNewIncidents(lastCursor); - await _incidentUpdater.UpdateActiveEvents(nextCursor ?? DateTime.UtcNow); - - if (nextCursor.HasValue) + using (_logger.Scope( + "Beginning to update service status.", + "Finished updating service status.", + "Updating service status.")) { - await _cursor.Set(nextCursor.Value); + var lastCursor = await _cursor.Get(); + + await _incidentUpdater.RefreshExistingIncidents(); + var nextCursor = await _incidentUpdater.FetchNewIncidents(lastCursor); + + await _eventUpdater.UpdateAllActiveEvents(nextCursor ?? DateTime.UtcNow); + + if (nextCursor.HasValue) + { + await _cursor.Set(nextCursor.Value); + } } } } diff --git a/src/StatusAggregator/Table/TableWrapper.cs b/src/StatusAggregator/Table/TableWrapper.cs index ba4c86c9a..091bbcb56 100644 --- a/src/StatusAggregator/Table/TableWrapper.cs +++ b/src/StatusAggregator/Table/TableWrapper.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Table; @@ -8,7 +7,9 @@ namespace StatusAggregator.Table { public class TableWrapper : ITableWrapper { - public TableWrapper(CloudStorageAccount storageAccount, string tableName) + public TableWrapper( + CloudStorageAccount storageAccount, + string tableName) { var tableClient = storageAccount.CreateCloudTableClient(); _table = tableClient.GetTableReference(tableName); diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs index 0a3fb2fad..f1b7a8fd8 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs @@ -14,6 +14,7 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; +using NuGet.Jobs.Extensions; using NuGet.Services.KeyVault; using NuGet.Services.Sql; @@ -61,7 +62,7 @@ private static PackageLicenseReport CreateReport(JObject messageEvent) public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var secretInjector = serviceContainer.GetService(); var dbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); _packageDbConnectionFactory = new AzureSqlConnectionFactory(dbConnectionString, secretInjector); diff --git a/tests/StatusAggregator.Tests/EventUpdaterTests.cs b/tests/StatusAggregator.Tests/EventUpdaterTests.cs index 2c86a5645..f3c7cea7c 100644 --- a/tests/StatusAggregator.Tests/EventUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/EventUpdaterTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Moq; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -28,6 +29,7 @@ public class EventUpdaterTests private Mock _tableWrapperMock { get; } private Mock _messageUpdaterMock { get; } + private Mock> _loggerMock { get; } private EventUpdater _eventUpdater { get; } private EventEntity _eventEntity { get; } @@ -35,7 +37,7 @@ public EventUpdaterTests() { _tableWrapperMock = new Mock(); _messageUpdaterMock = new Mock(); - _eventUpdater = new EventUpdater(_tableWrapperMock.Object, _messageUpdaterMock.Object); + _eventUpdater = new EventUpdater(_tableWrapperMock.Object, _messageUpdaterMock.Object, _loggerMock.Object); _eventEntity = new EventEntity() { @@ -45,17 +47,6 @@ public EventUpdaterTests() }; } - private static IncidentEntity CreateIncidentEntity(DateTime? mitigationTime = null) - { - return new IncidentEntity() - { - PartitionKey = IncidentEntity.DefaultPartitionKey, - EventRowKey = RowKey, - CreationTime = DateTime.MinValue, - MitigationTime = mitigationTime - }; - } - [Fact] public async Task ThrowsIfEventNull() { @@ -137,5 +128,16 @@ public async Task ClosesEventIfClosableIncidents() x => x.CreateMessageForEventEnd(_eventEntity), Times.Once()); } + + private static IncidentEntity CreateIncidentEntity(DateTime? mitigationTime = null) + { + return new IncidentEntity() + { + PartitionKey = IncidentEntity.DefaultPartitionKey, + EventRowKey = RowKey, + CreationTime = DateTime.MinValue, + MitigationTime = mitigationTime + }; + } } } diff --git a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs index 01eb2a253..c619ef627 100644 --- a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs +++ b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Table; using Moq; using NuGet.Services.Incidents; @@ -20,15 +21,17 @@ public class IncidentFactoryTests private static DateTime CreationTime = new DateTime(2017, 7, 10); private Mock _tableWrapperMock { get; } - private Mock _eventUpdater { get; } + private Mock _eventUpdaterMock { get; } + private Mock> _loggerMock { get; } private IncidentFactory _incidentFactory { get; } private ParsedIncident _parsedIncident { get; } public IncidentFactoryTests() { _tableWrapperMock = new Mock(); - _eventUpdater = new Mock(); - _incidentFactory = new IncidentFactory(_tableWrapperMock.Object, _eventUpdater.Object); + _eventUpdaterMock = new Mock(); + _loggerMock = new Mock>(); + _incidentFactory = new IncidentFactory(_tableWrapperMock.Object, _eventUpdaterMock.Object, _loggerMock.Object); var incident = new Incident() { Id = Id, Source = new IncidentSourceData() { CreateDate = CreationTime } }; _parsedIncident = new ParsedIncident(incident, AffectedComponentPath, AffectedComponentStatus); diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj index c87ceb60a..2f28acd80 100644 --- a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj +++ b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj @@ -41,7 +41,6 @@ - @@ -63,7 +62,7 @@ 4.7.145 - 2.28.0-sb-incidents-35615 + 2.28.0-sb-incidents-35734 2.28.0-sb-morestatus-35606 From 10e159af066918919a2ff179e865eef132a0f06c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 10:45:49 -0700 Subject: [PATCH 29/49] reduce logger scope api --- .../Extensions/LoggerExtensions.cs | 31 +++++++------------ src/StatusAggregator/EventUpdater.cs | 10 ++---- src/StatusAggregator/IncidentFactory.cs | 5 +-- src/StatusAggregator/IncidentUpdater.cs | 15 ++------- src/StatusAggregator/MessageUpdater.cs | 16 +++------- .../Parse/AggregateIncidentParser.cs | 5 +-- src/StatusAggregator/Parse/IncidentParser.cs | 10 ++---- src/StatusAggregator/StatusExporter.cs | 20 +++--------- src/StatusAggregator/StatusUpdater.cs | 5 +-- 9 files changed, 30 insertions(+), 87 deletions(-) diff --git a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs index 0b3d0e911..759c8c4b7 100644 --- a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs +++ b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs @@ -7,17 +7,10 @@ public static class LoggerExtensions { public static IDisposable Scope( this ILogger logger, - string beginMessage, - string finishMessage, - string scopeMessage, - params object[] scopeMessageArgs) + string message, + params object[] args) { - return new LoggerScopeHelper( - logger, - beginMessage, - finishMessage, - scopeMessage, - scopeMessageArgs); + return new LoggerScopeHelper(logger, message, args); } private class LoggerScopeHelper : IDisposable @@ -25,29 +18,27 @@ private class LoggerScopeHelper : IDisposable private readonly ILogger _logger; private readonly IDisposable _scope; - private readonly string _finishMessage; + private readonly string _message; + private readonly object[] _args; private bool _isDisposed = false; public LoggerScopeHelper( - ILogger logger, - string beginMessage, - string finishMessage, - string scopeMessage, - object[] scopeMessageArgs) + ILogger logger, string message, object[] args) { _logger = logger; - _scope = logger.BeginScope(scopeMessage, scopeMessageArgs); - _finishMessage = finishMessage; + _message = message; + _args = args; - _logger.LogInformation(beginMessage); + _scope = logger.BeginScope(_message, _args); + _logger.LogInformation("Entering scope: " + _message, _args); } public void Dispose() { if (!_isDisposed) { - _logger.LogInformation(_finishMessage); + _logger.LogInformation("Leaving scope: " + _message, _args); _scope.Dispose(); _isDisposed = true; } diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index a23755b47..ab0ef5b5e 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -29,10 +29,7 @@ public EventUpdater( public async Task UpdateAllActiveEvents(DateTime cursor) { - using (_logger.Scope( - "Beginning to update all active events.", - "Finished updating all active events.", - "Updating all active events.")) + using (_logger.Scope("Updating all active events.")) { var activeEvents = _table.GetActiveEvents(); foreach (var activeEvent in activeEvents) @@ -46,10 +43,7 @@ public async Task UpdateEvent(EventEntity eventEntity, DateTime cursor) { eventEntity = eventEntity ?? throw new ArgumentNullException(nameof(eventEntity)); - using (_logger.Scope( - "Beginning to update event.", - "Finished updating event.", - "Updating event '{EventRowKey}' given cursor {Cursor}.", eventEntity.RowKey, cursor)) + using (_logger.Scope("Updating event '{EventRowKey}' given cursor {Cursor}.", eventEntity.RowKey, cursor)) { if (!eventEntity.IsActive) { diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index 1d6f8ab64..ca724cba7 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -35,10 +35,7 @@ public async Task CreateIncident(ParsedIncident parsedIncident) parsedIncident.CreationTime, parsedIncident.MitigationTime); - using (_logger.Scope( - "Beginning to create incident.", - "Finished creating incident.", - "Creating incident '{IncidentRowKey}'.", incidentEntity.RowKey)) + using (_logger.Scope("Creating incident '{IncidentRowKey}'.", incidentEntity.RowKey)) { // Find an event to attach this incident to var possibleEvents = _table diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index 66e7d53ab..59bc2a4c5 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -44,10 +44,7 @@ public IncidentUpdater( public async Task RefreshExistingIncidents() { - using (_logger.Scope( - "Beginning to refresh existing incidents.", - "Finished refreshing existing incidents.", - "Refreshing existing incidents.")) + using (_logger.Scope("Refreshing existing incidents.")) { var activeIncidentEntities = _table .CreateQuery() @@ -55,10 +52,7 @@ public async Task RefreshExistingIncidents() foreach (var activeIncidentEntity in activeIncidentEntities) { - using (_logger.Scope( - "Beginning to refresh incident.", - "Finished refreshing incident.", - "Refreshing incident '{IncidentRowKey}'.", activeIncidentEntity.RowKey)) + using (_logger.Scope("Refreshing incident '{IncidentRowKey}'.", activeIncidentEntity.RowKey)) { var activeIncident = await _incidentApiClient.GetIncident(activeIncidentEntity.IncidentApiId); activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; @@ -71,10 +65,7 @@ public async Task RefreshExistingIncidents() public async Task FetchNewIncidents(DateTime cursor) { - using (_logger.Scope( - "Beginning to fetch new incidents.", - "Finished fetching new incidents.", - "Fetching all new incidents since {Cursor}.", cursor)) + using (_logger.Scope("Fetching all new incidents since {Cursor}.", cursor)) { var incidents = (await _incidentApiClient.GetIncidents(GetRecentIncidentsQuery(cursor))) // The incident API trims the milliseconds from any filter. diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 19031fa2f..7d1523480 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -26,10 +26,7 @@ public MessageUpdater(ITableWrapper table, ILogger logger) public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime cursor) { - using (_logger.Scope( - "Beginning to create message for start of event.", - "Finished creating message for start of event.", - "Creating message for start of event.")) + using (_logger.Scope("Creating message for start of event.")) { if (cursor > eventEntity.StartTime + EventStartDelay) { @@ -46,7 +43,7 @@ public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime c } else { - // If we've already told customers about an event, we don't need to tell them it's now impacting them. + // If we've already told customers about an event, we don't need to tell them about it again. _logger.LogInformation("Event has messages associated with it, cannot create message for its start."); } } @@ -106,10 +103,7 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) throw new ArgumentException("Must pass in an event with an end time!", nameof(eventEntity)); } - using (_logger.Scope( - "Beginning to create message for end of event.", - "Finished creating message for end of event.", - "Creating message for end of event.")) + using (_logger.Scope("Creating message for end of event.")) { if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { @@ -119,7 +113,7 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) } else { - _logger.LogWarning("Failed to create a message for start of event!"); + _logger.LogWarning("Failed to create message!"); } } else @@ -133,7 +127,7 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) private Task CreateMessage(EventEntity eventEntity, DateTime time, string contents) { var messageEntity = new MessageEntity(eventEntity, time, contents); - _logger.LogInformation("Creating message for start of event with time {MessageTimestamp} and contents {MessageContents}", + _logger.LogInformation("Creating message with time {MessageTimestamp} and contents {MessageContents}", messageEntity.Time, messageEntity.Contents); return _table.InsertOrReplaceAsync(messageEntity); } diff --git a/src/StatusAggregator/Parse/AggregateIncidentParser.cs b/src/StatusAggregator/Parse/AggregateIncidentParser.cs index 8b393d5a6..6875e75a6 100644 --- a/src/StatusAggregator/Parse/AggregateIncidentParser.cs +++ b/src/StatusAggregator/Parse/AggregateIncidentParser.cs @@ -24,10 +24,7 @@ public AggregateIncidentParser( public IEnumerable ParseIncident(Incident incident) { - using (_logger.Scope( - "Beginning to parse incident.", - "Finished parsing incident.", - "Parsing incident {IncidentId}", incident.Id)) + using (_logger.Scope("Parsing incident {IncidentId}", incident.Id)) { var parsedIncidents = new List(); foreach (var incidentParser in _incidentParsers) diff --git a/src/StatusAggregator/Parse/IncidentParser.cs b/src/StatusAggregator/Parse/IncidentParser.cs index c3ebe68b1..0f6d7cfb5 100644 --- a/src/StatusAggregator/Parse/IncidentParser.cs +++ b/src/StatusAggregator/Parse/IncidentParser.cs @@ -42,10 +42,7 @@ public IncidentParser( public bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident) { - using (_logger.Scope( - "Beginning to parse incident with parser.", - "Finished parsing incident with parser.", - "Parsing incident with parser {IncidentParserType} using {RegExPattern}", + using (_logger.Scope("Parsing incident with parser {IncidentParserType} using {RegExPattern}", GetType(), _regExPattern)) { parsedIncident = null; @@ -62,10 +59,7 @@ private bool TryParseIncident(Incident incident, GroupCollection groups, out Par if (_filters.Any(f => { - using (_logger.Scope( - "Beginning filtering incident.", - "Finished filtering incident.", - "Filtering incident using filter {IncidentFilterType}", f.GetType())) + using (_logger.Scope("Filtering incident using filter {IncidentFilterType}", f.GetType())) { var shouldParse = f.ShouldParse(incident, groups); _logger.LogInformation("Filter returned {FilterResult}.", shouldParse); diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index ba87160ca..ccb179d5b 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -43,10 +43,7 @@ public StatusExporter( public async Task Export() { - using (_logger.Scope( - "Beginning to export service status.", - "Finished exporting service status.", - "Exporting service status.")) + using (_logger.Scope("Exporting service status.")) { var rootComponent = ComponentFactory.CreateNuGetServiceRootComponent(); @@ -73,10 +70,7 @@ public async Task Export() foreach (var activeEvent in activeEvents) { - using (_logger.Scope( - "Beginning to apply active event to root component.", - "Finished applying active event to root component.", - "Applying active event affecting '{AffectedComponentPath}' of severity {AffectedComponentStatus} at {StartTime} to root component", + using (_logger.Scope("Applying active event affecting '{AffectedComponentPath}' of severity {AffectedComponentStatus} at {StartTime} to root component", activeEvent.AffectedComponentPath, activeEvent.AffectedComponentStatus, activeEvent.StartTime)) { var currentComponent = rootComponent.GetByPath(activeEvent.AffectedComponentPath); @@ -92,19 +86,13 @@ public async Task Export() } string statusJson; - using (_logger.Scope( - "Beginning to serialize service status.", - "Finished serializing service status.", - "Serializing service status.")) + using (_logger.Scope("Serializing service status.")) { var status = new ServiceStatus(rootComponent, recentEvents); statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); } - using (_logger.Scope( - "Beginning to save service status to blob storage.", - "Finished saving service status to blob storage.", - "Saving service status to blob storage.")) + using (_logger.Scope("Saving service status to blob storage.")) { var blob = _container.GetBlockBlobReference(StatusBlobName); await blob.UploadTextAsync(statusJson); diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs index 2e9666f9e..6fd495836 100644 --- a/src/StatusAggregator/StatusUpdater.cs +++ b/src/StatusAggregator/StatusUpdater.cs @@ -27,10 +27,7 @@ public StatusUpdater( public async Task Update() { - using (_logger.Scope( - "Beginning to update service status.", - "Finished updating service status.", - "Updating service status.")) + using (_logger.Scope("Updating service status.")) { var lastCursor = await _cursor.Get(); From b90c7ff0eccd58bb217686760a1659904f86932f Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 11:07:42 -0700 Subject: [PATCH 30/49] small improvement to eventupdater and incidentupdater apis --- src/StatusAggregator/EventUpdater.cs | 7 ++++--- src/StatusAggregator/IEventUpdater.cs | 2 +- src/StatusAggregator/IIncidentUpdater.cs | 2 +- src/StatusAggregator/IncidentUpdater.cs | 12 +++++++----- src/StatusAggregator/StatusUpdater.cs | 4 ++-- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index ab0ef5b5e..c4006c855 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -27,11 +27,12 @@ public EventUpdater( _logger = logger; } - public async Task UpdateAllActiveEvents(DateTime cursor) + public async Task UpdateActiveEvents(DateTime cursor) { - using (_logger.Scope("Updating all active events.")) + using (_logger.Scope("Updating active events.")) { - var activeEvents = _table.GetActiveEvents(); + var activeEvents = _table.GetActiveEvents().ToList(); + _logger.LogInformation("Updating {ActiveEventsCount} active events.", activeEvents.Count()); foreach (var activeEvent in activeEvents) { await UpdateEvent(activeEvent, cursor); diff --git a/src/StatusAggregator/IEventUpdater.cs b/src/StatusAggregator/IEventUpdater.cs index 971d0d0ad..769a66ee4 100644 --- a/src/StatusAggregator/IEventUpdater.cs +++ b/src/StatusAggregator/IEventUpdater.cs @@ -13,7 +13,7 @@ public interface IEventUpdater /// Updates all active s. /// /// The current timestamp processed by the job. - Task UpdateAllActiveEvents(DateTime cursor); + Task UpdateActiveEvents(DateTime cursor); /// /// Update given . diff --git a/src/StatusAggregator/IIncidentUpdater.cs b/src/StatusAggregator/IIncidentUpdater.cs index 03c4f54ef..9259ad32e 100644 --- a/src/StatusAggregator/IIncidentUpdater.cs +++ b/src/StatusAggregator/IIncidentUpdater.cs @@ -13,7 +13,7 @@ public interface IIncidentUpdater /// /// Update the status of any active s. /// - Task RefreshExistingIncidents(); + Task RefreshActiveIncidents(); /// /// Fetches any new s and processes them. diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index 59bc2a4c5..fdfd16cd7 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -42,21 +42,23 @@ public IncidentUpdater( _logger = logger; } - public async Task RefreshExistingIncidents() + public async Task RefreshActiveIncidents() { - using (_logger.Scope("Refreshing existing incidents.")) + using (_logger.Scope("Refreshing active incidents.")) { var activeIncidentEntities = _table .CreateQuery() - .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive); + .Where(i => i.PartitionKey == IncidentEntity.DefaultPartitionKey && i.IsActive) + .ToList(); + _logger.LogInformation("Refreshing {ActiveIncidentsCount} active incidents.", activeIncidentEntities.Count()); foreach (var activeIncidentEntity in activeIncidentEntities) { - using (_logger.Scope("Refreshing incident '{IncidentRowKey}'.", activeIncidentEntity.RowKey)) + using (_logger.Scope("Refreshing active incident '{IncidentRowKey}'.", activeIncidentEntity.RowKey)) { var activeIncident = await _incidentApiClient.GetIncident(activeIncidentEntity.IncidentApiId); activeIncidentEntity.MitigationTime = activeIncident.MitigationData?.Date; - _logger.LogInformation("Updated mitigation time of incident to {MitigationTime}", activeIncidentEntity.MitigationTime); + _logger.LogInformation("Updated mitigation time of active incident to {MitigationTime}", activeIncidentEntity.MitigationTime); await _table.InsertOrReplaceAsync(activeIncidentEntity); } } diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs index 6fd495836..ff94a33f6 100644 --- a/src/StatusAggregator/StatusUpdater.cs +++ b/src/StatusAggregator/StatusUpdater.cs @@ -31,10 +31,10 @@ public async Task Update() { var lastCursor = await _cursor.Get(); - await _incidentUpdater.RefreshExistingIncidents(); + await _incidentUpdater.RefreshActiveIncidents(); var nextCursor = await _incidentUpdater.FetchNewIncidents(lastCursor); - await _eventUpdater.UpdateAllActiveEvents(nextCursor ?? DateTime.UtcNow); + await _eventUpdater.UpdateActiveEvents(nextCursor ?? DateTime.UtcNow); if (nextCursor.HasValue) { From e7f521cc2cb1d7886aed54da81599fb14ac59f32 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 11:48:51 -0700 Subject: [PATCH 31/49] fix logger extensions for null scope --- src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs | 2 +- tests/StatusAggregator.Tests/EventUpdaterTests.cs | 7 +++++-- tests/StatusAggregator.Tests/IncidentFactoryTests.cs | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs index 759c8c4b7..d23c5b8cf 100644 --- a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs +++ b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs @@ -39,7 +39,7 @@ public void Dispose() if (!_isDisposed) { _logger.LogInformation("Leaving scope: " + _message, _args); - _scope.Dispose(); + _scope?.Dispose(); // ILogger can return a null scope (most notably during testing with a Mock) _isDisposed = true; } } diff --git a/tests/StatusAggregator.Tests/EventUpdaterTests.cs b/tests/StatusAggregator.Tests/EventUpdaterTests.cs index f3c7cea7c..e02059d2c 100644 --- a/tests/StatusAggregator.Tests/EventUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/EventUpdaterTests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Moq; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -29,7 +30,6 @@ public class EventUpdaterTests private Mock _tableWrapperMock { get; } private Mock _messageUpdaterMock { get; } - private Mock> _loggerMock { get; } private EventUpdater _eventUpdater { get; } private EventEntity _eventEntity { get; } @@ -37,7 +37,10 @@ public EventUpdaterTests() { _tableWrapperMock = new Mock(); _messageUpdaterMock = new Mock(); - _eventUpdater = new EventUpdater(_tableWrapperMock.Object, _messageUpdaterMock.Object, _loggerMock.Object); + _eventUpdater = new EventUpdater( + _tableWrapperMock.Object, + _messageUpdaterMock.Object, + Mock.Of>()); _eventEntity = new EventEntity() { diff --git a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs index c619ef627..b5fd3e3af 100644 --- a/tests/StatusAggregator.Tests/IncidentFactoryTests.cs +++ b/tests/StatusAggregator.Tests/IncidentFactoryTests.cs @@ -22,7 +22,6 @@ public class IncidentFactoryTests private Mock _tableWrapperMock { get; } private Mock _eventUpdaterMock { get; } - private Mock> _loggerMock { get; } private IncidentFactory _incidentFactory { get; } private ParsedIncident _parsedIncident { get; } @@ -30,8 +29,10 @@ public IncidentFactoryTests() { _tableWrapperMock = new Mock(); _eventUpdaterMock = new Mock(); - _loggerMock = new Mock>(); - _incidentFactory = new IncidentFactory(_tableWrapperMock.Object, _eventUpdaterMock.Object, _loggerMock.Object); + _incidentFactory = new IncidentFactory( + _tableWrapperMock.Object, + _eventUpdaterMock.Object, + Mock.Of>()); var incident = new Incident() { Id = Id, Source = new IncidentSourceData() { CreateDate = CreationTime } }; _parsedIncident = new ParsedIncident(incident, AffectedComponentPath, AffectedComponentStatus); From 8ac112993fba9d275a7d8d3bd09c0fef9c826b25 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 11:52:39 -0700 Subject: [PATCH 32/49] remove unused usings in status aggregator config --- src/StatusAggregator/StatusAggregatorConfiguration.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/StatusAggregator/StatusAggregatorConfiguration.cs b/src/StatusAggregator/StatusAggregatorConfiguration.cs index aee6a89d0..47e128ec8 100644 --- a/src/StatusAggregator/StatusAggregatorConfiguration.cs +++ b/src/StatusAggregator/StatusAggregatorConfiguration.cs @@ -1,8 +1,5 @@ -using NuGet.Jobs; -using NuGet.Services.Incidents; -using NuGet.Services.Status; +using NuGet.Services.Status; using StatusAggregator.Parse; -using System; using System.Collections.Generic; namespace StatusAggregator From cb19be4090df71f869e82e6ecf3a560a4f5af584 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 12:08:01 -0700 Subject: [PATCH 33/49] add copyright --- src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs | 5 ++++- src/StatusAggregator/ComponentFactory.cs | 5 ++++- src/StatusAggregator/Cursor.cs | 5 ++++- src/StatusAggregator/EventUpdater.cs | 5 ++++- src/StatusAggregator/ICursor.cs | 5 ++++- src/StatusAggregator/IEventUpdater.cs | 5 ++++- src/StatusAggregator/IIncidentFactory.cs | 5 ++++- src/StatusAggregator/IIncidentUpdater.cs | 7 +++++-- src/StatusAggregator/IMessageUpdater.cs | 5 ++++- src/StatusAggregator/IStatusExporter.cs | 5 ++++- src/StatusAggregator/IStatusUpdater.cs | 5 ++++- src/StatusAggregator/IncidentFactory.cs | 5 ++++- src/StatusAggregator/IncidentUpdater.cs | 5 ++++- src/StatusAggregator/MessageUpdater.cs | 5 ++++- src/StatusAggregator/Parse/AggregateIncidentParser.cs | 5 ++++- src/StatusAggregator/Parse/EnvironmentFilter.cs | 5 ++++- .../Parse/EnvironmentPrefixIncidentParser.cs | 5 ++++- src/StatusAggregator/Parse/IAggregateIncidentParser.cs | 5 ++++- src/StatusAggregator/Parse/IIncidentParsingFilter.cs | 5 ++++- .../Parse/OutdatedSearchServiceInstanceIncidentParser.cs | 5 ++++- src/StatusAggregator/Parse/ParsedIncident.cs | 5 ++++- src/StatusAggregator/Parse/PingdomIncidentParser.cs | 5 ++++- src/StatusAggregator/Parse/SeverityFilter.cs | 5 ++++- .../Parse/ValidationDurationIncidentParser.cs | 5 ++++- src/StatusAggregator/StatusAggregator.cs | 5 ++++- src/StatusAggregator/StatusAggregatorConfiguration.cs | 5 ++++- src/StatusAggregator/StatusContractResolver.cs | 5 ++++- src/StatusAggregator/StatusExporter.cs | 5 ++++- src/StatusAggregator/StatusUpdater.cs | 5 ++++- src/StatusAggregator/Table/ITableWrapper.cs | 5 ++++- src/StatusAggregator/Table/TableWrapper.cs | 5 ++++- src/StatusAggregator/Table/TableWrapperExtensions.cs | 5 ++++- 32 files changed, 129 insertions(+), 33 deletions(-) diff --git a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs index d23c5b8cf..c34ecf1c4 100644 --- a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs +++ b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using System; namespace NuGet.Jobs.Extensions diff --git a/src/StatusAggregator/ComponentFactory.cs b/src/StatusAggregator/ComponentFactory.cs index 1b2333f5e..8bbae6412 100644 --- a/src/StatusAggregator/ComponentFactory.cs +++ b/src/StatusAggregator/ComponentFactory.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Status; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Status; namespace StatusAggregator { diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index c7a213819..a4804f08e 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Services.Status.Table; diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index c4006c855..8746ad80e 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; diff --git a/src/StatusAggregator/ICursor.cs b/src/StatusAggregator/ICursor.cs index 77e958e8b..bb6754b4e 100644 --- a/src/StatusAggregator/ICursor.cs +++ b/src/StatusAggregator/ICursor.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Threading.Tasks; namespace StatusAggregator diff --git a/src/StatusAggregator/IEventUpdater.cs b/src/StatusAggregator/IEventUpdater.cs index 769a66ee4..e32650d4a 100644 --- a/src/StatusAggregator/IEventUpdater.cs +++ b/src/StatusAggregator/IEventUpdater.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Threading.Tasks; using NuGet.Services.Status.Table; diff --git a/src/StatusAggregator/IIncidentFactory.cs b/src/StatusAggregator/IIncidentFactory.cs index 76c5b75c2..975507d8a 100644 --- a/src/StatusAggregator/IIncidentFactory.cs +++ b/src/StatusAggregator/IIncidentFactory.cs @@ -1,4 +1,7 @@ -using System.Threading.Tasks; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; using NuGet.Services.Status.Table; using StatusAggregator.Parse; diff --git a/src/StatusAggregator/IIncidentUpdater.cs b/src/StatusAggregator/IIncidentUpdater.cs index 9259ad32e..d0be4ba69 100644 --- a/src/StatusAggregator/IIncidentUpdater.cs +++ b/src/StatusAggregator/IIncidentUpdater.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Incidents; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Incidents; using NuGet.Services.Status.Table; using System; using System.Threading.Tasks; @@ -22,4 +25,4 @@ public interface IIncidentUpdater /// The most recent processed by the job or null if no s were processed. Task FetchNewIncidents(DateTime cursor); } -} +} \ No newline at end of file diff --git a/src/StatusAggregator/IMessageUpdater.cs b/src/StatusAggregator/IMessageUpdater.cs index d8677549d..7b922aee5 100644 --- a/src/StatusAggregator/IMessageUpdater.cs +++ b/src/StatusAggregator/IMessageUpdater.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Threading.Tasks; using NuGet.Services.Status.Table; diff --git a/src/StatusAggregator/IStatusExporter.cs b/src/StatusAggregator/IStatusExporter.cs index 7bac4f008..54af859d3 100644 --- a/src/StatusAggregator/IStatusExporter.cs +++ b/src/StatusAggregator/IStatusExporter.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Status; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Status; using System.Threading.Tasks; namespace StatusAggregator diff --git a/src/StatusAggregator/IStatusUpdater.cs b/src/StatusAggregator/IStatusUpdater.cs index f14363c12..b2565427e 100644 --- a/src/StatusAggregator/IStatusUpdater.cs +++ b/src/StatusAggregator/IStatusUpdater.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Status; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Status; using System.Threading.Tasks; namespace StatusAggregator diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index ca724cba7..f6aa2130d 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index fdfd16cd7..f60ea1db8 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; using NuGet.Services.Incidents; using NuGet.Services.Status.Table; diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 7d1523480..adc163871 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; diff --git a/src/StatusAggregator/Parse/AggregateIncidentParser.cs b/src/StatusAggregator/Parse/AggregateIncidentParser.cs index 6875e75a6..c3ebb616a 100644 --- a/src/StatusAggregator/Parse/AggregateIncidentParser.cs +++ b/src/StatusAggregator/Parse/AggregateIncidentParser.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; using NuGet.Services.Incidents; using System.Collections.Generic; diff --git a/src/StatusAggregator/Parse/EnvironmentFilter.cs b/src/StatusAggregator/Parse/EnvironmentFilter.cs index cd21e2195..9e4eeb88c 100644 --- a/src/StatusAggregator/Parse/EnvironmentFilter.cs +++ b/src/StatusAggregator/Parse/EnvironmentFilter.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; using NuGet.Services.Incidents; using System; diff --git a/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs b/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs index 89868c550..2af15eaee 100644 --- a/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs +++ b/src/StatusAggregator/Parse/EnvironmentPrefixIncidentParser.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; using System; using System.Collections.Generic; diff --git a/src/StatusAggregator/Parse/IAggregateIncidentParser.cs b/src/StatusAggregator/Parse/IAggregateIncidentParser.cs index ee25db1ec..8bbcdc6af 100644 --- a/src/StatusAggregator/Parse/IAggregateIncidentParser.cs +++ b/src/StatusAggregator/Parse/IAggregateIncidentParser.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Incidents; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Incidents; using System.Collections.Generic; namespace StatusAggregator.Parse diff --git a/src/StatusAggregator/Parse/IIncidentParsingFilter.cs b/src/StatusAggregator/Parse/IIncidentParsingFilter.cs index e471637db..efed05e9d 100644 --- a/src/StatusAggregator/Parse/IIncidentParsingFilter.cs +++ b/src/StatusAggregator/Parse/IIncidentParsingFilter.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Incidents; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Incidents; using System.Text.RegularExpressions; namespace StatusAggregator.Parse diff --git a/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index 6fe2b0c9a..a60c0792f 100644 --- a/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; using NuGet.Services.Status; using System.Collections.Generic; diff --git a/src/StatusAggregator/Parse/ParsedIncident.cs b/src/StatusAggregator/Parse/ParsedIncident.cs index e41562df2..685d9d934 100644 --- a/src/StatusAggregator/Parse/ParsedIncident.cs +++ b/src/StatusAggregator/Parse/ParsedIncident.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Incidents; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Incidents; using NuGet.Services.Status; using System; diff --git a/src/StatusAggregator/Parse/PingdomIncidentParser.cs b/src/StatusAggregator/Parse/PingdomIncidentParser.cs index 21b7666fa..5b5130802 100644 --- a/src/StatusAggregator/Parse/PingdomIncidentParser.cs +++ b/src/StatusAggregator/Parse/PingdomIncidentParser.cs @@ -1,4 +1,7 @@ -using System.Collections.Generic; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; diff --git a/src/StatusAggregator/Parse/SeverityFilter.cs b/src/StatusAggregator/Parse/SeverityFilter.cs index 20aa873e2..d8bc63b32 100644 --- a/src/StatusAggregator/Parse/SeverityFilter.cs +++ b/src/StatusAggregator/Parse/SeverityFilter.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; using System.Text.RegularExpressions; diff --git a/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs index f687d92fa..8877e8103 100644 --- a/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs @@ -1,4 +1,7 @@ -using System.Collections.Generic; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; diff --git a/src/StatusAggregator/StatusAggregator.cs b/src/StatusAggregator/StatusAggregator.cs index 11c235ac4..a2327ed8d 100644 --- a/src/StatusAggregator/StatusAggregator.cs +++ b/src/StatusAggregator/StatusAggregator.cs @@ -1,4 +1,7 @@ -using Microsoft.WindowsAzure.Storage.Blob; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage.Blob; using StatusAggregator.Table; using System.Threading.Tasks; diff --git a/src/StatusAggregator/StatusAggregatorConfiguration.cs b/src/StatusAggregator/StatusAggregatorConfiguration.cs index 47e128ec8..25bc33f2d 100644 --- a/src/StatusAggregator/StatusAggregatorConfiguration.cs +++ b/src/StatusAggregator/StatusAggregatorConfiguration.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Status; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Status; using StatusAggregator.Parse; using System.Collections.Generic; diff --git a/src/StatusAggregator/StatusContractResolver.cs b/src/StatusAggregator/StatusContractResolver.cs index 94bdc59f1..cb9419116 100644 --- a/src/StatusAggregator/StatusContractResolver.cs +++ b/src/StatusAggregator/StatusContractResolver.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections; using System.Reflection; using Newtonsoft.Json; diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index ccb179d5b..35bb87eda 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs index ff94a33f6..5b7d645f7 100644 --- a/src/StatusAggregator/StatusUpdater.cs +++ b/src/StatusAggregator/StatusUpdater.cs @@ -1,4 +1,7 @@ -using Microsoft.Extensions.Logging; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; using System; using System.Threading.Tasks; diff --git a/src/StatusAggregator/Table/ITableWrapper.cs b/src/StatusAggregator/Table/ITableWrapper.cs index d319ca1db..24aa23e38 100644 --- a/src/StatusAggregator/Table/ITableWrapper.cs +++ b/src/StatusAggregator/Table/ITableWrapper.cs @@ -1,4 +1,7 @@ -using System.Linq; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage.Table; diff --git a/src/StatusAggregator/Table/TableWrapper.cs b/src/StatusAggregator/Table/TableWrapper.cs index 091bbcb56..56d908421 100644 --- a/src/StatusAggregator/Table/TableWrapper.cs +++ b/src/StatusAggregator/Table/TableWrapper.cs @@ -1,4 +1,7 @@ -using System.Linq; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Table; diff --git a/src/StatusAggregator/Table/TableWrapperExtensions.cs b/src/StatusAggregator/Table/TableWrapperExtensions.cs index cfbc0c239..b3387a9d1 100644 --- a/src/StatusAggregator/Table/TableWrapperExtensions.cs +++ b/src/StatusAggregator/Table/TableWrapperExtensions.cs @@ -1,4 +1,7 @@ -using NuGet.Services.Status.Table; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Status.Table; using System.Collections.Generic; using System.Linq; From 5ed851965da15c5fe9e9c8bb4a284031dde35721 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 13:23:22 -0700 Subject: [PATCH 34/49] remove accidentally added usings --- src/ArchivePackages/ArchivePackages.Job.cs | 3 --- src/Gallery.CredentialExpiration/Job.cs | 5 +---- .../Tasks/SupportRequestsNotificationScheduledTask.cs | 3 --- src/Search.GenerateAuxiliaryData/Job.cs | 3 --- src/Stats.AggregateCdnDownloadsInGallery/Job.cs | 3 --- src/Stats.CreateAzureCdnWarehouseReports/Job.cs | 3 --- src/Stats.ImportAzureCdnStatistics/Job.cs | 3 --- .../RefreshClientDimensionJob.cs | 3 --- src/Stats.RollUpDownloadFacts/Job.cs | 3 --- src/UpdateLicenseReports/UpdateLicenseReports.Job.cs | 3 --- 10 files changed, 1 insertion(+), 31 deletions(-) diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 947c6ffc7..4e036b203 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -13,9 +13,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace ArchivePackages { diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index f20dac3c8..e0340eb0a 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -15,9 +15,6 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.Services.Storage; namespace Gallery.CredentialExpiration @@ -43,7 +40,7 @@ public class Job : JobBase public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { _whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf); - + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase); _galleryBrand = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand); diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs index 751b5d9f8..51ffd7d5d 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs @@ -7,9 +7,6 @@ using System.Data.SqlClient; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Notifications; using NuGet.SupportRequests.Notifications.Services; using NuGet.SupportRequests.Notifications.Templates; diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 69526b733..84ed5b8b4 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -11,9 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs index de57be944..ee4e750b5 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs @@ -11,9 +11,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using IPackageIdGroup = System.Linq.IGrouping; namespace Stats.AggregateCdnDownloadsInGallery diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs index e366792c1..4f8f55254 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs @@ -11,9 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs index 05054c077..5f82c0762 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/Job.cs @@ -10,9 +10,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; using System.Data.SqlClient; diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index bd74ef46c..5bc3fb1b7 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -8,9 +8,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.ImportAzureCdnStatistics; namespace Stats.RefreshClientDimension diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/Job.cs index b6e065a42..51d3dcc40 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/Job.cs @@ -9,9 +9,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Stats.RollUpDownloadFacts { diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs index 0b1db5227..d012fc1de 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs @@ -14,9 +14,6 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; -using NuGet.Jobs.Extensions; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace UpdateLicenseReports { From 922ff50f41c62588bd43cc2d5982d0099df26b6c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 13:53:25 -0700 Subject: [PATCH 35/49] add time thresholds in configuration --- .../Configuration/JobArgumentNames.cs | 7 +++- src/StatusAggregator/EventUpdater.cs | 6 ++- src/StatusAggregator/Job.cs | 39 +++++++++++++++---- src/StatusAggregator/MessageUpdater.cs | 10 +++-- .../StatusAggregatorConfiguration.cs | 19 +++++++++ src/StatusAggregator/StatusExporter.cs | 6 ++- .../EventUpdaterTests.cs | 2 +- 7 files changed, 71 insertions(+), 18 deletions(-) diff --git a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs index eace904e7..3f01ac002 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs @@ -125,9 +125,12 @@ public static class JobArgumentNames public const string StatusEnvironment = "StatusEnvironment"; public const string StatusMaximumSeverity = "StatusMaximumSeverity"; public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri"; - public const string StatusIncidentApiTeamId = "StatusIncidentApiTeamId"; public const string StatusIncidentApiCertificate = "StatusIncidentApiCertificate"; - + public const string StatusIncidentApiTeamId = "StatusIncidentApiTeamId"; + public const string StatusEventStartMessageDelayMinutes = "StatusEventStartMessageDelayMinutes"; + public const string StatusEventEndDelayMinutes = "StatusEventEndDelayMinutes"; + public const string StatusEventVisibilityPeriodDays = "StatusEventVisibilityPeriodDays"; + // Arguments specific to Stats.AggregateCdnDownloadsInGallery public static string BatchSleepSeconds = "BatchSleepSeconds"; } diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index 8746ad80e..915e88bf6 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -13,7 +13,7 @@ namespace StatusAggregator { public class EventUpdater : IEventUpdater { - public static TimeSpan EventEndDelay = TimeSpan.FromMinutes(10); + public readonly TimeSpan _eventEndDelay; private readonly ITableWrapper _table; private readonly IMessageUpdater _messageUpdater; @@ -23,10 +23,12 @@ public class EventUpdater : IEventUpdater public EventUpdater( ITableWrapper table, IMessageUpdater messageUpdater, + StatusAggregatorConfiguration configuration, ILogger logger) { _table = table; _messageUpdater = messageUpdater; + _eventEndDelay = TimeSpan.FromMinutes(configuration.EventEndDelayMinutes); _logger = logger; } @@ -65,7 +67,7 @@ public async Task UpdateEvent(EventEntity eventEntity, DateTime cursor) } var shouldDeactivate = !incidentsLinkedToEventQuery - .Where(i => i.IsActive || i.MitigationTime > cursor - EventEndDelay) + .Where(i => i.IsActive || i.MitigationTime > cursor - _eventEndDelay) .ToList() .Any(); diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 6ffbcbe53..71ddbffbd 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -92,24 +92,47 @@ private static void AddStorage(IServiceCollection serviceCollection) }); } + private const int _defaultEventStartMessageDelayMinutes = 15; + private const int _defaultEventEndDelayMinutes = 10; + private const int _defaultEventVisibilityPeriod = 10; + private static void AddConfiguration(IServiceCollection serviceCollection, IDictionary jobArgsDictionary) { var configuration = new StatusAggregatorConfiguration() { - StorageAccount = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount), - ContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName), - TableName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName), - Environments = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusEnvironment).Split(';'), - MaximumSeverity = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusMaximumSeverity) ?? int.MaxValue, - TeamId = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiTeamId) + StorageAccount = + JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusStorageAccount), + ContainerName = + JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusContainerName), + TableName = + JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusTableName), + Environments = + JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusEnvironment) + .Split(';'), + MaximumSeverity = + JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusMaximumSeverity) + ?? int.MaxValue, + TeamId = + JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiTeamId), + EventStartMessageDelayMinutes = + JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventStartMessageDelayMinutes) + ?? _defaultEventStartMessageDelayMinutes, + EventEndDelayMinutes = + JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventEndDelayMinutes) + ?? _defaultEventEndDelayMinutes, + EventVisibilityPeriodDays = + JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.StatusEventVisibilityPeriodDays) + ?? _defaultEventVisibilityPeriod, }; serviceCollection.AddSingleton(configuration); var incidentApiConfiguration = new IncidentApiConfiguration() { - BaseUri = new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)), - Certificate = GetCertificateFromJson(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificate)) + BaseUri = + new Uri(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiBaseUri)), + Certificate = + GetCertificateFromJson(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatusIncidentApiCertificate)) }; serviceCollection.AddSingleton(incidentApiConfiguration); diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index adc163871..0c28b0575 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -15,15 +15,19 @@ namespace StatusAggregator { public class MessageUpdater : IMessageUpdater { - private static TimeSpan EventStartDelay = TimeSpan.FromMinutes(15); + private readonly TimeSpan _eventStartMessageDelay; private readonly ITableWrapper _table; private readonly ILogger _logger; - public MessageUpdater(ITableWrapper table, ILogger logger) + public MessageUpdater( + ITableWrapper table, + StatusAggregatorConfiguration configuration, + ILogger logger) { _table = table; + _eventStartMessageDelay = TimeSpan.FromMinutes(configuration.EventStartMessageDelayMinutes); _logger = logger; } @@ -31,7 +35,7 @@ public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime c { using (_logger.Scope("Creating message for start of event.")) { - if (cursor > eventEntity.StartTime + EventStartDelay) + if (cursor > eventEntity.StartTime + _eventStartMessageDelay) { if (!_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { diff --git a/src/StatusAggregator/StatusAggregatorConfiguration.cs b/src/StatusAggregator/StatusAggregatorConfiguration.cs index 25bc33f2d..c38179351 100644 --- a/src/StatusAggregator/StatusAggregatorConfiguration.cs +++ b/src/StatusAggregator/StatusAggregatorConfiguration.cs @@ -38,7 +38,26 @@ public class StatusAggregatorConfiguration /// /// A team ID to use to query the incident API. + /// See . /// public string TeamId { get; set; } + + /// + /// The number of minutes that must pass before a message is created for a recently started event. + /// In other words, will wait this amount of time before it creates a start message for an event. + /// + public int EventStartMessageDelayMinutes { get; set; } + + /// + /// The number of minutes that must pass before an event whose incidents have all been mitigated is deactivated. + /// In other words, will wait this amount of time before it deactivates an event with all mitigated incidents. + /// + public int EventEndDelayMinutes { get; set; } + + /// + /// The number of days that a deactivated event is visible in the . + /// An event is only added to by if it is active or it has been deactivated for less than this number of days. + /// + public int EventVisibilityPeriodDays { get; set; } } } diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 35bb87eda..9a462637d 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -19,7 +19,7 @@ namespace StatusAggregator public class StatusExporter : IStatusExporter { private const string StatusBlobName = "status.json"; - private static TimeSpan EventVisibilityPeriod = TimeSpan.FromDays(7); + private readonly TimeSpan _eventVisibilityPeriod; private readonly CloudBlobContainer _container; private readonly ITableWrapper _table; @@ -37,10 +37,12 @@ public class StatusExporter : IStatusExporter public StatusExporter( CloudBlobContainer container, ITableWrapper table, + StatusAggregatorConfiguration configuration, ILogger logger) { _container = container; _table = table; + _eventVisibilityPeriod = TimeSpan.FromDays(configuration.EventVisibilityPeriodDays); _logger = logger; } @@ -54,7 +56,7 @@ public async Task Export() .CreateQuery() .Where(e => e.PartitionKey == EventEntity.DefaultPartitionKey && - (e.IsActive || (e.EndTime >= DateTime.UtcNow - EventVisibilityPeriod))) + (e.IsActive || (e.EndTime >= DateTime.UtcNow - _eventVisibilityPeriod))) .ToList() .Select(e => { diff --git a/tests/StatusAggregator.Tests/EventUpdaterTests.cs b/tests/StatusAggregator.Tests/EventUpdaterTests.cs index e02059d2c..6fc67b896 100644 --- a/tests/StatusAggregator.Tests/EventUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/EventUpdaterTests.cs @@ -24,7 +24,7 @@ public class EventUpdaterTests private static IEnumerable UnclosableIncidents => new[] { - CreateIncidentEntity(NextCreationTime + EventUpdater.EventEndDelay), // Incident closed too recently + CreateIncidentEntity(NextCreationTime + EventUpdater._eventEndDelay), // Incident closed too recently CreateIncidentEntity() // Active incident }; From 20a2fd6a5b50fffd1f0fc89dea706e033fca38d1 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 13:57:33 -0700 Subject: [PATCH 36/49] fix tests --- src/StatusAggregator/IncidentFactory.cs | 1 - tests/StatusAggregator.Tests/EventUpdaterTests.cs | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index f6aa2130d..c2faebe24 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; diff --git a/tests/StatusAggregator.Tests/EventUpdaterTests.cs b/tests/StatusAggregator.Tests/EventUpdaterTests.cs index 6fc67b896..6f7d49eb3 100644 --- a/tests/StatusAggregator.Tests/EventUpdaterTests.cs +++ b/tests/StatusAggregator.Tests/EventUpdaterTests.cs @@ -14,6 +14,8 @@ namespace StatusAggregator.Tests public class EventUpdaterTests { private const string RowKey = "rowkey"; + private const int EventEndDelayMinutes = 5; + private static readonly TimeSpan EventEndDelay = TimeSpan.FromMinutes(EventEndDelayMinutes); private static readonly DateTime NextCreationTime = new DateTime(2018, 7, 10); private static IEnumerable ClosableIncidents => new[] @@ -24,7 +26,7 @@ public class EventUpdaterTests private static IEnumerable UnclosableIncidents => new[] { - CreateIncidentEntity(NextCreationTime + EventUpdater._eventEndDelay), // Incident closed too recently + CreateIncidentEntity(NextCreationTime + EventEndDelay), // Incident closed too recently CreateIncidentEntity() // Active incident }; @@ -35,11 +37,17 @@ public class EventUpdaterTests public EventUpdaterTests() { + var configuration = new StatusAggregatorConfiguration() + { + EventEndDelayMinutes = EventEndDelayMinutes + }; + _tableWrapperMock = new Mock(); _messageUpdaterMock = new Mock(); _eventUpdater = new EventUpdater( _tableWrapperMock.Object, - _messageUpdaterMock.Object, + _messageUpdaterMock.Object, + configuration, Mock.Of>()); _eventEntity = new EventEntity() From ae083231d2605911dcedbf27bdcb1dde4fdbf6ef Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 14:44:45 -0700 Subject: [PATCH 37/49] use latest server common libraries --- src/StatusAggregator/StatusAggregator.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 5b14cae0d..5d220f53c 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -97,13 +97,13 @@ 1.1.1 - 2.28.0-sb-incidents-35734 + 2.28.0-sb-morestatus-35915 - 2.28.0-sb-morestatus-35606 + 2.28.0-sb-morestatus-35915 - 2.28.0-sb-morestatus-35606 + 2.28.0-sb-morestatus-35915 9.2.0 From 23302bd867660178cfd9ad6fb978cd83783a0c7b Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 14:50:36 -0700 Subject: [PATCH 38/49] remove unnecessary skip test --- .../SignatureValidatorIntegrationTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs index 1ba9630c6..742f2d98c 100644 --- a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs +++ b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs @@ -209,8 +209,8 @@ public async Task AcceptsValidSignedPackage() VerifyPackageSigningStatus(result, ValidationStatus.Succeeded, PackageSigningStatus.Valid); Assert.Empty(result.Issues); } - - [Fact(Skip = "This test is flaky, disabling for now.")] + + [Fact(Skip = "Flaky")] public async Task RejectsUntrustedSigningCertificate() { // Arrange From ba9646d9eaf1e7c4bd1f342bee689614bdf98c20 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 14:54:22 -0700 Subject: [PATCH 39/49] improve cursor logging --- src/StatusAggregator/Cursor.cs | 39 ++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index a4804f08e..96e9afb4b 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -4,6 +4,7 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NuGet.Jobs.Extensions; using NuGet.Services.Status.Table; using StatusAggregator.Table; @@ -26,28 +27,34 @@ public Cursor( public async Task Get() { - var cursor = await _table.Retrieve( - CursorEntity.DefaultPartitionKey, CursorEntity.DefaultRowKey); - - DateTime value; - if (cursor == null) - { - value = DateTime.MinValue; - _logger.LogInformation("Could not fetch cursor."); - } - else + using (_logger.Scope("Fetching cursor.")) { - value = cursor.Value; - _logger.LogInformation("Fetched cursor with value {Cursor}.", value); + var cursor = await _table.Retrieve( + CursorEntity.DefaultPartitionKey, CursorEntity.DefaultRowKey); + + DateTime value; + if (cursor == null) + { + value = DateTime.MinValue; + _logger.LogInformation("Could not fetch cursor."); + } + else + { + value = cursor.Value; + _logger.LogInformation("Fetched cursor with value {Cursor}.", value); + } + + return value; } - - return value; } public Task Set(DateTime value) { - var cursorEntity = new CursorEntity(value); - return _table.InsertOrReplaceAsync(cursorEntity); + using (_logger.Scope("Updating cursor to {Cursor}.", value)) + { + var cursorEntity = new CursorEntity(value); + return _table.InsertOrReplaceAsync(cursorEntity); + } } } } From 358ce1d919964394ad268b83926ea310e3a4347d Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 14:57:04 -0700 Subject: [PATCH 40/49] fix tests packages --- tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj index 2f28acd80..39c9b3192 100644 --- a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj +++ b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj @@ -62,13 +62,13 @@ 4.7.145 - 2.28.0-sb-incidents-35734 + 2.28.0-sb-morestatus-35915 - 2.28.0-sb-morestatus-35606 + 2.28.0-sb-morestatus-35915 - 2.28.0-sb-morestatus-35606 + 2.28.0-sb-morestatus-35915 4.3.0 From 9957520d178804496585cf19822d2702a849b4bd Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Wed, 25 Jul 2018 15:20:39 -0700 Subject: [PATCH 41/49] remove unnecessary whitespace change --- src/NuGet.Jobs.Common/JobBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index a36c7cf6c..e16184854 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -35,7 +35,7 @@ public void SetLogger(ILoggerFactory loggerFactory, ILogger logger) LoggerFactory = loggerFactory; Logger = logger; } - + public abstract void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary); public abstract Task Run(); From 97ae1168270c2092adeebe1a19698791cc88452d Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 27 Jul 2018 13:05:45 -0700 Subject: [PATCH 42/49] regex compliancy --- src/StatusAggregator/LogEvents.cs | 9 ++++++++ src/StatusAggregator/Parse/IncidentParser.cs | 22 +++++++++++++++++++- src/StatusAggregator/StatusAggregator.csproj | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/StatusAggregator/LogEvents.cs diff --git a/src/StatusAggregator/LogEvents.cs b/src/StatusAggregator/LogEvents.cs new file mode 100644 index 000000000..0075ca510 --- /dev/null +++ b/src/StatusAggregator/LogEvents.cs @@ -0,0 +1,9 @@ +using Microsoft.Extensions.Logging; + +namespace StatusAggregator +{ + public static class LogEvents + { + public static EventId RegexFailure = new EventId(400, "Failed to parse incident using Regex."); + } +} diff --git a/src/StatusAggregator/Parse/IncidentParser.cs b/src/StatusAggregator/Parse/IncidentParser.cs index 0f6d7cfb5..39fa83e2c 100644 --- a/src/StatusAggregator/Parse/IncidentParser.cs +++ b/src/StatusAggregator/Parse/IncidentParser.cs @@ -5,6 +5,7 @@ using NuGet.Jobs.Extensions; using NuGet.Services.Incidents; using NuGet.Services.Status; +using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; @@ -16,6 +17,8 @@ namespace StatusAggregator.Parse /// public abstract class IncidentParser : IIncidentParser { + private readonly static TimeSpan MaxRegexExecutionTime = TimeSpan.FromSeconds(5); + private readonly string _regExPattern; private readonly IEnumerable _filters; @@ -47,7 +50,24 @@ public bool TryParseIncident(Incident incident, out ParsedIncident parsedInciden { parsedIncident = null; var title = incident.Title; - var match = Regex.Match(title, _regExPattern); + + Match match = null; + try + { + match = Regex.Match(title, _regExPattern, RegexOptions.None, MaxRegexExecutionTime); + } + catch (Exception e) + { + _logger.LogError(LogEvents.RegexFailure, e, "Failed to parse incident using regex!"); + return false; + } + + if (match == null) + { + _logger.LogError("Parsed incident using regex successfully, but was unable to get match information!"); + return false; + } + _logger.LogInformation("Incident title is {IncidentTitle}, RegEx match result: {MatchResult}", title, match.Success); return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 5d220f53c..c4df719d0 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -47,6 +47,7 @@ + From 40e34cdb58d29f16c1e33ebf0e9a9459e8002bca Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 27 Jul 2018 13:21:19 -0700 Subject: [PATCH 43/49] other pr fixes --- src/StatusAggregator/IncidentUpdater.cs | 11 +++++++---- src/StatusAggregator/Program.cs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index f60ea1db8..01886c1a2 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -92,11 +92,14 @@ public async Task RefreshActiveIncidents() private string GetRecentIncidentsQuery(DateTime cursor) { - var cursorPart = cursor == DateTime.MinValue - ? "" - : $" and CreateDate gt datetime'{cursor.ToString("o")}'"; + var query = $"$filter = OwningTeamId eq '{_incidentApiTeamId}'"; - return $"$filter=OwningTeamId eq '{_incidentApiTeamId}'{cursorPart}"; + if (cursor != DateTime.MinValue) + { + query += $" and CreateDate gt datetime'{cursor.ToString("o")}'"; + } + + return query; } } } diff --git a/src/StatusAggregator/Program.cs b/src/StatusAggregator/Program.cs index ce9117e0d..127275b30 100644 --- a/src/StatusAggregator/Program.cs +++ b/src/StatusAggregator/Program.cs @@ -10,7 +10,7 @@ public class Program public static void Main(string[] args) { var job = new Job(); - JobRunner.Run(job, args).Wait(); + JobRunner.Run(job, args).GetAwaiter().GetResult(); } } } From 83d18a942fd222fdfbf83126a40c8173d3b8ba6a Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 27 Jul 2018 16:22:26 -0700 Subject: [PATCH 44/49] simplify message logic --- .../Configuration/JobConfigurationManager.cs | 3 +- .../Extensions/ServiceProviderExtensions.cs | 12 - .../NuGet.Jobs.Common.csproj | 1 - src/StatusAggregator/IncidentUpdater.cs | 5 +- src/StatusAggregator/Job.cs | 2 +- src/StatusAggregator/MessageUpdater.cs | 205 +++++++----------- 6 files changed, 89 insertions(+), 139 deletions(-) delete mode 100644 src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs diff --git a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs index 998635cd5..8fb816c6d 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.ComponentModel.Design; using System.Linq; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; using NuGet.Services.Configuration; @@ -198,7 +199,7 @@ private static Dictionary ReadCommandLineArguments(ILogger logge private static IDictionary InjectSecrets(IServiceContainer serviceContainer, Dictionary argsDictionary) { - var secretReaderFactory = serviceContainer.GetService(); + var secretReaderFactory = serviceContainer.GetRequiredService(); var secretReader = secretReaderFactory.CreateSecretReader(argsDictionary); if (secretReader == null) diff --git a/src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs deleted file mode 100644 index ee3a37f70..000000000 --- a/src/NuGet.Jobs.Common/Extensions/ServiceProviderExtensions.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; - -namespace NuGet.Jobs.Extensions -{ - public static class ServiceProviderExtensions - { - public static T GetService(this IServiceProvider provider) - { - return (T)provider.GetService(typeof(T)); - } - } -} diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index b792ac7eb..a33533479 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -50,7 +50,6 @@ - diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index 01886c1a2..4e38613d5 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -76,7 +76,8 @@ public async Task RefreshActiveIncidents() // The incident API trims the milliseconds from any filter. // Therefore, a query asking for incidents newer than '2018-06-29T00:00:00.5Z' will return an incident from '2018-06-29T00:00:00.25Z' // We must perform a check on the CreateDate ourselves to verify that no old incidents are returned. - .Where(i => i.CreateDate > cursor); + .Where(i => i.CreateDate > cursor) + .ToList(); var parsedIncidents = incidents .SelectMany(i => _aggregateIncidentParser.ParseIncident(i)) @@ -92,7 +93,7 @@ public async Task RefreshActiveIncidents() private string GetRecentIncidentsQuery(DateTime cursor) { - var query = $"$filter = OwningTeamId eq '{_incidentApiTeamId}'"; + var query = $"$filter=OwningTeamId eq '{_incidentApiTeamId}'"; if (cursor != DateTime.MinValue) { diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index 71ddbffbd..c1b8a4c9c 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -35,7 +35,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary() + .GetRequiredService() .Run(); } diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 0c28b0575..2dd27d0ea 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -35,73 +35,37 @@ public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime c { using (_logger.Scope("Creating message for start of event.")) { - if (cursor > eventEntity.StartTime + _eventStartMessageDelay) + if (cursor <= eventEntity.StartTime + _eventStartMessageDelay) { - if (!_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) - { - if (TryGetContentsForEventStartForEvent(eventEntity, out var contents)) - { - await CreateMessage(eventEntity, eventEntity.StartTime, contents); - } - else - { - _logger.LogWarning("Failed to create a message for start of event!"); - } - } - else - { - // If we've already told customers about an event, we don't need to tell them about it again. - _logger.LogInformation("Event has messages associated with it, cannot create message for its start."); - } + // We don't want to show events that are too recent to avoid noisy events. + _logger.LogInformation("Event is too recent, cannot create message for its start."); + return; + } + + if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) + { + // If we've already told customers about an event, we don't need to tell them about it again. + _logger.LogInformation("Event has messages associated with it, cannot create message for its start."); + return; + } + + if (TryGetContentsForMessageForEventStart(eventEntity, out var contents)) + { + await CreateMessage(eventEntity, eventEntity.StartTime, contents); } else { - // We don't want to show events that are too recent to avoid noisy events. - _logger.LogInformation("Event is too recent, cannot create message for its start."); + _logger.LogWarning("Failed to create a message for start of event!"); } } } - private bool TryGetContentsForEventStartForEvent(EventEntity eventEntity, out string contents) - { - return TryGetContentsForEventHelper(eventEntity, _innerMessageMapForEventStart, "is", "", out contents); - } + private const string _messageForEventStartTemplate = "{0} is {1}. You may encounter issues {2}."; - private const string _youMayEncounterIssues = "You may encounter issues "; - private static readonly IEnumerable _innerMessageMapForEventStart = new InnerMessageForComponentPathPrefix[] + private bool TryGetContentsForMessageForEventStart(EventEntity eventEntity, out string contents) { - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.GalleryName), - $"{_youMayEncounterIssues}browsing the NuGet Gallery."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName), - $"{_youMayEncounterIssues}restoring packages from NuGet.org's V3 feed from China."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName), - $"{_youMayEncounterIssues}restoring packages from NuGet.org's V3 feed."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName), - $"{_youMayEncounterIssues}restoring packages from NuGet.org's V2 feed."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName), - $"{_youMayEncounterIssues}restoring packages."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName), - $"{_youMayEncounterIssues}searching for packages from China."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName), - $"{_youMayEncounterIssues}searching for packages."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName), - "New packages will likely take longer than usual before becoming available for download."), - }; + return TryGetContentsForEventHelper(eventEntity, _messageForEventStartTemplate, out contents); + } public async Task CreateMessageForEventEnd(EventEntity eventEntity) { @@ -112,21 +76,20 @@ public async Task CreateMessageForEventEnd(EventEntity eventEntity) using (_logger.Scope("Creating message for end of event.")) { - if (_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) + if (!_table.GetMessagesLinkedToEvent(eventEntity).ToList().Any()) { - if (TryGetContentsForEventEndForEvent(eventEntity, out var contents)) - { - await CreateMessage(eventEntity, eventEntity.EndTime.Value, contents); - } - else - { - _logger.LogWarning("Failed to create message!"); - } + // If we've never told customers about an event, we don't need to tell them it's no longer impacting them. + _logger.LogInformation("Event has no messages associated with it, cannot create message for its end."); + return; + } + + if (TryGetContentsForMessageForEventEnd(eventEntity, out var contents)) + { + await CreateMessage(eventEntity, eventEntity.EndTime.Value, contents); } else { - // If we've never told customers about an event, we don't need to tell them it's no longer impacting them. - _logger.LogInformation("Event has no messages associated with it, cannot create message for its end."); + _logger.LogWarning("Failed to create message!"); } } } @@ -139,53 +102,16 @@ private Task CreateMessage(EventEntity eventEntity, DateTime time, string conten return _table.InsertOrReplaceAsync(messageEntity); } - private bool TryGetContentsForEventEndForEvent(EventEntity eventEntity, out string contents) - { - return TryGetContentsForEventHelper(eventEntity, _innerMessageMapForEventEnd, "is no longer", " Thank you for your patience.", out contents); - } + private const string _messageForEventEndTemplate = "{0} is no longer {1}. You should no longer encounter any issues {2}. Thank you for your patience."; - private const string _youShouldNoLongerEncounterAnyIssues = "You should no longer encounter any issues "; - private static readonly IEnumerable _innerMessageMapForEventEnd = new InnerMessageForComponentPathPrefix[] + private bool TryGetContentsForMessageForEventEnd(EventEntity eventEntity, out string contents) { - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.GalleryName), - $"{_youShouldNoLongerEncounterAnyIssues}browsing the NuGet Gallery."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName), - $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V3 feed from China."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName), - $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V3 feed."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName), - $"{_youShouldNoLongerEncounterAnyIssues}restoring packages from NuGet.org's V2 feed."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName), - $"{_youShouldNoLongerEncounterAnyIssues}restoring packages."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName), - $"{_youShouldNoLongerEncounterAnyIssues}searching for packages from China."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName), - $"{_youShouldNoLongerEncounterAnyIssues}searching for packages."), - - new InnerMessageForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName), - "New packages should become available for download as quickly as usual."), - }; - - private static string[] _componentStatusNames = Enum.GetNames(typeof(ComponentStatus)); + return TryGetContentsForEventHelper(eventEntity, _messageForEventEndTemplate, out contents); + } + private bool TryGetContentsForEventHelper( EventEntity eventEntity, - IEnumerable innerMessageMap, - string boldedPartInnerString, - string messageSuffix, + string messageTemplate, out string contents) { contents = null; @@ -199,33 +125,68 @@ private bool TryGetContentsForEventHelper( } var componentNames = path.Split(Constants.ComponentPathDivider); - var name = string.Join(" ", componentNames.Skip(1).Reverse()); - var boldedPart = $"{name} {boldedPartInnerString} {_componentStatusNames[eventEntity.AffectedComponentStatus].ToLowerInvariant()}."; + var componentName = string.Join(" ", componentNames.Skip(1).Reverse()); + var componentStatus = ((ComponentStatus)eventEntity.AffectedComponentStatus).ToString().ToLowerInvariant(); - string innerContents = innerMessageMap + string actionDescription = _actionDescriptionForComponentPathMap .FirstOrDefault(m => m.Matches(path))? - .InnerMessage; + .ActionDescription; - if (innerContents == null) + if (actionDescription == null) { - _logger.LogWarning("Could not find an inner message for path {ComponentPath}.", path); + _logger.LogWarning("Could not find an action description for path {ComponentPath}.", path); return false; } - contents = $"{boldedPart} {innerContents}{messageSuffix}"; + contents = string.Format(messageTemplate, componentName, componentStatus, actionDescription); return !string.IsNullOrEmpty(contents); } - private class InnerMessageForComponentPathPrefix + private static readonly IEnumerable _actionDescriptionForComponentPathMap = new ActionDescriptionForComponentPathPrefix[] + { + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.GalleryName), + $"browsing the NuGet Gallery"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName), + $"restoring packages from NuGet.org's V3 feed from China"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName), + $"restoring packages from NuGet.org's V3 feed"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName), + $"restoring packages from NuGet.org's V2 feed"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName), + $"restoring packages"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName), + $"searching for packages from China"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName), + $"searching for packages"), + + new ActionDescriptionForComponentPathPrefix( + ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName), + "uploading new packages"), + }; + + private class ActionDescriptionForComponentPathPrefix { public string ComponentPathPrefix { get; } - public string InnerMessage { get; } + public string ActionDescription { get; } - public InnerMessageForComponentPathPrefix(string componentPathPrefix, string innerWarning) + public ActionDescriptionForComponentPathPrefix(string componentPathPrefix, string actionDescription) { ComponentPathPrefix = componentPathPrefix; - InnerMessage = innerWarning; + ActionDescription = actionDescription; } public bool Matches(string componentPath) From fd671572d630ab43cff0f33c22adc4dbf699b649 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Fri, 27 Jul 2018 16:46:34 -0700 Subject: [PATCH 45/49] argument null exception --- src/StatusAggregator/Cursor.cs | 4 ++-- src/StatusAggregator/EventUpdater.cs | 8 ++++---- src/StatusAggregator/IncidentFactory.cs | 7 ++++--- src/StatusAggregator/IncidentUpdater.cs | 14 +++++++------- src/StatusAggregator/MessageUpdater.cs | 6 +++--- .../Parse/AggregateIncidentParser.cs | 5 +++-- src/StatusAggregator/Parse/EnvironmentFilter.cs | 4 ++-- src/StatusAggregator/Parse/IncidentParser.cs | 6 +++--- src/StatusAggregator/Parse/ParsedIncident.cs | 5 +++++ .../Parse/PingdomIncidentParser.cs | 3 ++- src/StatusAggregator/Parse/SeverityFilter.cs | 5 +++-- src/StatusAggregator/StatusAggregator.cs | 9 +++++---- src/StatusAggregator/StatusExporter.cs | 8 ++++---- src/StatusAggregator/StatusUpdater.cs | 8 ++++---- 14 files changed, 51 insertions(+), 41 deletions(-) diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index 96e9afb4b..d1ebc3924 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -17,8 +17,8 @@ public Cursor( ITableWrapper table, ILogger logger) { - _table = table; - _logger = logger; + _table = table ?? throw new ArgumentNullException(nameof(table)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } private readonly ITableWrapper _table; diff --git a/src/StatusAggregator/EventUpdater.cs b/src/StatusAggregator/EventUpdater.cs index 915e88bf6..b5d320f59 100644 --- a/src/StatusAggregator/EventUpdater.cs +++ b/src/StatusAggregator/EventUpdater.cs @@ -26,10 +26,10 @@ public EventUpdater( StatusAggregatorConfiguration configuration, ILogger logger) { - _table = table; - _messageUpdater = messageUpdater; - _eventEndDelay = TimeSpan.FromMinutes(configuration.EventEndDelayMinutes); - _logger = logger; + _table = table ?? throw new ArgumentNullException(nameof(table)); + _messageUpdater = messageUpdater ?? throw new ArgumentNullException(nameof(messageUpdater)); + _eventEndDelay = TimeSpan.FromMinutes(configuration?.EventEndDelayMinutes ?? throw new ArgumentNullException(nameof(configuration))); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task UpdateActiveEvents(DateTime cursor) diff --git a/src/StatusAggregator/IncidentFactory.cs b/src/StatusAggregator/IncidentFactory.cs index c2faebe24..8fd1399e8 100644 --- a/src/StatusAggregator/IncidentFactory.cs +++ b/src/StatusAggregator/IncidentFactory.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -23,9 +24,9 @@ public IncidentFactory( IEventUpdater eventUpdater, ILogger logger) { - _table = table; - _eventUpdater = eventUpdater; - _logger = logger; + _table = table ?? throw new ArgumentNullException(nameof(table)); + _eventUpdater = eventUpdater ?? throw new ArgumentNullException(nameof(eventUpdater)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task CreateIncident(ParsedIncident parsedIncident) diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index 4e38613d5..3a1f970e7 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -36,13 +36,13 @@ public IncidentUpdater( StatusAggregatorConfiguration configuration, ILogger logger) { - _table = table; - _eventUpdater = eventUpdater; - _incidentApiClient = incidentApiClient; - _aggregateIncidentParser = aggregateIncidentParser; - _incidentFactory = incidentFactory; - _incidentApiTeamId = configuration.TeamId; - _logger = logger; + _table = table ?? throw new ArgumentNullException(nameof(table)); + _eventUpdater = eventUpdater ?? throw new ArgumentNullException(nameof(eventUpdater)); + _incidentApiClient = incidentApiClient ?? throw new ArgumentNullException(nameof(incidentApiClient)); + _aggregateIncidentParser = aggregateIncidentParser ?? throw new ArgumentNullException(nameof(aggregateIncidentParser)); + _incidentFactory = incidentFactory ?? throw new ArgumentNullException(nameof(incidentFactory)); + _incidentApiTeamId = configuration?.TeamId ?? throw new ArgumentNullException(nameof(configuration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task RefreshActiveIncidents() diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index 2dd27d0ea..da683333d 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -26,9 +26,9 @@ public MessageUpdater( StatusAggregatorConfiguration configuration, ILogger logger) { - _table = table; - _eventStartMessageDelay = TimeSpan.FromMinutes(configuration.EventStartMessageDelayMinutes); - _logger = logger; + _table = table ?? throw new ArgumentNullException(nameof(table)); + _eventStartMessageDelay = TimeSpan.FromMinutes(configuration?.EventStartMessageDelayMinutes ?? throw new ArgumentNullException(nameof(configuration))); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime cursor) diff --git a/src/StatusAggregator/Parse/AggregateIncidentParser.cs b/src/StatusAggregator/Parse/AggregateIncidentParser.cs index c3ebb616a..934714bcc 100644 --- a/src/StatusAggregator/Parse/AggregateIncidentParser.cs +++ b/src/StatusAggregator/Parse/AggregateIncidentParser.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using NuGet.Jobs.Extensions; using NuGet.Services.Incidents; +using System; using System.Collections.Generic; namespace StatusAggregator.Parse @@ -21,8 +22,8 @@ public AggregateIncidentParser( IEnumerable incidentParsers, ILogger logger) { - _incidentParsers = incidentParsers; - _logger = logger; + _incidentParsers = incidentParsers ?? throw new ArgumentNullException(nameof(incidentParsers)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public IEnumerable ParseIncident(Incident incident) diff --git a/src/StatusAggregator/Parse/EnvironmentFilter.cs b/src/StatusAggregator/Parse/EnvironmentFilter.cs index 9e4eeb88c..bf5943496 100644 --- a/src/StatusAggregator/Parse/EnvironmentFilter.cs +++ b/src/StatusAggregator/Parse/EnvironmentFilter.cs @@ -26,8 +26,8 @@ public EnvironmentFilter( StatusAggregatorConfiguration configuration, ILogger logger) { - _environments = configuration.Environments; - _logger = logger; + _environments = configuration?.Environments ?? throw new ArgumentNullException(nameof(configuration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public bool ShouldParse(Incident incident, GroupCollection groups) diff --git a/src/StatusAggregator/Parse/IncidentParser.cs b/src/StatusAggregator/Parse/IncidentParser.cs index 39fa83e2c..b6e39deed 100644 --- a/src/StatusAggregator/Parse/IncidentParser.cs +++ b/src/StatusAggregator/Parse/IncidentParser.cs @@ -29,9 +29,9 @@ public IncidentParser( string regExPattern, ILogger logger) { - _regExPattern = regExPattern; + _regExPattern = regExPattern ?? throw new ArgumentNullException(nameof(regExPattern)); _filters = Enumerable.Empty(); - _logger = logger; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public IncidentParser( @@ -40,7 +40,7 @@ public IncidentParser( ILogger logger) : this(regExPattern, logger) { - _filters = filters.ToList(); + _filters = filters?.ToList() ?? throw new ArgumentNullException(nameof(filters)); } public bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident) diff --git a/src/StatusAggregator/Parse/ParsedIncident.cs b/src/StatusAggregator/Parse/ParsedIncident.cs index 685d9d934..4ce85ad10 100644 --- a/src/StatusAggregator/Parse/ParsedIncident.cs +++ b/src/StatusAggregator/Parse/ParsedIncident.cs @@ -17,6 +17,11 @@ public ParsedIncident( string affectedComponentPath, ComponentStatus affectedComponentStatus) { + if (incident == null) + { + throw new ArgumentNullException(nameof(incident)); + } + Id = incident.Id; CreationTime = incident.Source.CreateDate; MitigationTime = incident.MitigationData?.Date; diff --git a/src/StatusAggregator/Parse/PingdomIncidentParser.cs b/src/StatusAggregator/Parse/PingdomIncidentParser.cs index 5b5130802..2278edd09 100644 --- a/src/StatusAggregator/Parse/PingdomIncidentParser.cs +++ b/src/StatusAggregator/Parse/PingdomIncidentParser.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; @@ -22,7 +23,7 @@ public PingdomIncidentParser( ILogger logger) : base(SubtitleRegEx, filters, logger) { - _logger = logger; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) diff --git a/src/StatusAggregator/Parse/SeverityFilter.cs b/src/StatusAggregator/Parse/SeverityFilter.cs index d8bc63b32..53346e5fc 100644 --- a/src/StatusAggregator/Parse/SeverityFilter.cs +++ b/src/StatusAggregator/Parse/SeverityFilter.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging; using NuGet.Services.Incidents; +using System; using System.Text.RegularExpressions; namespace StatusAggregator.Parse @@ -20,8 +21,8 @@ public SeverityFilter( StatusAggregatorConfiguration configuration, ILogger logger) { - _maximumSeverity = configuration.MaximumSeverity; - _logger = logger; + _maximumSeverity = configuration?.MaximumSeverity ?? throw new ArgumentNullException(nameof(configuration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public bool ShouldParse(Incident incident, GroupCollection groups) diff --git a/src/StatusAggregator/StatusAggregator.cs b/src/StatusAggregator/StatusAggregator.cs index a2327ed8d..03b650877 100644 --- a/src/StatusAggregator/StatusAggregator.cs +++ b/src/StatusAggregator/StatusAggregator.cs @@ -3,6 +3,7 @@ using Microsoft.WindowsAzure.Storage.Blob; using StatusAggregator.Table; +using System; using System.Threading.Tasks; namespace StatusAggregator @@ -21,10 +22,10 @@ public StatusAggregator( IStatusUpdater statusUpdater, IStatusExporter statusExporter) { - _container = container; - _table = table; - _statusUpdater = statusUpdater; - _statusExporter = statusExporter; + _container = container ?? throw new ArgumentNullException(nameof(container)); + _table = table ?? throw new ArgumentNullException(nameof(table)); + _statusUpdater = statusUpdater ?? throw new ArgumentNullException(nameof(statusUpdater)); + _statusExporter = statusExporter ?? throw new ArgumentNullException(nameof(statusExporter)); } public async Task Run() diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index 9a462637d..ec2d4a98d 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -40,10 +40,10 @@ public StatusExporter( StatusAggregatorConfiguration configuration, ILogger logger) { - _container = container; - _table = table; - _eventVisibilityPeriod = TimeSpan.FromDays(configuration.EventVisibilityPeriodDays); - _logger = logger; + _container = container ?? throw new ArgumentNullException(nameof(container)); + _table = table ?? throw new ArgumentNullException(nameof(table)); + _eventVisibilityPeriod = TimeSpan.FromDays(configuration?.EventVisibilityPeriodDays ?? throw new ArgumentNullException(nameof(configuration))); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task Export() diff --git a/src/StatusAggregator/StatusUpdater.cs b/src/StatusAggregator/StatusUpdater.cs index 5b7d645f7..c6be8d687 100644 --- a/src/StatusAggregator/StatusUpdater.cs +++ b/src/StatusAggregator/StatusUpdater.cs @@ -22,10 +22,10 @@ public StatusUpdater( IEventUpdater eventUpdater, ILogger logger) { - _cursor = cursor; - _incidentUpdater = incidentUpdater; - _eventUpdater = eventUpdater; - _logger = logger; + _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); + _incidentUpdater = incidentUpdater ?? throw new ArgumentNullException(nameof(IncidentUpdater)); + _eventUpdater = eventUpdater ?? throw new ArgumentNullException(nameof(eventUpdater)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task Update() From 4dd79330d2b0efb1fb43f81bc4f2873995a6c1be Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 30 Jul 2018 09:37:44 -0700 Subject: [PATCH 46/49] remove try get enum --- .../Configuration/JobConfigurationManager.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs index 8fb816c6d..fcaeeb51c 100644 --- a/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs +++ b/src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs @@ -128,13 +128,6 @@ public static bool TryGetBoolArgument(IDictionary jobArgsDiction return null; } - public static T TryGetEnumArgument(IDictionary jobArgsDictionary, string argName, T defaultValue) - where T : struct - { - var argumentString = TryGetArgument(jobArgsDictionary, argName); - return !string.IsNullOrEmpty(argumentString) && Enum.TryParse(argumentString, out T result) ? result : defaultValue; - } - private static Dictionary ReadCommandLineArguments(ILogger logger, string[] commandLineArgs) { var argsDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); From 287ea8b7e01773d384e3c4c72974a2b41abedd46 Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 30 Jul 2018 10:02:56 -0700 Subject: [PATCH 47/49] add back statusaggregator tests --- test.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test.ps1 b/test.ps1 index f49773427..c7ccc7f08 100644 --- a/test.ps1 +++ b/test.ps1 @@ -38,6 +38,7 @@ Function Run-Tests { "tests\Validation.PackageSigning.RevalidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.RevalidateCertificate.Tests.dll", ` "tests\Validation.PackageSigning.Core.Tests\bin\$Configuration\Validation.PackageSigning.Core.Tests.dll", ` "tests\Validation.Common.Job.Tests\bin\$Configuration\Validation.Common.Job.Tests.dll", ` + "tests\StatusAggregator\bin\$Configuration\StatusAggregator.dll", ` "tests\Monitoring.RebootSearchInstance.Tests\bin\$Configuration\NuGet.Monitoring.RebootSearchInstance.Tests.dll", ` "tests\Validation.Symbols.Tests\bin\$Configuration\Validation.Symbols.Tests.dll" @@ -88,4 +89,4 @@ if ($BuildErrors) { Error-Log "Tests completed with $($BuildErrors.Count) error(s):`r`n$($ErrorLines -join "`r`n")" -Fatal } -Write-Host ("`r`n" * 3) \ No newline at end of file +Write-Host ("`r`n" * 3) From 33e8f41c21c13e37cd421aa4086d3f07e931066b Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 30 Jul 2018 10:33:00 -0700 Subject: [PATCH 48/49] this one's for u loic --- .../Extensions/LoggerExtensions.cs | 3 +++ src/StatusAggregator/Cursor.cs | 4 ++-- src/StatusAggregator/IStatusExporter.cs | 2 +- src/StatusAggregator/IncidentUpdater.cs | 5 +--- src/StatusAggregator/MessageUpdater.cs | 18 +++++++------- ...ory.cs => NuGetServiceComponentFactory.cs} | 7 ++++-- ...atedSearchServiceInstanceIncidentParser.cs | 2 +- .../Parse/PingdomIncidentParser.cs | 24 +++++++++---------- .../Parse/ValidationDurationIncidentParser.cs | 2 +- src/StatusAggregator/StatusAggregator.csproj | 2 +- src/StatusAggregator/StatusExporter.cs | 9 ++++--- 11 files changed, 42 insertions(+), 36 deletions(-) rename src/StatusAggregator/{ComponentFactory.cs => NuGetServiceComponentFactory.cs} (93%) diff --git a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs index c34ecf1c4..2276b06bb 100644 --- a/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs +++ b/src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs @@ -8,6 +8,9 @@ namespace NuGet.Jobs.Extensions { public static class LoggerExtensions { + /// + /// Calls and logs a message when entering and leaving the scope. + /// public static IDisposable Scope( this ILogger logger, string message, diff --git a/src/StatusAggregator/Cursor.cs b/src/StatusAggregator/Cursor.cs index d1ebc3924..dd1bc25af 100644 --- a/src/StatusAggregator/Cursor.cs +++ b/src/StatusAggregator/Cursor.cs @@ -8,7 +8,6 @@ using NuGet.Services.Status.Table; using StatusAggregator.Table; - namespace StatusAggregator { public class Cursor : ICursor @@ -35,8 +34,9 @@ public async Task Get() DateTime value; if (cursor == null) { + // If we can't find a cursor, the job is likely uninitialized, so start at the beginning of time. value = DateTime.MinValue; - _logger.LogInformation("Could not fetch cursor."); + _logger.LogInformation("Could not fetch cursor, reinitializing cursor at {Cursor}.", value); } else { diff --git a/src/StatusAggregator/IStatusExporter.cs b/src/StatusAggregator/IStatusExporter.cs index 54af859d3..0ed21b316 100644 --- a/src/StatusAggregator/IStatusExporter.cs +++ b/src/StatusAggregator/IStatusExporter.cs @@ -11,6 +11,6 @@ public interface IStatusExporter /// /// Builds a and exports it to public storage so that it can be consumed by other services. /// - Task Export(); + Task Export(); } } diff --git a/src/StatusAggregator/IncidentUpdater.cs b/src/StatusAggregator/IncidentUpdater.cs index 3a1f970e7..c67aba462 100644 --- a/src/StatusAggregator/IncidentUpdater.cs +++ b/src/StatusAggregator/IncidentUpdater.cs @@ -16,17 +16,14 @@ namespace StatusAggregator public class IncidentUpdater : IIncidentUpdater { private readonly ITableWrapper _table; - private readonly IEventUpdater _eventUpdater; - private readonly IAggregateIncidentParser _aggregateIncidentParser; private readonly IIncidentApiClient _incidentApiClient; private readonly IIncidentFactory _incidentFactory; + private readonly ILogger _logger; private readonly string _incidentApiTeamId; - private readonly ILogger _logger; - public IncidentUpdater( ITableWrapper table, IEventUpdater eventUpdater, diff --git a/src/StatusAggregator/MessageUpdater.cs b/src/StatusAggregator/MessageUpdater.cs index da683333d..7c4900c2d 100644 --- a/src/StatusAggregator/MessageUpdater.cs +++ b/src/StatusAggregator/MessageUpdater.cs @@ -117,7 +117,7 @@ private bool TryGetContentsForEventHelper( contents = null; var path = eventEntity.AffectedComponentPath; - var component = ComponentFactory.CreateNuGetServiceRootComponent().GetByPath(path); + var component = NuGetServiceComponentFactory.CreateNuGetServiceRootComponent().GetByPath(path); if (component == null) { _logger.LogWarning("Could not find a component with path {ComponentPath}.", path); @@ -146,35 +146,35 @@ private bool TryGetContentsForEventHelper( private static readonly IEnumerable _actionDescriptionForComponentPathMap = new ActionDescriptionForComponentPathPrefix[] { new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.GalleryName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName), $"browsing the NuGet Gallery"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.ChinaRegionName), $"restoring packages from NuGet.org's V3 feed from China"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName), $"restoring packages from NuGet.org's V3 feed"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName), $"restoring packages from NuGet.org's V2 feed"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.RestoreName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName), $"restoring packages"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName), $"searching for packages from China"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.SearchName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName), $"searching for packages"), new ActionDescriptionForComponentPathPrefix( - ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName), + ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName), "uploading new packages"), }; diff --git a/src/StatusAggregator/ComponentFactory.cs b/src/StatusAggregator/NuGetServiceComponentFactory.cs similarity index 93% rename from src/StatusAggregator/ComponentFactory.cs rename to src/StatusAggregator/NuGetServiceComponentFactory.cs index 8bbae6412..9d7407004 100644 --- a/src/StatusAggregator/ComponentFactory.cs +++ b/src/StatusAggregator/NuGetServiceComponentFactory.cs @@ -5,7 +5,10 @@ namespace StatusAggregator { - public static class ComponentFactory + /// + /// Helps create an that represents the NuGet service as well as paths to its subcomponents. + /// + public static class NuGetServiceComponentFactory { public const string RootName = "NuGet"; public const string GalleryName = "NuGet.org"; @@ -25,7 +28,7 @@ public static class ComponentFactory public const string SeaInstanceName = "Southeast Asia"; /// - /// Creates the NuGet service root component. + /// Creates an that represents the NuGet service. /// public static IComponent CreateNuGetServiceRootComponent() { diff --git a/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs b/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs index a60c0792f..f0843d41c 100644 --- a/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs +++ b/src/StatusAggregator/Parse/OutdatedSearchServiceInstanceIncidentParser.cs @@ -22,7 +22,7 @@ public OutdatedSearchServiceInstanceIncidentParser( protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName); + affectedComponentPath = ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName); return true; } diff --git a/src/StatusAggregator/Parse/PingdomIncidentParser.cs b/src/StatusAggregator/Parse/PingdomIncidentParser.cs index 2278edd09..4f202cf9f 100644 --- a/src/StatusAggregator/Parse/PingdomIncidentParser.cs +++ b/src/StatusAggregator/Parse/PingdomIncidentParser.cs @@ -37,56 +37,56 @@ protected override bool TryParseAffectedComponentPath(Incident incident, GroupCo { case "CDN DNS": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName); break; case "CDN Global": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.GlobalRegionName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.GlobalRegionName); break; case "CDN China": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V3ProtocolName, ComponentFactory.ChinaRegionName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V3ProtocolName, NuGetServiceComponentFactory.ChinaRegionName); break; case "Gallery DNS": case "Gallery Home": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.GalleryName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName); break; case "Gallery USNC /": case "Gallery USNC /Packages": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.GalleryName, ComponentFactory.UsncInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName, NuGetServiceComponentFactory.UsncInstanceName); break; case "Gallery USSC /": case "Gallery USSC /Packages": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.GalleryName, ComponentFactory.UsscInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.GalleryName, NuGetServiceComponentFactory.UsscInstanceName); break; case "Gallery USNC /api/v2/Packages()": case "Gallery USNC /api/v2/package/NuGet.GalleryUptime/1.0.0": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName, ComponentFactory.UsncInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName, NuGetServiceComponentFactory.UsncInstanceName); break; case "Gallery USSC /api/v2/Packages()": case "Gallery USSC /api/v2/package/NuGet.GalleryUptime/1.0.0": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.RestoreName, ComponentFactory.V2ProtocolName, ComponentFactory.UsscInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.RestoreName, NuGetServiceComponentFactory.V2ProtocolName, NuGetServiceComponentFactory.UsscInstanceName); break; case "Search USNC /query": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.GlobalRegionName, ComponentFactory.UsncInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.GlobalRegionName, NuGetServiceComponentFactory.UsncInstanceName); break; case "Search USSC /query": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.GlobalRegionName, ComponentFactory.UsscInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.GlobalRegionName, NuGetServiceComponentFactory.UsscInstanceName); break; case "Search EA /query": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName, ComponentFactory.EaInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName, NuGetServiceComponentFactory.EaInstanceName); break; case "Search SEA /query": affectedComponentPath = ComponentUtility.GetPath( - ComponentFactory.RootName, ComponentFactory.SearchName, ComponentFactory.ChinaRegionName, ComponentFactory.SeaInstanceName); + NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.SearchName, NuGetServiceComponentFactory.ChinaRegionName, NuGetServiceComponentFactory.SeaInstanceName); break; default: return false; diff --git a/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs b/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs index 8877e8103..8d9d80480 100644 --- a/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs +++ b/src/StatusAggregator/Parse/ValidationDurationIncidentParser.cs @@ -22,7 +22,7 @@ public ValidationDurationIncidentParser( protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) { - affectedComponentPath = ComponentUtility.GetPath(ComponentFactory.RootName, ComponentFactory.UploadName); + affectedComponentPath = ComponentUtility.GetPath(NuGetServiceComponentFactory.RootName, NuGetServiceComponentFactory.UploadName); return true; } diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index c4df719d0..00eba2850 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -46,7 +46,7 @@ - + diff --git a/src/StatusAggregator/StatusExporter.cs b/src/StatusAggregator/StatusExporter.cs index ec2d4a98d..8ebc12082 100644 --- a/src/StatusAggregator/StatusExporter.cs +++ b/src/StatusAggregator/StatusExporter.cs @@ -46,11 +46,11 @@ public StatusExporter( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task Export() + public async Task Export() { using (_logger.Scope("Exporting service status.")) { - var rootComponent = ComponentFactory.CreateNuGetServiceRootComponent(); + var rootComponent = NuGetServiceComponentFactory.CreateNuGetServiceRootComponent(); var recentEvents = _table .CreateQuery() @@ -90,10 +90,11 @@ public async Task Export() } } + ServiceStatus status; string statusJson; using (_logger.Scope("Serializing service status.")) { - var status = new ServiceStatus(rootComponent, recentEvents); + status = new ServiceStatus(rootComponent, recentEvents); statusJson = JsonConvert.SerializeObject(status, _statusBlobJsonSerializerSettings); } @@ -102,6 +103,8 @@ public async Task Export() var blob = _container.GetBlockBlobReference(StatusBlobName); await blob.UploadTextAsync(statusJson); } + + return status; } } } From 33bce6648645ed1a70c5bea8b92a4ea37c42fc2c Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 30 Jul 2018 11:17:38 -0700 Subject: [PATCH 49/49] move incident title log to scope --- src/StatusAggregator/Parse/IncidentParser.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/StatusAggregator/Parse/IncidentParser.cs b/src/StatusAggregator/Parse/IncidentParser.cs index b6e39deed..86745c693 100644 --- a/src/StatusAggregator/Parse/IncidentParser.cs +++ b/src/StatusAggregator/Parse/IncidentParser.cs @@ -45,11 +45,12 @@ public IncidentParser( public bool TryParseIncident(Incident incident, out ParsedIncident parsedIncident) { - using (_logger.Scope("Parsing incident with parser {IncidentParserType} using {RegExPattern}", - GetType(), _regExPattern)) + var title = incident.Title; + + using (_logger.Scope("Using parser {IncidentParserType} with pattern {RegExPattern} to parse incident with title {IncidentTitle}", + GetType(), _regExPattern, title)) { parsedIncident = null; - var title = incident.Title; Match match = null; try @@ -68,7 +69,7 @@ public bool TryParseIncident(Incident incident, out ParsedIncident parsedInciden return false; } - _logger.LogInformation("Incident title is {IncidentTitle}, RegEx match result: {MatchResult}", title, match.Success); + _logger.LogInformation("RegEx match result: {MatchResult}", title, match.Success); return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident); } }