diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln index a3f8581ba..df39eb99f 100644 --- a/NuGet.Jobs.sln +++ b/NuGet.Jobs.sln @@ -135,6 +135,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate.Tests", "tests\NuGet.Services.Revalidate.Tests\NuGet.Services.Revalidate.Tests.csproj", "{19780DCB-B307-4254-B10C-4335FC784DEA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core", "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj", "{17510A22-176F-4E96-A867-E79F1B54F54F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monitoring.RebootSearchInstance", "src\Monitoring.RebootSearchInstance\Monitoring.RebootSearchInstance.csproj", "{ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monitoring.RebootSearchInstance.Tests", "tests\Monitoring.RebootSearchInstance.Tests\Monitoring.RebootSearchInstance.Tests.csproj", "{21C0A0EE-8696-4013-950F-D6495D0C6E40}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -351,6 +357,18 @@ Global {19780DCB-B307-4254-B10C-4335FC784DEA}.Debug|Any CPU.Build.0 = Debug|Any CPU {19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.ActiveCfg = Release|Any CPU {19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.Build.0 = Release|Any CPU + {17510A22-176F-4E96-A867-E79F1B54F54F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {17510A22-176F-4E96-A867-E79F1B54F54F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {17510A22-176F-4E96-A867-E79F1B54F54F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {17510A22-176F-4E96-A867-E79F1B54F54F}.Release|Any CPU.Build.0 = Release|Any CPU + {ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}.Release|Any CPU.Build.0 = Release|Any CPU + {21C0A0EE-8696-4013-950F-D6495D0C6E40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {21C0A0EE-8696-4013-950F-D6495D0C6E40}.Debug|Any CPU.Build.0 = Debug|Any CPU + {21C0A0EE-8696-4013-950F-D6495D0C6E40}.Release|Any CPU.ActiveCfg = Release|Any CPU + {21C0A0EE-8696-4013-950F-D6495D0C6E40}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -408,6 +426,9 @@ Global {60152AB1-2EB4-4D44-B6D6-EEE24209A1F7} = {6A776396-02B1-475D-A104-26940ADB04AB} {1963909D-8BE3-4CB8-B57E-AB6A8CB22FED} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} {19780DCB-B307-4254-B10C-4335FC784DEA} = {6A776396-02B1-475D-A104-26940ADB04AB} + {17510A22-176F-4E96-A867-E79F1B54F54F} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02} + {ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66} = {814F9B31-4AF3-46CC-AD61-CEB40F47083A} + {21C0A0EE-8696-4013-950F-D6495D0C6E40} = {6A776396-02B1-475D-A104-26940ADB04AB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B} diff --git a/README.md b/README.md index 507081ce7..6318aa200 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,13 @@ NuGet.Jobs 7. Also, add settings.job file to mark the job as singleton, if the job will be run as a webjob, and it be a continuously running singleton +## Specific instructions + +This repository has a lot of jobs used for completely different purposes. For this reason, each should should have its +own documentation. This specific documentation is far from complete... but you have to start somewhere! + +- [Monitoring.RebootSearchInstance](src/Monitoring.RebootSearchInstance/README.md) - check each region for stuck search instances and restart them + Open Source Code of Conduct =================== This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/build.ps1 b/build.ps1 index 74762a2ab..c2f0decf2 100644 --- a/build.ps1 +++ b/build.ps1 @@ -114,7 +114,9 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { ` "$PSScriptRoot\src\NuGet.Jobs.Common\Properties\AssemblyInfo.g.cs", "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs", "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs" + "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs", + "$PSScriptRoot\src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs", + "$PSScriptRoot\src\Monitoring.RebootSearchInstance\Properties\AssemblyInfo.g.cs" $versionMetadata | ForEach-Object { Set-VersionInfo -Path $_ -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA @@ -173,7 +175,9 @@ Invoke-BuildStep 'Creating artifacts' { "src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", ` "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", ` "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", ` - "src/PackageLagMonitor/Monitoring.PackageLag.csproj" ` + "src/PackageLagMonitor/Monitoring.PackageLag.csproj", ` + "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj", ` + "src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.csproj" ` + $ProjectsWithSymbols Foreach ($Project in $Projects) { diff --git a/src/Monitoring.RebootSearchInstance/App.config b/src/Monitoring.RebootSearchInstance/App.config new file mode 100644 index 000000000..b50c74f35 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/FeedClient.cs b/src/Monitoring.RebootSearchInstance/FeedClient.cs new file mode 100644 index 000000000..bc8ef8054 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/FeedClient.cs @@ -0,0 +1,73 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using System.Xml.Linq; +using Autofac; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class FeedClient : IFeedClient + { + private const string FeedRelativeUrl = + "api/v2/Packages?" + + "$filter={0}%20ne%20null&" + + "$top=1&" + + "$orderby={0}%20desc&" + + "$select={0}"; + + private static readonly IEnumerable TimeStampProperties = new string[] + { + "Created", + "LastEdited", + }; + + private readonly HttpClient _httpClient; + private readonly IOptionsSnapshot _configuration; + private readonly ILogger _logger; + + public FeedClient(HttpClient httpClient, IOptionsSnapshot configuration, ILogger logger) + { + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + } + + public async Task GetLatestFeedTimeStampAsync() + { + var tasks = TimeStampProperties + .Select(GetLatestFeedTimeStampAsync) + .ToList(); + + await Task.WhenAll(tasks); + + return tasks.Max(t => t.Result); + } + + private async Task GetLatestFeedTimeStampAsync(string propertyName) + { + var galleryBaseUrl = _configuration.Value.FeedUrl; + var feedUrl = new Uri(galleryBaseUrl).GetLeftPart(UriPartial.Authority); + var url = $"{feedUrl}/{string.Format(FeedRelativeUrl, propertyName)}"; + using (var stream = await _httpClient.GetStreamAsync(url)) + { + var packagesResultObject = XDocument.Load(stream); + var propertyValue = DateTimeOffset.Parse( + packagesResultObject.Descendants() + .Where(a => a.Name.LocalName == propertyName) + .FirstOrDefault().Value, + formatProvider: null, + styles: DateTimeStyles.AssumeUniversal); + _logger.LogInformation("Feed at {GalleryBaseUrl} has {TimeStampPropertyName} cursor at {TimeStampPropertyValue}", galleryBaseUrl, propertyName, propertyValue); + return propertyValue; + } + } + } +} diff --git a/src/Monitoring.RebootSearchInstance/IFeedClient.cs b/src/Monitoring.RebootSearchInstance/IFeedClient.cs new file mode 100644 index 000000000..82ca0f2e0 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/IFeedClient.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public interface IFeedClient + { + Task GetLatestFeedTimeStampAsync(); + } +} \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/ISearchInstanceRebooter.cs b/src/Monitoring.RebootSearchInstance/ISearchInstanceRebooter.cs new file mode 100644 index 000000000..b8c0b3c83 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/ISearchInstanceRebooter.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public interface ISearchInstanceRebooter + { + Task RunAsync(CancellationToken token); + } +} \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/ITelemetryService.cs b/src/Monitoring.RebootSearchInstance/ITelemetryService.cs new file mode 100644 index 000000000..3447585bd --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/ITelemetryService.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public interface ITelemetryService + { + void TrackHealthyInstanceCount(string region, int count); + void TrackInstanceCount(string region, int count); + void TrackInstanceReboot(string region, int index); + void TrackInstanceRebootDuration(string region, int index, TimeSpan duration, InstanceHealth health); + void TrackUnhealthyInstanceCount(string region, int count); + void TrackUnknownInstanceCount(string region, int count); + } +} \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/InstanceHealth.cs b/src/Monitoring.RebootSearchInstance/InstanceHealth.cs new file mode 100644 index 000000000..0d524ea42 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/InstanceHealth.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public enum InstanceHealth + { + Healthy, + Unhealthy, + Unknown, + } +} diff --git a/src/Monitoring.RebootSearchInstance/Job.cs b/src/Monitoring.RebootSearchInstance/Job.cs new file mode 100644 index 000000000..3bbe5643c --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Job.cs @@ -0,0 +1,80 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Montoring.PackageLag; +using NuGet.Jobs.Validation; +using NuGet.Services.AzureManagement; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class Job : JsonConfigurationJob + { + private const string AzureManagementSectionName = "AzureManagement"; + private const string MonitorConfigurationSectionName = "MonitorConfiguration"; + + public override async Task Run() + { + using (var scope = _serviceProvider.CreateScope()) + { + var rebooter = scope.ServiceProvider.GetRequiredService(); + await rebooter.RunAsync(CancellationToken.None); + } + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + services.Configure(configurationRoot.GetSection(MonitorConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(MonitorConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(AzureManagementSectionName)); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddSingleton(); + services.AddTransient(p => p.GetService>().Value); + + services.AddSingleton(p => + { + var assembly = Assembly.GetEntryAssembly(); + var assemblyName = assembly.GetName().Name; + var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; + + var client = new HttpClient(new WebRequestHandler + { + AllowPipelining = true, + AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), + ServerCertificateCustomValidationCallback = + (httpRequestMessage, cert, cetChain, policyErrors) => + { + if (policyErrors == System.Net.Security.SslPolicyErrors.RemoteCertificateNameMismatch || policyErrors == System.Net.Security.SslPolicyErrors.None) + { + return true; + } + + return false; + }, + }); + + client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); + client.Timeout = TimeSpan.FromSeconds(10); + + return client; + }); + } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + } +} diff --git a/src/Monitoring.RebootSearchInstance/MonitorConfiguration.cs b/src/Monitoring.RebootSearchInstance/MonitorConfiguration.cs new file mode 100644 index 000000000..39c500d13 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/MonitorConfiguration.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Jobs.Montoring.PackageLag; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class MonitorConfiguration : SearchServiceConfiguration + { + public string FeedUrl { get; set; } + public string Role { get; set; } + public string RoleInstanceFormat { get; set; } + + /// + /// This configuration is in seconds to match the monitor configuration. If the search instance lag is less + /// than this duration, that instance is considered healthy. + /// + public int HealthyThresholdInSeconds { get; set; } + + /// + /// This configuration is in seconds to match the monitor configuration. If the search instance lag is more + /// than this duration, that instance is considered unhealthy. + /// + public int UnhealthyThresholdInSeconds { get; set; } + + /// + /// The time to wait for a restarted instance to become healthy before moving on. + /// + public TimeSpan WaitForHealthyDuration { get; set; } + + /// + /// The time to wait before checking a region for unhealthy instances again. + /// + public TimeSpan SleepDuration { get; set; } + + /// + /// How long the process should run before ending (allowing the caller to restart the process as desired). + /// + public TimeSpan ProcessLifetime { get; set; } + + /// + /// How frequently to poll an instance that was just restarted to check for it's new health status. + /// + public TimeSpan InstancePollFrequency { get; set; } + } +} diff --git a/src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.csproj b/src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.csproj new file mode 100644 index 000000000..e17bee527 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.csproj @@ -0,0 +1,90 @@ + + + + + Debug + AnyCPU + {ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66} + Exe + NuGet.Monitoring.RebootSearchInstance + NuGet.Monitoring.RebootSearchInstance + v4.6.2 + 512 + true + PackageReference + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} + NuGet.Jobs.Common + + + {b5147169-e941-4cf8-9fcd-1c123acd3149} + Monitoring.PackageLag + + + {fa87d075-a934-4443-8d0b-5db32640b6d7} + Validation.Common.Job + + + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.nuspec b/src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.nuspec new file mode 100644 index 000000000..f588c7874 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.nuspec @@ -0,0 +1,24 @@ + + + + Monitoring.RebootSearchInstance + $version$ + Monitoring.RebootSearchInstance + .NET Foundation + .NET Foundation + Monitoring.RebootSearchInstance + Copyright .NET Foundation + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Program.cs b/src/Monitoring.RebootSearchInstance/Program.cs new file mode 100644 index 000000000..8490edecc --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Program.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Jobs; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class Program + { + public static void Main(string[] args) + { + var job = new Job(); + JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); + } + } +} diff --git a/src/Monitoring.RebootSearchInstance/Properties/AssemblyInfo.cs b/src/Monitoring.RebootSearchInstance/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..c672867f1 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Monitoring.RebootSearchInstance")] +[assembly: ComVisible(false)] +[assembly: Guid("ecd8dfce-8e3c-4510-afe3-d7ec168e8d66")] diff --git a/src/Monitoring.RebootSearchInstance/README.md b/src/Monitoring.RebootSearchInstance/README.md new file mode 100644 index 000000000..3e394d46e --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/README.md @@ -0,0 +1,34 @@ +# Monitoring.RebootSearchInstance + +There is a tricky bug in the search service today that sporadically causes a search index to stop loading in an +instance. This results stale search results in that region for some requests (whenever the load balancer happens +to hit that bad instance). This also results in a phone call to our on-call person due to index lag. No fun! + +Therefore, this job runs in each environment (DEV, INT, PROD) as singleton and restarts stuck search instances. + +## Configuration + +The configuration files are JSON. There is one for DEV, INT, and PROD with placeholder tokens that are meant to be +replaced by Octopus. + +## Running Locally + +This section is written with the assumption that the reader is a NuGet team member. + +**First,** open `Settings\dev.json` in a text editor and replace the `#{PROPERTY.NAME}` values with the correct values. +I recommend getting the values for our DEV environment from Octopus. I have put a filled out `dev.json` in OneNote so +you can get it from there. Note that it has some KeyVault placeholders in it so you need the DEV KeyVault certificate +installed on your machine-wide ("Local Machine") certificate store. + +**Second,** build this project in Visual Studio 2017. + +**Third,** open a command prompt in the `bin\Debug` directory and run the following command: + + +``` +NuGet.Monitoring.RebootSearchInstance.exe -Configuration ..\..\Settings\dev.json +``` + +You can add the `-InstrumentationKey AI_KEY` option if you want the logs to go to ApplicationInsights. + +This job will keep running for a day, checking each configured reason every 5 minutes. \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Scripts/Functions.ps1 b/src/Monitoring.RebootSearchInstance/Scripts/Functions.ps1 new file mode 100644 index 000000000..85b36a8a0 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Scripts/Functions.ps1 @@ -0,0 +1,30 @@ +Function Uninstall-NuGetService() { + Param ([string]$ServiceName) + + if (Get-Service $ServiceName -ErrorAction SilentlyContinue) + { + Write-Host Removing service $ServiceName... + Stop-Service $ServiceName -Force + sc.exe delete $ServiceName + Write-Host Removed service $ServiceName. + } else { + Write-Host Skipping removal of service $ServiceName - no such service exists. + } +} + +Function Install-NuGetService() { + Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) + + Write-Host Installing service $ServiceName... + + $installService = "nssm install $ServiceName $ScriptToRun" + cmd /C $installService + + Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic + sc.exe failure $ServiceName reset= 30 actions= restart/5000 + + # Run service + net start $ServiceName + + Write-Host Installed service $ServiceName. +} \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Scripts/Monitoring.RebootSearchInstance.cmd b/src/Monitoring.RebootSearchInstance/Scripts/Monitoring.RebootSearchInstance.cmd new file mode 100644 index 000000000..937d55344 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Scripts/Monitoring.RebootSearchInstance.cmd @@ -0,0 +1,16 @@ +@echo OFF + +cd bin + +:Top +echo "Starting job - #{Jobs.Monitoring.RebootSearchInstance.Title}" + +title #{Jobs.Monitoring.RebootSearchInstance.Title} + +start /w NuGet.Monitoring.RebootSearchInstance.exe ^ + -Configuration #{Jobs.Monitoring.RebootSearchInstance.Configuration} ^ + -InstrumentationKey "#{Jobs.Monitoring.RebootSearchInstance.InstrumentationKey}" + +echo "Finished #{Jobs.Monitoring.RebootSearchInstance.Title}" + +goto Top diff --git a/src/Monitoring.RebootSearchInstance/Scripts/PostDeploy.ps1 b/src/Monitoring.RebootSearchInstance/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..4aded4737 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Scripts/PostDeploy.ps1 @@ -0,0 +1,18 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Installing services... + +$currentDirectory = [string](Get-Location) + +$jobsToInstall.Split("{;}") | %{ + $serviceName = $_ + $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] + $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] + $scriptToRun = "$currentDirectory\$scriptToRun" + + Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun +} + +Write-Host Installed services. \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Scripts/PreDeploy.ps1 b/src/Monitoring.RebootSearchInstance/Scripts/PreDeploy.ps1 new file mode 100644 index 000000000..cce8f6cf9 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Scripts/PreDeploy.ps1 @@ -0,0 +1,11 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Removing services... + +$jobsToInstall.Split("{;}") | %{ + Uninstall-NuGetService -ServiceName $_ +} + +Write-Host Removed services. \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Scripts/nssm.exe b/src/Monitoring.RebootSearchInstance/Scripts/nssm.exe new file mode 100644 index 000000000..6ccfe3cfb Binary files /dev/null and b/src/Monitoring.RebootSearchInstance/Scripts/nssm.exe differ diff --git a/src/Monitoring.RebootSearchInstance/SearchInstanceRebooter.cs b/src/Monitoring.RebootSearchInstance/SearchInstanceRebooter.cs new file mode 100644 index 000000000..ef61915a0 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/SearchInstanceRebooter.cs @@ -0,0 +1,268 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Montoring.PackageLag; +using NuGet.Services.AzureManagement; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class SearchInstanceRebooter : ISearchInstanceRebooter + { + private readonly IFeedClient _feedClient; + private readonly ISearchServiceClient _searchServiceClient; + private readonly IAzureManagementAPIWrapper _azureManagementAPIWrapper; + private readonly ITelemetryService _telemetryService; + private readonly IOptionsSnapshot _configuration; + private readonly ILogger _logger; + private readonly TimeSpan _healthyThreshold; + private readonly TimeSpan _unhealthyThreshold; + + public SearchInstanceRebooter( + IFeedClient feedClient, + ISearchServiceClient searchServiceClient, + IAzureManagementAPIWrapper azureManagementAPIWrapper, + ITelemetryService telemetryService, + IOptionsSnapshot configuration, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _searchServiceClient = searchServiceClient ?? throw new ArgumentNullException(nameof(searchServiceClient)); + _azureManagementAPIWrapper = azureManagementAPIWrapper ?? throw new ArgumentNullException(nameof(azureManagementAPIWrapper)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _healthyThreshold = TimeSpan.FromSeconds(_configuration.Value.HealthyThresholdInSeconds); + _unhealthyThreshold = TimeSpan.FromSeconds(_configuration.Value.UnhealthyThresholdInSeconds); + } + + public async Task RunAsync(CancellationToken token) + { + var tasks = _configuration + .Value + .RegionInformations + .Select(r => ProcessRegionLoopAsync(r, token)) + .ToList(); + + await Task.WhenAll(tasks); + } + + private async Task ProcessRegionLoopAsync(RegionInformation regionInformation, CancellationToken token) + { + using (_logger.BeginScope( + "Starting search instance check loop for region {Region}, service {ServiceName}.", + regionInformation.Region, + regionInformation.ServiceName)) + { + var stopwatch = Stopwatch.StartNew(); + + do + { + try + { + _logger.LogInformation("Checking region {Region} for search instances that need rebooting.", regionInformation.Region); + + await ProcessRegionOnceAsync(regionInformation, token); + + _logger.LogInformation( + "Sleeping for {SleepDuration} before checking region {Region} again.", + _configuration.Value.SleepDuration, + regionInformation.Region); + } + catch (Exception ex) + { + _logger.LogError( + (EventId)0, + ex, + "An exception was thrown when processing region {Region}.", regionInformation.Region); + } + + await Task.Delay(_configuration.Value.SleepDuration); + } + while (stopwatch.Elapsed < _configuration.Value.ProcessLifetime); + } + } + + private async Task ProcessRegionOnceAsync(RegionInformation regionInformation, CancellationToken token) + { + var instances = await _searchServiceClient.GetSearchEndpointsAsync(regionInformation, token); + var latestFeedTimeStamp = await _feedClient.GetLatestFeedTimeStampAsync(); + + _logger.LogDebug("The latest feed timestamp is {LastFeedTimeStamp}.", latestFeedTimeStamp); + + // Determine the health of all instances. + var tasks = instances + .Select(i => GetInstanceAndHealthAsync(latestFeedTimeStamp, i, token)) + .ToList(); + var instancesAndHealths = await Task.WhenAll(tasks); + + var healthyCount = instancesAndHealths.Count(x => x.Health == InstanceHealth.Healthy); + var unhealthyCount = instancesAndHealths.Count(x => x.Health == InstanceHealth.Unhealthy); + + if (healthyCount == 0) + { + _logger.LogWarning( + "There are no healthy instances on {Region}. No instances will be restarted since there is " + + "likely a broader issue going on.", + regionInformation.Region); + } + else if (unhealthyCount == 0) + { + _logger.LogInformation( + "There are no unhealthy instances on {Region}. No more work is necessary.", + regionInformation.Region); + } + else + { + // Restart the first unhealthy instance and wait for it to come back. + var unhealthyInstance = instancesAndHealths + .Where(i => i.Health == InstanceHealth.Unhealthy) + .OrderBy(i => i.Instance.Index) + .FirstOrDefault(); + + await RestartInstanceAndWaitForHealthyAsync( + regionInformation, + latestFeedTimeStamp, + unhealthyInstance.Instance, + token); + } + + // Emit telemetry + _telemetryService.TrackHealthyInstanceCount(regionInformation.Region, healthyCount); + _telemetryService.TrackUnhealthyInstanceCount(regionInformation.Region, unhealthyCount); + _telemetryService.TrackUnknownInstanceCount( + regionInformation.Region, + instancesAndHealths.Count(x => x.Health == InstanceHealth.Unknown)); + _telemetryService.TrackInstanceCount( + regionInformation.Region, + instancesAndHealths.Count()); + } + + private async Task RestartInstanceAndWaitForHealthyAsync( + RegionInformation regionInformation, + DateTimeOffset latestFeedTimeStamp, + Instance instance, + CancellationToken token) + { + _telemetryService.TrackInstanceReboot(regionInformation.Region, instance.Index); + + var roleInstance = string.Format(_configuration.Value.RoleInstanceFormat, instance.Index); + + _logger.LogWarning("Rebooting role instance {RoleInstance} in region {Region}.", roleInstance, regionInformation.Region); + + await _azureManagementAPIWrapper.RebootCloudServiceRoleInstanceAsync( + _configuration.Value.Subscription, + regionInformation.ResourceGroup, + regionInformation.ServiceName, + instance.Slot, + _configuration.Value.Role, + roleInstance, + token); + + InstanceHealth health; + var waitStopwatch = Stopwatch.StartNew(); + do + { + _logger.LogDebug( + "Checking the health status of role instance {RoleInstance} after rebooting.", + roleInstance); + + health = await DetermineInstanceHealthAsync(latestFeedTimeStamp, instance, token); + if (health != InstanceHealth.Healthy) + { + await Task.Delay(_configuration.Value.InstancePollFrequency); + } + } + while (waitStopwatch.Elapsed < _configuration.Value.WaitForHealthyDuration + && health != InstanceHealth.Healthy); + + waitStopwatch.Stop(); + _logger.LogInformation( + "After waiting {WaitDuration}, instance {DiagUrl} has health of {Health}.", + waitStopwatch.Elapsed, + instance.DiagUrl, + health); + _telemetryService.TrackInstanceRebootDuration( + regionInformation.Region, + instance.Index, + waitStopwatch.Elapsed, + health); + } + + private async Task GetInstanceAndHealthAsync( + DateTimeOffset latestFeedTimeStamp, + Instance instance, + CancellationToken token) + { + var health = await DetermineInstanceHealthAsync(latestFeedTimeStamp, instance, token); + + _logger.LogInformation( + "Instance {DiagUrl} has health of {Health}.", + instance.DiagUrl, + health); + + return new InstanceAndHealth(instance, health); + } + + private async Task DetermineInstanceHealthAsync( + DateTimeOffset latestFeedTimeStamp, + Instance instance, + CancellationToken token) + { + DateTimeOffset commitDateTime; + try + { + commitDateTime = await _searchServiceClient.GetCommitDateTimeAsync(instance, token); + } + catch (Exception ex) + { + _logger.LogInformation( + (EventId)0, + ex, + "An exception was thrown when getting the commit timestamp from {DiagUrl}. Considering this " + + "instance as an unknown state.", + instance.DiagUrl); + + return InstanceHealth.Unknown; + } + + var lag = latestFeedTimeStamp - commitDateTime; + + _logger.LogDebug( + "Instance {DiagUrl} has commit timestamp {CommitTimeStamp} and lag of {Lag}.", + instance.DiagUrl, + commitDateTime, + lag); + + if (lag <= _healthyThreshold) + { + return InstanceHealth.Healthy; + } + + if (lag > _unhealthyThreshold) + { + return InstanceHealth.Unhealthy; + } + + return InstanceHealth.Unknown; + } + + private class InstanceAndHealth + { + public InstanceAndHealth(Instance instance, InstanceHealth health) + { + Instance = instance; + Health = health; + } + + public Instance Instance { get; } + public InstanceHealth Health { get; } + } + } +} diff --git a/src/Monitoring.RebootSearchInstance/Settings/dev.json b/src/Monitoring.RebootSearchInstance/Settings/dev.json new file mode 100644 index 000000000..a2bd3b569 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Settings/dev.json @@ -0,0 +1,49 @@ +{ + "AzureManagement": { + "ClientId": "#{Jobs.Monitoring.RebootSearchInstance.AzureManagement.ClientId}", + "ClientSecret": "#{Jobs.Monitoring.RebootSearchInstance.AzureManagement.ClientSecret}" + }, + + "MonitorConfiguration": { + "FeedUrl": "#{Jobs.common.v3.f2c.Gallery}", + "InstancePortMinimum": "#{Jobs.Monitoring.PackageLag.InstancePortMinimum}", + "Subscription": "#{Deployment.Azure.SubscriptionId}", + "Role": "NuGet.Services.BasicSearch", + "RoleInstanceFormat": "NuGet.Services.BasicSearch_IN_{0}", + "HealthyThresholdInSeconds": 2400, + "UnhealthyThresholdInSeconds": 3600, + "ProcessLifetime": "1.00:00:00", + "SleepDuration": "00:05:00", + "WaitForHealthyDuration": "00:10:00", + "InstancePollFrequency": "00:00:10", + "RegionInformations": [ + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.Primary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.Primary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.Primary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.Secondary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.Secondary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.Secondary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.ChinaPrimary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.ChinaPrimary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.ChinaPrimary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.ChinaSecondary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.ChinaSecondary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.ChinaSecondary.Region}" + } + ] + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": false, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} diff --git a/src/Monitoring.RebootSearchInstance/Settings/int.json b/src/Monitoring.RebootSearchInstance/Settings/int.json new file mode 100644 index 000000000..f64d1e7be --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Settings/int.json @@ -0,0 +1,39 @@ +{ + "AzureManagement": { + "ClientId": "#{Jobs.Monitoring.RebootSearchInstance.AzureManagement.ClientId}", + "ClientSecret": "#{Jobs.Monitoring.RebootSearchInstance.AzureManagement.ClientSecret}" + }, + + "MonitorConfiguration": { + "FeedUrl": "#{Jobs.common.v3.f2c.Gallery}", + "InstancePortMinimum": "#{Jobs.Monitoring.PackageLag.InstancePortMinimum}", + "Subscription": "#{Deployment.Azure.SubscriptionId}", + "Role": "NuGet.Services.BasicSearch", + "RoleInstanceFormat": "NuGet.Services.BasicSearch_IN_{0}", + "HealthyThresholdInSeconds": 2400, + "UnhealthyThresholdInSeconds": 3600, + "ProcessLifetime": "1.00:00:00", + "SleepDuration": "00:05:00", + "WaitForHealthyDuration": "00:10:00", + "InstancePollFrequency": "00:00:10", + "RegionInformations": [ + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.Primary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.Primary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.Primary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.Secondary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.Secondary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.Secondary.Region}" + } + ] + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": false, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} diff --git a/src/Monitoring.RebootSearchInstance/Settings/prod.json b/src/Monitoring.RebootSearchInstance/Settings/prod.json new file mode 100644 index 000000000..37cf6b6e5 --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/Settings/prod.json @@ -0,0 +1,49 @@ +{ + "AzureManagement": { + "ClientId": "#{Jobs.Monitoring.RebootSearchInstance.AzureManagement.ClientId}", + "ClientSecret": "#{Jobs.Monitoring.RebootSearchInstance.AzureManagement.ClientSecret}" + }, + + "MonitorConfiguration": { + "FeedUrl": "#{Jobs.common.v3.f2c.Gallery}", + "InstancePortMinimum": "#{Jobs.Monitoring.PackageLag.InstancePortMinimum}", + "Subscription": "#{Deployment.Azure.SubscriptionId}", + "Role": "NuGet.Services.BasicSearch", + "RoleInstanceFormat": "NuGet.Services.BasicSearch_IN_{0}", + "HealthyThresholdInSeconds": 2400, + "UnhealthyThresholdInSeconds": 3600, + "ProcessLifetime": "1.00:00:00", + "SleepDuration": "00:05:00", + "WaitForHealthyDuration": "00:10:00", + "InstancePollFrequency": "00:00:10", + "RegionInformations": [ + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.Primary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.Primary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.Primary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.Secondary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.Secondary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.Secondary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.ChinaPrimary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.ChinaPrimary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.ChinaPrimary.Region}" + }, + { + "ResourceGroup": "#{Jobs.Monitoring.PackageLag.ChinaSecondary.ResourceGroup}", + "ServiceName": "#{Jobs.Monitoring.PackageLag.ChinaSecondary.ServiceName}", + "Region": "#{Jobs.Monitoring.PackageLag.ChinaSecondary.Region}" + } + ] + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": false, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} diff --git a/src/Monitoring.RebootSearchInstance/TelemetryService.cs b/src/Monitoring.RebootSearchInstance/TelemetryService.cs new file mode 100644 index 000000000..823192b6f --- /dev/null +++ b/src/Monitoring.RebootSearchInstance/TelemetryService.cs @@ -0,0 +1,91 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Options; +using NuGet.Services.Logging; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class TelemetryService : ITelemetryService + { + private const string Region = "Region"; + private const string Subscription = "Subscription"; + private const string InstanceIndex = "InstanceIndex"; + private const string Health = "Health"; + + private const string Prefix = "RebootSearchInstance."; + private const string HealthyInstanceCount = Prefix + "HealthyInstances"; + private const string UnhealthyInstanceCount = Prefix + "UnhealthyInstances"; + private const string UnknownInstanceCount = Prefix + "UnknownInstances"; + private const string InstanceCount = Prefix + "Instances"; + private const string InstanceReboot = Prefix + "InstanceReboot"; + private const string InstanceRebootDuration = Prefix + "InstanceRebootDurationSeconds"; + + private readonly ITelemetryClient _telemetryClient; + private readonly IOptionsSnapshot _configuration; + private readonly string _subscription; + + public TelemetryService(ITelemetryClient telemetryClient, IOptionsSnapshot configuration) + { + _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + _subscription = _configuration.Value.Subscription; + } + + public void TrackHealthyInstanceCount(string region, int count) + { + TrackInstanceCount(HealthyInstanceCount, region, count); + } + + public void TrackUnhealthyInstanceCount(string region, int count) + { + TrackInstanceCount(UnhealthyInstanceCount, region, count); + } + + public void TrackUnknownInstanceCount(string region, int count) + { + TrackInstanceCount(UnknownInstanceCount, region, count); + } + + public void TrackInstanceCount(string region, int count) + { + TrackInstanceCount(InstanceCount, region, count); + } + + public void TrackInstanceReboot(string region, int index) + { + _telemetryClient.TrackMetric(InstanceReboot, 1, new Dictionary + { + { Subscription, _subscription }, + { Region, region }, + { InstanceIndex, index.ToString() }, + }); + } + + public void TrackInstanceRebootDuration( + string region, + int index, + TimeSpan duration, + InstanceHealth health) + { + _telemetryClient.TrackMetric(InstanceRebootDuration, duration.TotalSeconds, new Dictionary + { + { Subscription, _subscription }, + { Region, region }, + { InstanceIndex, index.ToString() }, + { Health, health.ToString() }, + }); + } + + private void TrackInstanceCount(string name, string region, int count) + { + _telemetryClient.TrackMetric(name, count, new Dictionary + { + { Subscription, _subscription }, + { Region, region }, + }); + } + } +} diff --git a/src/NuGet.Jobs.Common/JobRunner.cs b/src/NuGet.Jobs.Common/JobRunner.cs index b72dbcec0..0d3d414c1 100644 --- a/src/NuGet.Jobs.Common/JobRunner.cs +++ b/src/NuGet.Jobs.Common/JobRunner.cs @@ -40,6 +40,25 @@ static JobRunner() /// Args contains args to the job runner like (dbg, once and so on) and for the job itself /// public static async Task Run(JobBase job, string[] commandLineArgs) + { + await Run(job, commandLineArgs, runContinuously: null); + } + + /// + /// This is a static method to run a job whose args are passed in + /// By default, + /// a) The job will be run the job once, irrespective of the 'once' argument. + /// b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument + /// + /// Job to run + /// Args contains args to the job runner like (dbg, once and so on) and for the job itself + /// + public static async Task RunOnce(JobBase job, string[] commandLineArgs) + { + await Run(job, commandLineArgs, runContinuously: false); + } + + private static async Task Run(JobBase job, string[] commandLineArgs, bool? runContinuously) { if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase)) { @@ -72,7 +91,16 @@ public static async Task Run(JobBase job, string[] commandLineArgs) // Configure our logging again with Application Insights initialized. loggerFactory = ConfigureLogging(job); - var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once); + var hasOnceArgument = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once); + + if (runContinuously.HasValue && hasOnceArgument) + { + _logger.LogWarning( + $"This job is designed to {(runContinuously.Value ? "run continuously" : "run once")} so " + + $"the -{JobArgumentNames.Once} argument is {(runContinuously.Value ? "ignored" : "redundant")}."); + } + + runContinuously = runContinuously ?? !hasOnceArgument; var reinitializeAfterSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.ReinitializeAfterSeconds); var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds @@ -85,11 +113,21 @@ public static async Task Run(JobBase job, string[] commandLineArgs) } } - if (sleepDuration == null) + if (!sleepDuration.HasValue) { - _logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms..."); + if (runContinuously.Value) + { + _logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms..."); + } + sleepDuration = 5000; } + else if (!runContinuously.Value) + { + _logger.LogWarning( + $"The job is designed to run once so the -{JobArgumentNames.Sleep} and " + + $"-{JobArgumentNames.Interval} arguments are ignored."); + } if (!reinitializeAfterSeconds.HasValue) { @@ -97,13 +135,19 @@ public static async Task Run(JobBase job, string[] commandLineArgs) $"{JobArgumentNames.ReinitializeAfterSeconds} command line argument is not provided or is not a valid integer. " + "The job will reinitialize on every iteration"); } + else if (!runContinuously.Value) + { + _logger.LogWarning( + $"The job is designed to run once so the -{JobArgumentNames.ReinitializeAfterSeconds} " + + $"argument is ignored."); + } // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled. ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // Run the job loop - await JobLoop(job, runContinuously, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary); + await JobLoop(job, runContinuously.Value, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary); } catch (Exception ex) { diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs index 191799ada..8e6c9f831 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Job.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Job.cs @@ -25,7 +25,6 @@ using NuGet.Jobs.Validation; using NuGet.Jobs.Validation.Common; using NuGet.Jobs.Validation.PackageSigning.Messages; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.ScanAndSign; using NuGet.Jobs.Validation.Storage; using NuGet.Services.Configuration; diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index 7c92ce233..ebf1fb92c 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -55,6 +55,10 @@ + + + + @@ -147,6 +151,10 @@ {dfac2769-4b67-4fbc-ad60-d93a39dd45ae} Validation.ScanAndSign.Core + + {17510a22-176f-4e96-a867-e79f1b54f54f} + Validation.Symbols.Core + diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/BaseSignatureProcessor.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/BaseSignatureProcessor.cs index a5f641a43..64ff6ac61 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/BaseSignatureProcessor.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/BaseSignatureProcessor.cs @@ -4,7 +4,7 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Jobs.Validation.PackageSigning.Storage; +using NuGet.Jobs.Validation; using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator.Telemetry; diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureProcessor.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureProcessor.cs index f275f1bea..66708bd40 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureProcessor.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureProcessor.cs @@ -4,7 +4,6 @@ using System; using Microsoft.Extensions.Logging; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator.Telemetry; diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureValidator.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureValidator.cs index 29d72560d..5310d8fbc 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureValidator.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureValidator.cs @@ -7,7 +7,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign; using NuGet.Services.Validation.Orchestrator.Telemetry; diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/Scan/ScanValidator.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/Scan/ScanValidator.cs index 6dfb35702..03b0ad0b1 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/Scan/ScanValidator.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/Scan/ScanValidator.cs @@ -6,7 +6,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Jobs.Validation.ScanAndSign; using NuGet.Services.Validation.Vcs; diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs index f2876eff1..01d6d9e0c 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Jobs.Validation.ScanAndSign; using NuGet.Services.Validation.Vcs; diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs index 3b3814066..aee75d3f2 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; +using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator; using NuGet.Services.Validation.Orchestrator.Telemetry; using Error = NuGet.Services.Validation.Orchestrator.Error; diff --git a/src/NuGet.Services.Validation.Orchestrator/Program.cs b/src/NuGet.Services.Validation.Orchestrator/Program.cs index da9501182..241f0499d 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Program.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Program.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Linq; using NuGet.Jobs; namespace NuGet.Services.Validation.Orchestrator @@ -10,12 +9,8 @@ class Program { static int Main(string[] args) { - if (!args.Contains(JobArgumentNames.Once)) - { - args = args.Concat(new[] { "-" + JobArgumentNames.Once }).ToArray(); - } var job = new Job(); - JobRunner.Run(job, args).GetAwaiter().GetResult(); + JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); // if configuration validation failed, return non-zero status so we can detect failures in automation return job.ConfigurationValidated ? 0 : 1; diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/ISymbolsMessageEnqueuer.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/ISymbolsMessageEnqueuer.cs new file mode 100644 index 000000000..84d6e0c5d --- /dev/null +++ b/src/NuGet.Services.Validation.Orchestrator/Symbols/ISymbolsMessageEnqueuer.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Validation.Symbols +{ + public interface ISymbolsMessageEnqueuer + { + /// + /// Enqueues a message to one of the topics used by the Symbol validators + /// + /// The validtion request. + /// A that will be completed when the execution is completed. + Task EnqueueSymbolsValidationMessageAsync(IValidationRequest request); + } +} diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsMessageEnqueuer.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsMessageEnqueuer.cs new file mode 100644 index 000000000..7927f30ff --- /dev/null +++ b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsMessageEnqueuer.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Validation.Symbols.Core; +using NuGet.Services.ServiceBus; + +namespace NuGet.Services.Validation.Symbols +{ + public class SymbolsMessageEnqueuer : ISymbolsMessageEnqueuer + { + private readonly ITopicClient _topicClient; + private readonly IOptionsSnapshot _configuration; + private readonly IBrokeredMessageSerializer _serializer; + + public SymbolsMessageEnqueuer( + ITopicClient topicClient, + IBrokeredMessageSerializer serializer, + IOptionsSnapshot configuration) + { + _topicClient = topicClient ?? throw new ArgumentNullException(nameof(topicClient)); + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + } + + public async Task EnqueueSymbolsValidationMessageAsync(IValidationRequest request) + { + var message = new SymbolsValidatorMessage(validationId: request.ValidationId, + symbolPackageKey: request.PackageKey, + packageId: request.PackageId, + packageNormalizedVersion: request.PackageVersion, + snupkgUrl: request.NupkgUrl); + var brokeredMessage = _serializer.Serialize(message); + + var visibleAt = DateTimeOffset.UtcNow + (_configuration.Value.MessageDelay ?? TimeSpan.Zero); + brokeredMessage.ScheduledEnqueueTimeUtc = visibleAt; + + await _topicClient.SendAsync(brokeredMessage); + } + } +} diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsValidationConfiguration.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsValidationConfiguration.cs new file mode 100644 index 000000000..798f42969 --- /dev/null +++ b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsValidationConfiguration.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Jobs.Configuration; + +namespace NuGet.Services.Validation.Symbols +{ + public class SymbolsValidationConfiguration + { + /// + /// The Service Bus configuration used to enqueue symbol validations. + /// + public ServiceBusConfiguration ServiceBus { get; set; } + + /// + /// The visibility delay to apply to Service Bus messages requesting a new validation. + /// + public TimeSpan? MessageDelay { get; set; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsValidator.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsValidator.cs new file mode 100644 index 000000000..7416d35d4 --- /dev/null +++ b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsValidator.cs @@ -0,0 +1,94 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Validation; +using NuGet.Jobs.Validation.Storage; +using NuGet.Services.Validation.Orchestrator; +using NuGet.Services.Validation.Orchestrator.Telemetry; +using Error = NuGet.Services.Validation.Orchestrator.Error; + +namespace NuGet.Services.Validation.Symbols +{ + [ValidatorName(ValidatorName.SymbolsValidator)] + public class SymbolsValidator : BaseValidator, IValidator + { + private readonly IValidatorStateService _validatorStateService; + private readonly ISymbolsMessageEnqueuer _symbolMessageEnqueuer; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public SymbolsValidator( + IValidatorStateService validatorStateService, + ISymbolsMessageEnqueuer symbolMessageEnqueuer, + ITelemetryService telemetryService, + ILogger logger) + { + _validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService)); + _symbolMessageEnqueuer = symbolMessageEnqueuer ?? throw new ArgumentNullException(nameof(symbolMessageEnqueuer)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetResultAsync(IValidationRequest request) + { + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + + var validatorStatus = await _validatorStateService.GetStatusAsync(request); + var result = validatorStatus.ToValidationResult(); + if (validatorStatus.State == ValidationStatus.Failed) + { + _logger.LogInformation( + "SymbolValidationFailure "+ + "status = {ValidationStatus}, snupkg URL = {NupkgUrl}, validation issues = {Issues}", + result.Status, + result.NupkgUrl, + result.Issues.Select(i => i.IssueCode)); + } + return result; + } + + /// + /// The pattern used for the StartAsync: + /// 1. Check if a validation was already started + /// 2. Only if a validation was not started queue the message to be processed. + /// 3. After the message is queued, update the ValidatorStatus for the . + /// + /// The request to be send to the validator job queue. + /// The validation status. + public async Task StartAsync(IValidationRequest request) + { + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + + var validatorStatus = await _validatorStateService.GetStatusAsync(request); + + if (validatorStatus.State != ValidationStatus.NotStarted) + { + _logger.LogWarning( + "Symbol validation for {PackageId} {PackageNormalizedVersion} has already started.", + request.PackageId, + request.PackageVersion); + + return validatorStatus.ToValidationResult(); + } + + // Due to race conditions or failure of method TryAddValidatorStatusAsync the same message can be enqueued multiple times + // Log this information to postmortem evaluate this behavior + _telemetryService.TrackSymbolsMessageEnqueued(ValidatorName.SymbolsValidator, request.ValidationId); + await _symbolMessageEnqueuer.EnqueueSymbolsValidationMessageAsync(request); + + var result = await _validatorStateService.TryAddValidatorStatusAsync(request, validatorStatus, ValidationStatus.Incomplete); + + return result.ToValidationResult(); + } + } +} diff --git a/src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs b/src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs index 1ed67fa3a..6a2024284 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs @@ -117,5 +117,12 @@ public interface ITelemetryService /// A metric to of how long it took to hash a package. /// IDisposable TrackDurationToHashPackage(string packageId, string normalizedVersion, long packageSize, string hashAlgorithm, string streamType); + + /// + /// A metric to track the messages sent from Orchestrator to Validators and enqueued by validators. Ideally the messages should not duplicate. + /// + /// The validator name the message was sent to. + /// The validationId. + void TrackSymbolsMessageEnqueued(string validatorName, Guid validationId); } } \ No newline at end of file diff --git a/src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs b/src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs index 914f5d666..4bef9f864 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs @@ -31,6 +31,7 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet private const string DurationToHashPackageSeconds = OrchestratorPrefix + "DurationToHashPackageSeconds"; private const string MessageDeliveryLag = OrchestratorPrefix + "MessageDeliveryLag"; private const string MessageEnqueueLag = OrchestratorPrefix + "MessageEnqueueLag"; + private const string SymbolsMessageEnqueued = OrchestratorPrefix + "SymbolsMessageEnqueued"; private const string DurationToStartPackageSigningValidatorSeconds = PackageSigningPrefix + "DurationToStartSeconds"; @@ -49,6 +50,8 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet private const string HashAlgorithm = "HashAlgorithm"; private const string StreamType = "StreamType"; private const string MessageType = "MessageType"; + private const string ValidationId = "ValidationId"; + private const string OperationDateTime = "OperationDateTime"; private readonly ITelemetryClient _telemetryClient; @@ -243,5 +246,15 @@ public void TrackEnqueueLag(TimeSpan enqueueLag) { { MessageType, typeof(TMessage).Name } }); + + public void TrackSymbolsMessageEnqueued(string validatorName, Guid validationId) + => _telemetryClient.TrackMetric( + SymbolsMessageEnqueued, + 1, + new Dictionary + { + { ValidatorType, validatorName }, + { ValidationId, validationId.ToString()} + }); } } diff --git a/src/PackageLagMonitor/ISearchServiceClient.cs b/src/PackageLagMonitor/ISearchServiceClient.cs new file mode 100644 index 000000000..31ccc9250 --- /dev/null +++ b/src/PackageLagMonitor/ISearchServiceClient.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Jobs.Montoring.PackageLag +{ + public interface ISearchServiceClient + { + Task GetCommitDateTimeAsync( + Instance instance, + CancellationToken token); + + Task> GetSearchEndpointsAsync( + RegionInformation regionInformation, + CancellationToken token); + } +} \ No newline at end of file diff --git a/src/PackageLagMonitor/Instance.cs b/src/PackageLagMonitor/Instance.cs index 568033811..605082bd8 100644 --- a/src/PackageLagMonitor/Instance.cs +++ b/src/PackageLagMonitor/Instance.cs @@ -1,13 +1,26 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + namespace NuGet.Jobs.Montoring.PackageLag { public class Instance { - public int Index { get; set; } - public string DiagUrl { get; set; } - public string BaseQueryUrl { get; set; } - public string Region { get; set; } + public Instance(string slot, int index, string diagUrl, string baseQueryUrl, string region) + { + Slot = slot ?? throw new ArgumentNullException(nameof(slot)); + Index = index; + DiagUrl = diagUrl ?? throw new ArgumentNullException(nameof(diagUrl)); + BaseQueryUrl = baseQueryUrl ?? throw new ArgumentNullException(nameof(baseQueryUrl)); + Region = region ?? throw new ArgumentNullException(nameof(region)); + } + + public string Slot { get; } + public int Index { get; } + public string DiagUrl { get; } + public string BaseQueryUrl { get; } + public string Region { get; } } } diff --git a/src/PackageLagMonitor/Job.cs b/src/PackageLagMonitor/Job.cs index 07f880513..5fbc8de37 100644 --- a/src/PackageLagMonitor/Job.cs +++ b/src/PackageLagMonitor/Job.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -31,11 +30,6 @@ public class Job : JobBase private const string AzureManagementSectionName = "AzureManagement"; private const string MonitorConfigurationSectionName = "MonitorConfiguration"; - - /// - /// To be used for request - /// - private const string ProductionSlot = "production"; private const int MAX_CATALOG_RETRY_COUNT = 5; private static readonly TimeSpan KeyVaultSecretCachingTimeout = TimeSpan.FromDays(1); @@ -43,6 +37,7 @@ public class Job : JobBase private IAzureManagementAPIWrapper _azureManagementApiWrapper; private IPackageLagTelemetryService _telemetryService; private HttpClient _httpClient; + private ISearchServiceClient _searchServiceClient; private ICatalogClient _catalogClient; private IServiceProvider _serviceProvider; private PackageLagMonitorConfiguration _configuration; @@ -56,6 +51,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); _catalogClient = _serviceProvider.GetService(); _httpClient = _serviceProvider.GetService(); + _searchServiceClient = _serviceProvider.GetService(); _telemetryService = _serviceProvider.GetService(); } @@ -127,6 +123,7 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo services.AddTransient(p => p.GetService>().Value); services.AddSingleton(); services.AddSingleton(); + services.AddTransient(); } private static IServiceProvider CreateProvider(IServiceCollection services) @@ -142,7 +139,13 @@ public async override Task Run() var token = new CancellationToken(); try { - var instances = await GetSearchEndpointsAsync(token); + var regionInformations = _configuration.RegionInformations; + var instances = new List(); + + foreach (var regionInformation in regionInformations) + { + instances.AddRange(await _searchServiceClient.GetSearchEndpointsAsync(regionInformation, token)); + } var maxCommit = DateTimeOffset.MinValue; @@ -150,19 +153,9 @@ public async override Task Run() { try { - using (var diagResponse = await _httpClient.GetAsync( - instance.DiagUrl, - HttpCompletionOption.ResponseContentRead, - token)) - { - var diagContent = diagResponse.Content; - var searchDiagResultRaw = await diagContent.ReadAsStringAsync(); - var searchDiagResultObject = JsonConvert.DeserializeObject(searchDiagResultRaw); - - var commitDateTime = DateTimeOffset.Parse(searchDiagResultObject.CommitUserData.CommitTimeStamp); + var commitDateTime = await _searchServiceClient.GetCommitDateTimeAsync(instance, token); - maxCommit = commitDateTime > maxCommit ? commitDateTime : maxCommit; - } + maxCommit = commitDateTime > maxCommit ? commitDateTime : maxCommit; } catch (Exception e) { @@ -211,61 +204,5 @@ public async override Task Run() return; } } - - private async Task> GetSearchEndpointsAsync(CancellationToken token) - { - var regionInformations = _configuration.RegionInformations; - var subscription = _configuration.Subscription; - var instances = new List(); - - foreach (var regionInformation in regionInformations) - { - string result = await _azureManagementApiWrapper.GetCloudServicePropertiesAsync( - subscription, - regionInformation.ResourceGroup, - regionInformation.ServiceName, - ProductionSlot, - token); - - var cloudService = AzureHelper.ParseCloudServiceProperties(result); - - instances.AddRange(GetInstances(cloudService.Uri, cloudService.InstanceCount, regionInformation.Region)); - } - - return instances; - } - - private List GetInstances(Uri endpointUri, int instanceCount, string region) - { - var instancePortMinimum = _configuration.InstancePortMinimum; - - Logger.LogInformation("Testing {InstanceCount} instances, starting at port {InstancePortMinimum}.", instanceCount, instancePortMinimum); - - return Enumerable - .Range(0, instanceCount) - .Select(i => - { - var diagUriBuilder = new UriBuilder(endpointUri); - - diagUriBuilder.Scheme = "https"; - diagUriBuilder.Port = instancePortMinimum + i; - diagUriBuilder.Path = "search/diag"; - - var queryBaseUriBuilder = new UriBuilder(endpointUri); - - queryBaseUriBuilder.Scheme = "https"; - queryBaseUriBuilder.Port = instancePortMinimum + i; - queryBaseUriBuilder.Path = "search/query"; - - return new Instance - { - Index = i, - DiagUrl = diagUriBuilder.Uri.ToString(), - BaseQueryUrl = queryBaseUriBuilder.Uri.ToString(), - Region = region - }; - }) - .ToList(); - } } } diff --git a/src/PackageLagMonitor/Monitoring.PackageLag.csproj b/src/PackageLagMonitor/Monitoring.PackageLag.csproj index cb274210e..a41d91d01 100644 --- a/src/PackageLagMonitor/Monitoring.PackageLag.csproj +++ b/src/PackageLagMonitor/Monitoring.PackageLag.csproj @@ -49,13 +49,17 @@ + + + + @@ -107,7 +111,7 @@ 0.5.0-CI-20180510-012541 - 2.25.0 + 2.26.0 2.25.0 diff --git a/src/PackageLagMonitor/PackageLagMonitorConfiguration.cs b/src/PackageLagMonitor/PackageLagMonitorConfiguration.cs index 6c16520b2..7224c1109 100644 --- a/src/PackageLagMonitor/PackageLagMonitorConfiguration.cs +++ b/src/PackageLagMonitor/PackageLagMonitorConfiguration.cs @@ -1,27 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Collections.Generic; namespace NuGet.Jobs.Montoring.PackageLag { - public class PackageLagMonitorConfiguration + public class PackageLagMonitorConfiguration : SearchServiceConfiguration { - public int InstancePortMinimum { get; set; } - public string ServiceIndexUrl { get; set; } - - public string Subscription { get; set; } - - public List RegionInformations { get; set; } - } - - public class RegionInformation - { - public string ResourceGroup { get; set; } - - public string ServiceName { get; set; } - - public string Region { get; set; } } } diff --git a/src/PackageLagMonitor/RegionInformation.cs b/src/PackageLagMonitor/RegionInformation.cs new file mode 100644 index 000000000..4eddadb77 --- /dev/null +++ b/src/PackageLagMonitor/RegionInformation.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + + +namespace NuGet.Jobs.Montoring.PackageLag +{ + public class RegionInformation + { + public string ResourceGroup { get; set; } + + public string ServiceName { get; set; } + + public string Region { get; set; } + } +} diff --git a/src/PackageLagMonitor/SearchServiceClient.cs b/src/PackageLagMonitor/SearchServiceClient.cs new file mode 100644 index 000000000..16b0f5eb4 --- /dev/null +++ b/src/PackageLagMonitor/SearchServiceClient.cs @@ -0,0 +1,116 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using NuGet.Services.AzureManagement; + +namespace NuGet.Jobs.Montoring.PackageLag +{ + public class SearchServiceClient : ISearchServiceClient + { + /// + /// To be used for request + /// + private const string ProductionSlot = "production"; + + private readonly IAzureManagementAPIWrapper _azureManagementApiWrapper; + private readonly HttpClient _httpClient; + private readonly IOptionsSnapshot _configuration; + private readonly ILogger _logger; + + public SearchServiceClient( + IAzureManagementAPIWrapper azureManagementApiWrapper, + HttpClient httpClient, + IOptionsSnapshot configuration, + ILogger logger) + { + _azureManagementApiWrapper = azureManagementApiWrapper; + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + } + + public async Task GetCommitDateTimeAsync(Instance instance, CancellationToken token) + { + using (var diagResponse = await _httpClient.GetAsync( + instance.DiagUrl, + HttpCompletionOption.ResponseContentRead, + token)) + { + var diagContent = diagResponse.Content; + var searchDiagResultRaw = await diagContent.ReadAsStringAsync(); + var searchDiagResultObject = JsonConvert.DeserializeObject(searchDiagResultRaw); + + var commitDateTime = DateTimeOffset.Parse( + searchDiagResultObject.CommitUserData.CommitTimeStamp, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal); + + return commitDateTime; + } + } + + public async Task> GetSearchEndpointsAsync( + RegionInformation regionInformation, + CancellationToken token) + { + var result = await _azureManagementApiWrapper.GetCloudServicePropertiesAsync( + _configuration.Value.Subscription, + regionInformation.ResourceGroup, + regionInformation.ServiceName, + ProductionSlot, + token); + + var cloudService = AzureHelper.ParseCloudServiceProperties(result); + + var instances = GetInstances(cloudService.Uri, cloudService.InstanceCount, regionInformation); + + return instances; + } + + private List GetInstances(Uri endpointUri, int instanceCount, RegionInformation regionInformation) + { + var instancePortMinimum = _configuration.Value.InstancePortMinimum; + + _logger.LogInformation( + "Testing {InstanceCount} instances, starting at port {InstancePortMinimum} for region {Region}.", + instanceCount, + instancePortMinimum, + regionInformation.Region); + + return Enumerable + .Range(0, instanceCount) + .Select(i => + { + var diagUriBuilder = new UriBuilder(endpointUri); + + diagUriBuilder.Scheme = "https"; + diagUriBuilder.Port = instancePortMinimum + i; + diagUriBuilder.Path = "search/diag"; + + var queryBaseUriBuilder = new UriBuilder(endpointUri); + + queryBaseUriBuilder.Scheme = "https"; + queryBaseUriBuilder.Port = instancePortMinimum + i; + queryBaseUriBuilder.Path = "search/query"; + + return new Instance( + ProductionSlot, + i, + diagUriBuilder.Uri.ToString(), + queryBaseUriBuilder.Uri.ToString(), + regionInformation.Region); + }) + .ToList(); + } + } +} diff --git a/src/PackageLagMonitor/SearchServiceConfiguration.cs b/src/PackageLagMonitor/SearchServiceConfiguration.cs new file mode 100644 index 000000000..f9bac90c9 --- /dev/null +++ b/src/PackageLagMonitor/SearchServiceConfiguration.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace NuGet.Jobs.Montoring.PackageLag +{ + public class SearchServiceConfiguration + { + public int InstancePortMinimum { get; set; } + + public string Subscription { get; set; } + + public List RegionInformations { get; set; } + } +} diff --git a/src/Validation.Common.Job/ExceptionExtensions.cs b/src/Validation.Common.Job/ExceptionExtensions.cs index ac498632c..63aa32787 100644 --- a/src/Validation.Common.Job/ExceptionExtensions.cs +++ b/src/Validation.Common.Job/ExceptionExtensions.cs @@ -5,7 +5,7 @@ using System.Data.SqlClient; using System.Linq; -namespace NuGet.Jobs.Validation.PackageSigning +namespace NuGet.Jobs.Validation { public static class ExceptionExtensions { diff --git a/src/Validation.Common.Job/Storage/AddStatusResult.cs b/src/Validation.Common.Job/Storage/AddStatusResult.cs index 3649ff138..a6ddc7580 100644 --- a/src/Validation.Common.Job/Storage/AddStatusResult.cs +++ b/src/Validation.Common.Job/Storage/AddStatusResult.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace NuGet.Jobs.Validation.PackageSigning.Storage +namespace NuGet.Jobs.Validation.Storage { /// /// The possible results for . diff --git a/src/Validation.Common.Job/Storage/IValidatorStateService.cs b/src/Validation.Common.Job/Storage/IValidatorStateService.cs index f6da43e7d..6e94a3237 100644 --- a/src/Validation.Common.Job/Storage/IValidatorStateService.cs +++ b/src/Validation.Common.Job/Storage/IValidatorStateService.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using NuGet.Services.Validation; -namespace NuGet.Jobs.Validation.PackageSigning.Storage +namespace NuGet.Jobs.Validation.Storage { /// /// A service used to persist a 's validation statuses. diff --git a/src/Validation.Common.Job/Storage/SaveStatusResult.cs b/src/Validation.Common.Job/Storage/SaveStatusResult.cs index dfa2ed55f..b03295f6e 100644 --- a/src/Validation.Common.Job/Storage/SaveStatusResult.cs +++ b/src/Validation.Common.Job/Storage/SaveStatusResult.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace NuGet.Jobs.Validation.PackageSigning.Storage +namespace NuGet.Jobs.Validation.Storage { /// /// The possible results for . diff --git a/src/Validation.Common.Job/Storage/SerializedValidationIssue.cs b/src/Validation.Common.Job/Storage/SerializedValidationIssue.cs index b67bee697..29ee4e64f 100644 --- a/src/Validation.Common.Job/Storage/SerializedValidationIssue.cs +++ b/src/Validation.Common.Job/Storage/SerializedValidationIssue.cs @@ -3,7 +3,7 @@ using NuGet.Services.Validation; -namespace NuGet.Jobs.Validation.PackageSigning.Storage +namespace NuGet.Jobs.Validation.Storage { public class SerializedValidationIssue : IValidationIssue { diff --git a/src/Validation.Common.Job/Storage/ValidatorStateService.cs b/src/Validation.Common.Job/Storage/ValidatorStateService.cs index c24a2368d..21cba38a8 100644 --- a/src/Validation.Common.Job/Storage/ValidatorStateService.cs +++ b/src/Validation.Common.Job/Storage/ValidatorStateService.cs @@ -10,7 +10,7 @@ using Microsoft.Extensions.Logging; using NuGet.Services.Validation; -namespace NuGet.Jobs.Validation.PackageSigning.Storage +namespace NuGet.Jobs.Validation.Storage { public class ValidatorStateService : IValidatorStateService { diff --git a/src/Validation.Common.Job/Storage/ValidatorStatusExtensions.cs b/src/Validation.Common.Job/Storage/ValidatorStatusExtensions.cs index adb0f6f7c..21ccafbdb 100644 --- a/src/Validation.Common.Job/Storage/ValidatorStatusExtensions.cs +++ b/src/Validation.Common.Job/Storage/ValidatorStatusExtensions.cs @@ -5,7 +5,7 @@ using System.Linq; using NuGet.Services.Validation; -namespace NuGet.Jobs.Validation.PackageSigning.Storage +namespace NuGet.Jobs.Validation.Storage { public static class ValidatorStatusExtensions { diff --git a/src/Validation.Common.Job/Validation/ValidatorName.cs b/src/Validation.Common.Job/Validation/ValidatorName.cs index 40cf05cdb..af6a2e90c 100644 --- a/src/Validation.Common.Job/Validation/ValidatorName.cs +++ b/src/Validation.Common.Job/Validation/ValidatorName.cs @@ -11,5 +11,6 @@ public static class ValidatorName public const string ScanOnly = "ScanOnly"; public const string PackageSignatureProcessor = "PackageSigningValidator"; public const string PackageSignatureValidator = "PackageSigningValidator2"; + public const string SymbolsValidator = "SymbolsValidator"; } } diff --git a/src/Validation.PackageSigning.ProcessSignature/Program.cs b/src/Validation.PackageSigning.ProcessSignature/Program.cs index ef67a165c..b23db59e6 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Program.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Program.cs @@ -8,7 +8,7 @@ public class Program public static void Main(string[] args) { var job = new Job(); - JobRunner.Run(job, args).GetAwaiter().GetResult(); + JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); } } } diff --git a/src/Validation.PackageSigning.ProcessSignature/SignatureValidationMessageHandler.cs b/src/Validation.PackageSigning.ProcessSignature/SignatureValidationMessageHandler.cs index f9da077a5..3dc856e3b 100644 --- a/src/Validation.PackageSigning.ProcessSignature/SignatureValidationMessageHandler.cs +++ b/src/Validation.PackageSigning.ProcessSignature/SignatureValidationMessageHandler.cs @@ -9,7 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs.Validation.PackageSigning.Messages; -using NuGet.Jobs.Validation.PackageSigning.Storage; +using NuGet.Jobs.Validation.Storage; using NuGet.Packaging.Signing; using NuGet.Services.ServiceBus; using NuGet.Services.Validation; diff --git a/src/Validation.PackageSigning.ValidateCertificate/Program.cs b/src/Validation.PackageSigning.ValidateCertificate/Program.cs index b7d4e1e12..0d325968c 100644 --- a/src/Validation.PackageSigning.ValidateCertificate/Program.cs +++ b/src/Validation.PackageSigning.ValidateCertificate/Program.cs @@ -10,7 +10,7 @@ class Program static void Main(string[] args) { var job = new Job(); - JobRunner.Run(job, args).GetAwaiter().GetResult(); + JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); } } } diff --git a/src/Validation.Symbols.Core/ISymbolsValidatorMessage.cs b/src/Validation.Symbols.Core/ISymbolsValidatorMessage.cs new file mode 100644 index 000000000..fe94061b2 --- /dev/null +++ b/src/Validation.Symbols.Core/ISymbolsValidatorMessage.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Jobs.Validation.Symbols.Core +{ + interface ISymbolsValidatorMessage + { + /// + /// The Validation id + /// Validation Id will be unique per validator validation. + /// It should be a 1:1 matching between ValidationId and PackageValidation.Key + /// + Guid ValidationId { get; } + + /// + /// The id of the symbol package. + /// + int SymbolsPackageKey { get; } + + /// + /// The package Id. + /// + string PackageId { get; } + + /// + /// The package normalized version. + /// + string PackageNormalizedVersion { get; } + + /// + /// The Url of the snupkg. + /// + string SnupkgUrl { get; } + } +} diff --git a/src/Validation.Symbols.Core/Properties/AssemblyInfo.cs b/src/Validation.Symbols.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..7a856c0b4 --- /dev/null +++ b/src/Validation.Symbols.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,15 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Validation.Symbols.Core")] +[assembly: AssemblyDescription("Shared libraries for package signing validation")] +[assembly: AssemblyCompany(".NET Foundation")] +[assembly: AssemblyProduct("Validation.Symbols.Core")] +[assembly: AssemblyCopyright("Copyright © .NET Foundation 2017")] +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("17510a22-176f-4e96-a867-e79f1b54f54f")] diff --git a/src/Validation.Symbols.Core/SymbolsValidatorMessage.cs b/src/Validation.Symbols.Core/SymbolsValidatorMessage.cs new file mode 100644 index 000000000..0592bd4ff --- /dev/null +++ b/src/Validation.Symbols.Core/SymbolsValidatorMessage.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Jobs.Validation.Symbols.Core +{ + public class SymbolsValidatorMessage : ISymbolsValidatorMessage + { + public SymbolsValidatorMessage(Guid validationId, + int symbolPackageKey, + string packageId, + string packageNormalizedVersion, + string snupkgUrl) + { + ValidationId = validationId; + SymbolsPackageKey = symbolPackageKey; + PackageId = packageId; + PackageNormalizedVersion = packageNormalizedVersion; + SnupkgUrl = snupkgUrl; + } + + public Guid ValidationId { get; } + + public int SymbolsPackageKey { get; } + + public string PackageId { get; } + + public string PackageNormalizedVersion { get; } + + public string SnupkgUrl { get; } + } +} diff --git a/src/Validation.Symbols.Core/SymbolsValidatorMessageSerializer.cs b/src/Validation.Symbols.Core/SymbolsValidatorMessageSerializer.cs new file mode 100644 index 000000000..68dc6332b --- /dev/null +++ b/src/Validation.Symbols.Core/SymbolsValidatorMessageSerializer.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.ServiceBus; + +namespace NuGet.Jobs.Validation.Symbols.Core +{ + public class SymbolsValidatorMessageSerializer : IBrokeredMessageSerializer + { + private const string SchemaName = "SignatureValidationMessageData"; + + private IBrokeredMessageSerializer _serializer = + new BrokeredMessageSerializer(); + + public SymbolsValidatorMessage Deserialize(IBrokeredMessage message) + { + var deserializedMessage = _serializer.Deserialize(message); + + return new SymbolsValidatorMessage( + deserializedMessage.ValidationId, + deserializedMessage.SymbolsPackageKey, + deserializedMessage.PackageId, + deserializedMessage.PackageNormalizedVersion, + deserializedMessage.SnupkgUrl); + } + + public IBrokeredMessage Serialize(SymbolsValidatorMessage message) + => _serializer.Serialize(new SymbolsValidatorMessageDataV1 + { + ValidationId = message.ValidationId, + SymbolsPackageKey = message.SymbolsPackageKey, + PackageId = message.PackageId, + PackageNormalizedVersion = message.PackageNormalizedVersion, + SnupkgUrl = message.SnupkgUrl + }); + + [Schema(Name = SchemaName, Version = 1)] + private class SymbolsValidatorMessageDataV1 + { + public Guid ValidationId { get; set; } + + public int SymbolsPackageKey { get; set; } + + public string PackageId { get; set; } + + public string PackageNormalizedVersion { get; set; } + + public string SnupkgUrl { get; set; } + } + } +} diff --git a/src/Validation.Symbols.Core/Validation.Symbols.Core.csproj b/src/Validation.Symbols.Core/Validation.Symbols.Core.csproj new file mode 100644 index 000000000..1ddf8cb31 --- /dev/null +++ b/src/Validation.Symbols.Core/Validation.Symbols.Core.csproj @@ -0,0 +1,65 @@ + + + + + Debug + AnyCPU + {17510A22-176F-4E96-A867-E79F1B54F54F} + Library + Properties + Validation.Symbols.Core + Validation.Symbols.Core + v4.6.2 + 512 + true + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + + + + + + + + + + + + + + + + + + + + 2.26.0-sb-status-34752 + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/test.ps1 b/test.ps1 index 0c42d19a2..9c7f8e907 100644 --- a/test.ps1 +++ b/test.ps1 @@ -37,7 +37,8 @@ Function Run-Tests { "tests\Validation.PackageSigning.ValidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.ValidateCertificate.Tests.dll", ` "tests\Validation.PackageSigning.RevalidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.RevalidateCertificate.Tests.dll", ` "tests\Validation.PackageSigning.Core.Tests\bin\$Configuration\Validation.PackageSigning.Core.Tests.dll", ` - "tests\Validation.Common.Job.Tests\bin\$Configuration\Validation.Common.Job.Tests.dll" + "tests\Validation.Common.Job.Tests\bin\$Configuration\Validation.Common.Job.Tests.dll", ` + "tests\Monitoring.RebootSearchInstance.Tests\bin\$Configuration\NuGet.Monitoring.RebootSearchInstance.Tests.dll" $TestCount = 0 diff --git a/tests/Monitoring.RebootSearchInstance.Tests/Monitoring.RebootSearchInstance.Tests.csproj b/tests/Monitoring.RebootSearchInstance.Tests/Monitoring.RebootSearchInstance.Tests.csproj new file mode 100644 index 000000000..8c1e4a2a6 --- /dev/null +++ b/tests/Monitoring.RebootSearchInstance.Tests/Monitoring.RebootSearchInstance.Tests.csproj @@ -0,0 +1,71 @@ + + + + + Debug + AnyCPU + {21C0A0EE-8696-4013-950F-D6495D0C6E40} + Library + Properties + NuGet.Monitoring.RebootSearchInstance + NuGet.Monitoring.RebootSearchInstance.Tests + v4.6.2 + 512 + true + PackageReference + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + {ecd8dfce-8e3c-4510-afe3-d7ec168e8d66} + Monitoring.RebootSearchInstance + + + {B5147169-E941-4CF8-9FCD-1C123ACD3149} + Monitoring.PackageLag + + + {b4b7564a-965b-447b-927f-6749e2c08880} + Validation.PackageSigning.Core.Tests + + + + + 4.7.145 + + + 2.3.1 + + + 2.3.1 + + + + \ No newline at end of file diff --git a/tests/Monitoring.RebootSearchInstance.Tests/Properties/AssemblyInfo.cs b/tests/Monitoring.RebootSearchInstance.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..49273fee3 --- /dev/null +++ b/tests/Monitoring.RebootSearchInstance.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Monitoring.RebootSearchInstance.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("21c0a0ee-8696-4013-950f-d6495d0c6e40")] diff --git a/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs b/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs new file mode 100644 index 000000000..ccef2e6ae --- /dev/null +++ b/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs @@ -0,0 +1,273 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Jobs.Montoring.PackageLag; +using NuGet.Services.AzureManagement; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class SearchInstanceRebooterFacts + { + private const string _region = "USSC"; + private const string _slot = "Production"; + private const string _resourceGroup = "test-rg"; + private const string _serviceName = "test-search-0"; + private const string _role = "SearchService"; + private const string _subscription = "TEST"; + private readonly ITestOutputHelper _output; + private readonly Mock _feedClient; + private readonly Mock _searchServiceClient; + private readonly Mock _azureManagementAPIWrapper; + private readonly Mock _telemetryService; + private readonly Mock> _configurationMock; + private readonly MonitorConfiguration _configuration; + private readonly ILogger _logger; + private readonly CancellationToken _token; + private readonly List _instances; + private readonly DateTimeOffset _feedTimestamp; + private readonly SearchInstanceRebooter _target; + + public SearchInstanceRebooterFacts(ITestOutputHelper output) + { + _output = output; + _feedClient = new Mock(); + _searchServiceClient = new Mock(); + _azureManagementAPIWrapper = new Mock(); + _telemetryService = new Mock(); + _configurationMock = new Mock>(); + _configuration = new MonitorConfiguration + { + ProcessLifetime = TimeSpan.Zero, + SleepDuration = TimeSpan.Zero, + HealthyThresholdInSeconds = 60, + UnhealthyThresholdInSeconds = 120, + Role = _role, + Subscription = _subscription, + RoleInstanceFormat = "RoleInstance_{0}", + RegionInformations = new List + { + new RegionInformation + { + Region = _region, + ResourceGroup = _resourceGroup, + ServiceName = _serviceName, + }, + }, + }; + _logger = new LoggerFactory() + .AddXunit(_output) + .CreateLogger(); + + _token = CancellationToken.None; + _instances = new List + { + new Instance( + _slot, + 0, + "http://localhost:801/search/diag", + "http://localhost:801/query", + _region), + new Instance( + _slot, + 1, + "http://localhost:802/search/diag", + "http://localhost:802/query", + _region), + new Instance( + _slot, + 2, + "http://localhost:803/search/diag", + "http://localhost:803/query", + _region), + }; + _feedTimestamp = new DateTimeOffset(2018, 1, 1, 8, 0, 0, TimeSpan.Zero); + + _configurationMock + .Setup(x => x.Value) + .Returns(() => _configuration); + _searchServiceClient + .Setup(x => x.GetSearchEndpointsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _instances); + _feedClient + .Setup(x => x.GetLatestFeedTimeStampAsync()) + .ReturnsAsync(() => _feedTimestamp); + + _target = new SearchInstanceRebooter( + _feedClient.Object, + _searchServiceClient.Object, + _azureManagementAPIWrapper.Object, + _telemetryService.Object, + _configurationMock.Object, + _logger); + } + + [Fact] + public async Task RestartsFirstUnhealthyInstance() + { + _searchServiceClient + .Setup(x => x.GetCommitDateTimeAsync(It.Is(i => i.Index == 0), It.IsAny())) + .ReturnsAsync(DateTimeOffset.MaxValue); + _searchServiceClient + .SetupSequence(x => x.GetCommitDateTimeAsync(It.Is(i => i.Index == 1), It.IsAny())) + .ReturnsAsync(DateTimeOffset.MinValue) + .ThrowsAsync(new IOException("The instance is not up yet")); + _searchServiceClient + .Setup(x => x.GetCommitDateTimeAsync(It.Is(i => i.Index == 2), It.IsAny())) + .ReturnsAsync(DateTimeOffset.MinValue); + + await _target.RunAsync(_token); + + _azureManagementAPIWrapper.Verify( + x => x.RebootCloudServiceRoleInstanceAsync( + _subscription, + _resourceGroup, + _serviceName, + "Production", + _role, + "RoleInstance_1", + It.IsAny()), + Times.Once); + _azureManagementAPIWrapper.Verify( + x => x.RebootCloudServiceRoleInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + + _telemetryService.Verify(x => x.TrackHealthyInstanceCount(_region, 1), Times.Once); + _telemetryService.Verify(x => x.TrackUnhealthyInstanceCount(_region, 2), Times.Once); + _telemetryService.Verify(x => x.TrackUnknownInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackInstanceCount(_region, 3), Times.Once); + + _telemetryService.Verify(x => x.TrackInstanceReboot(_region, 1), Times.Once); + _telemetryService.Verify(x => x.TrackInstanceReboot(It.IsAny(), It.IsAny()), Times.Once); + + _telemetryService.Verify( + x => x.TrackInstanceRebootDuration(_region, 1, It.IsAny(), InstanceHealth.Unknown), + Times.Once); + _telemetryService.Verify( + x => x.TrackInstanceRebootDuration( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task TreatsExceptionWhenGettingCommitTimestampAsUnknown() + { + _searchServiceClient + .SetupSequence(x => x.GetCommitDateTimeAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new IOException("Some problem with the network.")) + .ReturnsAsync(DateTimeOffset.MaxValue) + .ReturnsAsync(DateTimeOffset.MaxValue); + + await _target.RunAsync(_token); + + _azureManagementAPIWrapper.Verify( + x => x.RebootCloudServiceRoleInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + _telemetryService.Verify(x => x.TrackHealthyInstanceCount(_region, 2), Times.Once); + _telemetryService.Verify(x => x.TrackUnhealthyInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackUnknownInstanceCount(_region, 1), Times.Once); + _telemetryService.Verify(x => x.TrackInstanceCount(_region, 3), Times.Once); + } + + [Fact] + public async Task TreatsLagBetweenThresholdsAsUnknown() + { + _searchServiceClient + .Setup(x => x.GetCommitDateTimeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(_feedTimestamp.Subtract(TimeSpan.FromSeconds(_configuration.UnhealthyThresholdInSeconds - 1))); + + await _target.RunAsync(_token); + + _azureManagementAPIWrapper.Verify( + x => x.RebootCloudServiceRoleInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + _telemetryService.Verify(x => x.TrackHealthyInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackUnhealthyInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackUnknownInstanceCount(_region, 3), Times.Once); + _telemetryService.Verify(x => x.TrackInstanceCount(_region, 3), Times.Once); + } + + [Fact] + public async Task DoesNothingWhenThereAreNoUnhealthyInstances() + { + _searchServiceClient + .Setup(x => x.GetCommitDateTimeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => DateTimeOffset.MaxValue); + + await _target.RunAsync(_token); + + _azureManagementAPIWrapper.Verify( + x => x.RebootCloudServiceRoleInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + _telemetryService.Verify(x => x.TrackHealthyInstanceCount(_region, 3), Times.Once); + _telemetryService.Verify(x => x.TrackUnhealthyInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackUnknownInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackInstanceCount(_region, 3), Times.Once); + } + + [Fact] + public async Task DoesNothingWhenThereAreNoHealthyInstances() + { + _searchServiceClient + .Setup(x => x.GetCommitDateTimeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => DateTimeOffset.MinValue); + + await _target.RunAsync(_token); + + _azureManagementAPIWrapper.Verify( + x => x.RebootCloudServiceRoleInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + _telemetryService.Verify(x => x.TrackHealthyInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackUnhealthyInstanceCount(_region, 3), Times.Once); + _telemetryService.Verify(x => x.TrackUnknownInstanceCount(_region, 0), Times.Once); + _telemetryService.Verify(x => x.TrackInstanceCount(_region, 3), Times.Once); + } + } +} diff --git a/tests/Monitoring.RebootSearchInstance.Tests/TelemetryServiceFacts.cs b/tests/Monitoring.RebootSearchInstance.Tests/TelemetryServiceFacts.cs new file mode 100644 index 000000000..a0e83b4e0 --- /dev/null +++ b/tests/Monitoring.RebootSearchInstance.Tests/TelemetryServiceFacts.cs @@ -0,0 +1,154 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.Logging; +using Xunit; + +namespace NuGet.Monitoring.RebootSearchInstance +{ + public class TelemetryServiceFacts + { + private readonly Mock _telemetryClient; + private readonly Mock> _configurationMock; + private readonly MonitorConfiguration _configuration; + private const string _subscription = "TEST"; + private const string _region = "USSC"; + private const int _count = 42; + private const int _index = 1; + private static readonly TimeSpan _duration = TimeSpan.FromMilliseconds(2342); + private const InstanceHealth _health = InstanceHealth.Unknown; + private readonly TelemetryService _target; + private IDictionary _properties; + + public TelemetryServiceFacts() + { + _telemetryClient = new Mock(); + _configurationMock = new Mock>(); + _configuration = new MonitorConfiguration + { + Subscription = _subscription, + }; + + _telemetryClient + .Setup(x => x.TrackMetric(It.IsAny(), It.IsAny(), It.IsAny>())) + .Callback>((_, __, p) => _properties = p); + _configurationMock + .Setup(x => x.Value) + .Returns(() => _configuration); + + _target = new TelemetryService( + _telemetryClient.Object, + _configurationMock.Object); + } + + [Fact] + public void TrackHealthyInstanceCount() + { + _target.TrackHealthyInstanceCount(_region, _count); + + _telemetryClient.Verify( + x => x.TrackMetric( + "RebootSearchInstance.HealthyInstances", + _count, + It.IsAny>()), + Times.Once); + Assert.NotNull(_properties); + Assert.Equal(new[] { "Region", "Subscription" }, _properties.Keys.OrderBy(x => x)); + Assert.Equal(_region, _properties["Region"]); + Assert.Equal(_subscription, _properties["Subscription"]); + } + + [Fact] + public void TrackUnhealthyInstanceCount() + { + _target.TrackUnhealthyInstanceCount(_region, _count); + + _telemetryClient.Verify( + x => x.TrackMetric( + "RebootSearchInstance.UnhealthyInstances", + _count, + It.IsAny>()), + Times.Once); + Assert.NotNull(_properties); + Assert.Equal(new[] { "Region", "Subscription" }, _properties.Keys.OrderBy(x => x)); + Assert.Equal(_region, _properties["Region"]); + Assert.Equal(_subscription, _properties["Subscription"]); + } + + [Fact] + public void TrackUnknownInstanceCount() + { + _target.TrackUnknownInstanceCount(_region, _count); + + _telemetryClient.Verify( + x => x.TrackMetric( + "RebootSearchInstance.UnknownInstances", + _count, + It.IsAny>()), + Times.Once); + Assert.NotNull(_properties); + Assert.Equal(new[] { "Region", "Subscription" }, _properties.Keys.OrderBy(x => x)); + Assert.Equal(_region, _properties["Region"]); + Assert.Equal(_subscription, _properties["Subscription"]); + } + + [Fact] + public void TrackInstanceCount() + { + _target.TrackInstanceCount(_region, _count); + + _telemetryClient.Verify( + x => x.TrackMetric( + "RebootSearchInstance.Instances", + _count, + It.IsAny>()), + Times.Once); + Assert.NotNull(_properties); + Assert.Equal(new[] { "Region", "Subscription" }, _properties.Keys.OrderBy(x => x)); + Assert.Equal(_region, _properties["Region"]); + Assert.Equal(_subscription, _properties["Subscription"]); + } + + [Fact] + public void TrackInstanceReboot() + { + _target.TrackInstanceReboot(_region, _index); + + _telemetryClient.Verify( + x => x.TrackMetric( + "RebootSearchInstance.InstanceReboot", + 1, + It.IsAny>()), + Times.Once); + Assert.NotNull(_properties); + Assert.Equal(new[] { "InstanceIndex", "Region", "Subscription" }, _properties.Keys.OrderBy(x => x)); + Assert.Equal(_index.ToString(), _properties["InstanceIndex"]); + Assert.Equal(_region, _properties["Region"]); + Assert.Equal(_subscription, _properties["Subscription"]); + } + + [Fact] + public void TrackInstanceRebootDuration() + { + _target.TrackInstanceRebootDuration(_region, _index, _duration, _health); + + _telemetryClient.Verify( + x => x.TrackMetric( + "RebootSearchInstance.InstanceRebootDurationSeconds", + _duration.TotalSeconds, + It.IsAny>()), + Times.Once); + Assert.NotNull(_properties); + Assert.Equal(new[] { "Health", "InstanceIndex", "Region", "Subscription" }, _properties.Keys.OrderBy(x => x)); + Assert.Equal("Unknown", _properties["Health"]); + Assert.Equal(_index.ToString(), _properties["InstanceIndex"]); + Assert.Equal(_region, _properties["Region"]); + Assert.Equal(_subscription, _properties["Subscription"]); + } + } +} diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/NuGet.Services.Validation.Orchestrator.Tests.csproj b/tests/NuGet.Services.Validation.Orchestrator.Tests/NuGet.Services.Validation.Orchestrator.Tests.csproj index 0c692ad0d..0139c47c8 100644 --- a/tests/NuGet.Services.Validation.Orchestrator.Tests/NuGet.Services.Validation.Orchestrator.Tests.csproj +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/NuGet.Services.Validation.Orchestrator.Tests.csproj @@ -55,6 +55,8 @@ + + @@ -105,6 +107,10 @@ {91C060DA-736F-4DA9-A57F-CB3AC0E6CB10} Validation.PackageSigning.Core + + {17510a22-176f-4e96-a867-e79f1b54f54f} + Validation.Symbols.Core + {b4b7564a-965b-447b-927f-6749e2c08880} Validation.PackageSigning.Core.Tests diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureProcessorFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureProcessorFacts.cs index 548d3c4ea..700750f8b 100644 --- a/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureProcessorFacts.cs +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureProcessorFacts.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Logging; using Moq; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageSigning.ProcessSignature; diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureValidatorFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureValidatorFacts.cs index 21b4e9a45..496a17e66 100644 --- a/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureValidatorFacts.cs +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ProcessSignature/PackageSignatureValidatorFacts.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Options; using Moq; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign; using NuGet.Services.Validation.Orchestrator.Telemetry; diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ValidateCertificate/PackageCertificatesValidatorFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ValidateCertificate/PackageCertificatesValidatorFacts.cs index 8a9b2a914..abed60eb8 100644 --- a/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ValidateCertificate/PackageCertificatesValidatorFacts.cs +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/PackageSigning/ValidateCertificate/PackageCertificatesValidatorFacts.cs @@ -8,7 +8,7 @@ using Microsoft.Extensions.Logging; using Moq; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; +using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageSigning.ValidateCertificate; using Tests.ContextHelpers; diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/Symbol/SymbolMessageEnqueuerFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/Symbol/SymbolMessageEnqueuerFacts.cs new file mode 100644 index 000000000..ec89e1a44 --- /dev/null +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/Symbol/SymbolMessageEnqueuerFacts.cs @@ -0,0 +1,74 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Validation.Symbols.Core; +using NuGet.Services.ServiceBus; +using Moq; +using Xunit; + + +namespace NuGet.Services.Validation.Symbols +{ + public class SymbolMessageEnqueuerFacts + { + [Fact] + public async Task SendsSerializeMessage() + { + SymbolsValidatorMessage message = null; + _serializer + .Setup(x => x.Serialize(It.IsAny())) + .Returns(() => _brokeredMessage.Object) + .Callback(x => message = x); + + await _target.EnqueueSymbolsValidationMessageAsync(_validationRequest.Object); + + Assert.Equal(_validationRequest.Object.ValidationId, message.ValidationId); + Assert.Equal(_validationRequest.Object.PackageId, message.PackageId); + Assert.Equal(_validationRequest.Object.PackageVersion, message.PackageNormalizedVersion); + Assert.Equal(_validationRequest.Object.NupkgUrl, message.SnupkgUrl); + _serializer.Verify( + x => x.Serialize(It.IsAny()), + Times.Once); + _topicClient.Verify(x => x.SendAsync(_brokeredMessage.Object), Times.Once); + _topicClient.Verify(x => x.SendAsync(It.IsAny()), Times.Once); + } + + private readonly Mock _topicClient; + private readonly Mock> _serializer; + private readonly Mock> _options; + private readonly SymbolsValidationConfiguration _configuration; + private readonly Mock _brokeredMessage; + private readonly Mock _validationRequest; + private readonly SymbolsMessageEnqueuer _target; + + public SymbolMessageEnqueuerFacts() + { + _configuration = new SymbolsValidationConfiguration(); + _brokeredMessage = new Mock(); + _validationRequest = new Mock(); + + _validationRequest.Setup(x => x.ValidationId).Returns(new Guid("ab2629ce-2d67-403a-9a42-49748772ae90")); + _validationRequest.Setup(x => x.PackageId).Returns("NuGet.Versioning"); + _validationRequest.Setup(x => x.PackageVersion).Returns("4.6.0"); + _validationRequest.Setup(x => x.NupkgUrl).Returns("http://example/nuget.versioning.4.6.0.nupkg?my-sas"); + _brokeredMessage.SetupProperty(x => x.ScheduledEnqueueTimeUtc); + + _topicClient = new Mock(); + _serializer = new Mock>(); + _options = new Mock>(); + + _options.Setup(x => x.Value).Returns(() => _configuration); + _serializer + .Setup(x => x.Serialize(It.IsAny())) + .Returns(() => _brokeredMessage.Object); + + _target = new SymbolsMessageEnqueuer( + _topicClient.Object, + _serializer.Object, + _options.Object); + } + } +} diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/Symbol/SymbolValidatorFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/Symbol/SymbolValidatorFacts.cs new file mode 100644 index 000000000..0f1d5d0a5 --- /dev/null +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/Symbol/SymbolValidatorFacts.cs @@ -0,0 +1,272 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Jobs.Validation; +using NuGet.Jobs.Validation.Storage; +using NuGet.Services.Validation.Orchestrator.Telemetry; +using NuGet.Services.Validation.Symbols; + +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.Validation.Orchestrator.Tests.Symbol +{ + public class SymbolValidatorFacts + { + private const int PackageKey = 1001; + private const string PackageId = "NuGet.Versioning"; + private const string PackageVersion = "1.2.3"; + private static readonly Guid ValidationId = new Guid("12345678-1234-1234-1234-123456789012"); + private const string NupkgUrl = "https://example/nuget.versioning/1.2.3/package.nupkg"; + + public class TheGetStatusMethod : FactsBase + { + private static readonly ValidationStatus[] possibleValidationStatuses = new ValidationStatus[] + { + ValidationStatus.Incomplete, + ValidationStatus.NotStarted, + ValidationStatus.Succeeded, + }; + + public TheGetStatusMethod(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [MemberData(nameof(PossibleValidationStatuses))] + public async Task ReturnsPersistedStatus(ValidationStatus status) + { + // Arrange + _validatorStateService + .Setup(x => x.GetStatusAsync(It.IsAny())) + .ReturnsAsync(new ValidatorStatus + { + ValidationId = ValidationId, + PackageKey = PackageKey, + ValidatorName = ValidatorName.SymbolsValidator, + State = status, + ValidatorIssues = new List(), + }); + + // Act & Assert + var actual = await _target.GetResultAsync(_validationRequest.Object); + + Assert.Equal(status, actual.Status); + } + + [Fact] + public async Task DoesReturnValidatorIssues() + { + // Arrange + _validatorStateService + .Setup(x => x.GetStatusAsync(It.IsAny())) + .ReturnsAsync(new ValidatorStatus + { + ValidationId = ValidationId, + PackageKey = PackageKey, + ValidatorName = ValidatorName.SymbolsValidator, + State = ValidationStatus.Failed, + ValidatorIssues = new List + { + new ValidatorIssue + { + IssueCode = ValidationIssueCode.Unknown, + Data = "Unknown", + }, + }, + }); + + // Act + var actual = await _target.GetResultAsync(_validationRequest.Object); + + // Assert + Assert.Equal(ValidationStatus.Failed, actual.Status); + Assert.Equal(1, actual.Issues.Count); + } + + public static IEnumerable PossibleValidationStatuses => possibleValidationStatuses.Select(s => new object[] { s }); + } + + public class TheStartValidationAsyncMethod : FactsBase + { + private static readonly ValidationStatus[] startedValidationStatuses = new ValidationStatus[] + { + ValidationStatus.Incomplete, + ValidationStatus.Succeeded, + }; + + public TheStartValidationAsyncMethod(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [MemberData(nameof(StartedValidationStatuses))] + public async Task ReturnsPersistedStatusesIfValidationAlreadyStarted(ValidationStatus status) + { + // Arrange + _validatorStateService + .Setup(x => x.GetStatusAsync(It.IsAny())) + .ReturnsAsync(new ValidatorStatus + { + ValidationId = ValidationId, + PackageKey = PackageKey, + ValidatorName = ValidatorName.SymbolsValidator, + State = status, + ValidatorIssues = new List(), + }); + + // Act & Assert + await _target.StartAsync(_validationRequest.Object); + + _symbolMessageEnqueuer + .Verify(x => x.EnqueueSymbolsValidationMessageAsync(It.IsAny()), Times.Never); + + _validatorStateService + .Verify(x => x.TryAddValidatorStatusAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + + _telemetryService.Verify( + x => x.TrackSymbolsMessageEnqueued(It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task StartsValidationIfNotStarted() + { + // Arrange + // The order of operations is important! The state MUST be persisted AFTER verification has been queued. + var statePersisted = false; + bool verificationQueuedBeforeStatePersisted = false; + + _validatorStateService + .Setup(x => x.GetStatusAsync(It.IsAny())) + .ReturnsAsync(new ValidatorStatus + { + ValidationId = ValidationId, + PackageKey = PackageKey, + ValidatorName = ValidatorName.SymbolsValidator, + State = ValidationStatus.NotStarted, + ValidatorIssues = new List(), + }); + + _symbolMessageEnqueuer + .Setup(x => x.EnqueueSymbolsValidationMessageAsync(It.IsAny())) + .Callback(() => + { + verificationQueuedBeforeStatePersisted = !statePersisted; + }) + .Returns(Task.FromResult(0)); + + _validatorStateService + .Setup(x => x.TryAddValidatorStatusAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => + { + statePersisted = true; + }) + .ReturnsAsync(new ValidatorStatus + { + State = ValidationStatus.Incomplete, + ValidatorIssues = new List(), + }); + + // Act + await _target.StartAsync(_validationRequest.Object); + + // Assert + _symbolMessageEnqueuer + .Verify(x => x.EnqueueSymbolsValidationMessageAsync(It.IsAny()), Times.Once); + + _validatorStateService + .Verify( + x => x.TryAddValidatorStatusAsync( + It.IsAny(), + It.IsAny(), + It.Is(s => s == ValidationStatus.Incomplete)), + Times.Once); + + _telemetryService.Verify( + x => x.TrackSymbolsMessageEnqueued( ValidatorName.SymbolsValidator, _validationRequest.Object.ValidationId), + Times.Once); + + Assert.True(verificationQueuedBeforeStatePersisted); + } + + [Fact] + public async Task DoesReturnValidatorIssues() + { + // Arrange + _validatorStateService + .Setup(x => x.GetStatusAsync(It.IsAny())) + .ReturnsAsync(new ValidatorStatus + { + ValidationId = ValidationId, + PackageKey = PackageKey, + ValidatorName = ValidatorName.SymbolsValidator, + State = ValidationStatus.Failed, + ValidatorIssues = new List + { + new ValidatorIssue + { + IssueCode = ValidationIssueCode.Unknown, + Data = "Unknown", + }, + }, + }); + + // Act + var actual = await _target.StartAsync(_validationRequest.Object); + + // Assert + Assert.Equal(ValidationStatus.Failed, actual.Status); + Assert.Equal(1, actual.Issues.Count); + } + + public static IEnumerable StartedValidationStatuses => startedValidationStatuses.Select(s => new object[] { s }); + } + + public abstract class FactsBase + { + protected readonly Mock _validatorStateService; + protected readonly Mock _symbolMessageEnqueuer; + protected readonly Mock _blobProvider; + protected readonly Mock _telemetryService; + protected readonly ILogger _logger; + protected readonly Mock _validationRequest; + protected readonly SymbolsValidator _target; + + protected readonly SymbolsValidationConfiguration _config; + + public FactsBase(ITestOutputHelper output) + { + _validatorStateService = new Mock(); + _symbolMessageEnqueuer = new Mock(); + _blobProvider = new Mock(); + _config = new SymbolsValidationConfiguration(); + _telemetryService = new Mock(); + var loggerFactory = new LoggerFactory().AddXunit(output); + _logger = loggerFactory.CreateLogger(); + + _validationRequest = new Mock(); + _validationRequest.Setup(x => x.NupkgUrl).Returns(NupkgUrl); + _validationRequest.Setup(x => x.PackageId).Returns(PackageId); + _validationRequest.Setup(x => x.PackageKey).Returns(PackageKey); + _validationRequest.Setup(x => x.PackageVersion).Returns(PackageVersion); + _validationRequest.Setup(x => x.ValidationId).Returns(ValidationId); + + _target = new SymbolsValidator( + _validatorStateService.Object, + _symbolMessageEnqueuer.Object, + _telemetryService.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidatorStateServiceFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidatorStateServiceFacts.cs index a3768082f..51a9b1656 100644 --- a/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidatorStateServiceFacts.cs +++ b/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidatorStateServiceFacts.cs @@ -9,8 +9,7 @@ using Microsoft.Extensions.Logging; using Moq; using NuGet.Jobs.Validation; -using NuGet.Jobs.Validation.PackageSigning.Storage; -using NuGet.Services.Validation.Orchestrator; +using NuGet.Jobs.Validation.Storage; using Tests.ContextHelpers; using Xunit; diff --git a/tests/Validation.PackageSigning.Core.Tests/Storage/ValidatorStatusExtensionsFacts.cs b/tests/Validation.PackageSigning.Core.Tests/Storage/ValidatorStatusExtensionsFacts.cs index 5a2676dbc..fa4e148ac 100644 --- a/tests/Validation.PackageSigning.Core.Tests/Storage/ValidatorStatusExtensionsFacts.cs +++ b/tests/Validation.PackageSigning.Core.Tests/Storage/ValidatorStatusExtensionsFacts.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; -using NuGet.Jobs.Validation.PackageSigning.Storage; +using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation; using Xunit; diff --git a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidationMessageHandlerFacts.cs b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidationMessageHandlerFacts.cs index b861e069b..eb8ece7b6 100644 --- a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidationMessageHandlerFacts.cs +++ b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidationMessageHandlerFacts.cs @@ -11,7 +11,7 @@ using NuGet.Jobs.Validation; using NuGet.Jobs.Validation.PackageSigning.Messages; using NuGet.Jobs.Validation.PackageSigning.ProcessSignature; -using NuGet.Jobs.Validation.PackageSigning.Storage; +using NuGet.Jobs.Validation.Storage; using NuGet.Services.Validation; using Xunit; diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs index 11b1c8de9..608245570 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Jobs.Validation.ScanAndSign; using NuGet.Services.Validation; diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanValidatorFacts.cs b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanValidatorFacts.cs index 9412d10c2..e0f79991c 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanValidatorFacts.cs +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanValidatorFacts.cs @@ -3,12 +3,10 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; -using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Jobs.Validation.Storage; using NuGet.Jobs.Validation.ScanAndSign; using NuGet.Services.Validation;