diff --git a/src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs new file mode 100644 index 000000000..9331f0dec --- /dev/null +++ b/src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Revalidate +{ + /// + /// The configuration needed to query an Application Insights account using + /// the REST endpoints. + /// + public class ApplicationInsightsConfiguration + { + /// + /// The Application Insights account identifier. + /// + public string AppId { get; set; } + + /// + /// The API Key used to access the Application Insights account. + /// + public string ApiKey { get; set; } + } +} diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs index d91d42d1d..252771914 100644 --- a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs +++ b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs @@ -40,6 +40,11 @@ public class RevalidationConfiguration /// public HealthConfiguration Health { get; set; } + /// + /// The configurations to authenticate to Application Insight's REST endpoints. + /// + public ApplicationInsightsConfiguration AppInsights { get; set; } + /// /// The configurations used by the in-memory queue of revalidations to start. /// diff --git a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs index 8768f2d12..b6cff5662 100644 --- a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs +++ b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs @@ -8,6 +8,7 @@ using System.Linq.Expressions; using System.Threading; using Microsoft.Extensions.Logging; +using Newtonsoft.Json; using NuGet.Versioning; using NuGetGallery; @@ -22,6 +23,8 @@ public class PackageFinder : IPackageFinder public const string DependencySetName = "Dependency"; public const string RemainingSetName = "Remaining"; + private const string PreinstalledPackagesResource = "NuGet.Services.Revalidate.Initialization.PreinstalledPackages.json"; + private static int BatchSize = 1000; private static string MicrosoftAccountName = "Microsoft"; @@ -46,16 +49,13 @@ public HashSet FindMicrosoftPackages() public HashSet FindPreinstalledPackages(HashSet except) { - var preinstalledPackagesNames = new HashSet(StringComparer.OrdinalIgnoreCase); + List preinstalledPackagesNames; + var assembly = typeof(PackageFinder).Assembly; - foreach (var path in _config.PreinstalledPaths) + using (var resource = assembly.GetManifestResourceStream(PreinstalledPackagesResource)) + using (var reader = new StreamReader(resource)) { - var expandedPath = Environment.ExpandEnvironmentVariables(path); - var packagesInPath = Directory.GetDirectories(expandedPath) - .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant()) - .Where(d => !d.StartsWith(".")); - - preinstalledPackagesNames.UnionWith(packagesInPath); + preinstalledPackagesNames = JsonConvert.DeserializeObject>(reader.ReadToEnd()); } var preinstalledPackages = FindRegistrationKeys(PreinstalledSetName, r => preinstalledPackagesNames.Contains(r.Id)); diff --git a/src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json b/src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json new file mode 100644 index 000000000..5a778201c --- /dev/null +++ b/src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json @@ -0,0 +1 @@ +["bundlerminifier.core","entityframework","libuv","microsoft.applicationinsights","microsoft.applicationinsights.agent.intercept","microsoft.applicationinsights.aspnetcore","microsoft.applicationinsights.dependencycollector","microsoft.applicationinsights.perfcountercollector","microsoft.applicationinsights.windowsserver.telemetrychannel","microsoft.aspnet.cryptography.internal","microsoft.aspnet.dataprotection","microsoft.aspnet.dataprotection.abstractions","microsoft.aspnet.dataprotection.extensions","microsoft.aspnet.identity.core","microsoft.aspnet.identity.entityframework","microsoft.aspnet.webapi.client","microsoft.aspnetcore","microsoft.aspnetcore.antiforgery","microsoft.aspnetcore.authentication","microsoft.aspnetcore.authentication.cookies","microsoft.aspnetcore.authentication.facebook","microsoft.aspnetcore.authentication.google","microsoft.aspnetcore.authentication.jwtbearer","microsoft.aspnetcore.authentication.microsoftaccount","microsoft.aspnetcore.authentication.oauth","microsoft.aspnetcore.authentication.openidconnect","microsoft.aspnetcore.authentication.twitter","microsoft.aspnetcore.authorization","microsoft.aspnetcore.azureappservicesintegration","microsoft.aspnetcore.cookiepolicy","microsoft.aspnetcore.cors","microsoft.aspnetcore.cryptography.internal","microsoft.aspnetcore.cryptography.keyderivation","microsoft.aspnetcore.dataprotection","microsoft.aspnetcore.dataprotection.abstractions","microsoft.aspnetcore.dataprotection.azurestorage","microsoft.aspnetcore.dataprotection.extensions","microsoft.aspnetcore.dataprotection.systemweb","microsoft.aspnetcore.diagnostics","microsoft.aspnetcore.diagnostics.abstractions","microsoft.aspnetcore.diagnostics.entityframeworkcore","microsoft.aspnetcore.hosting","microsoft.aspnetcore.hosting.abstractions","microsoft.aspnetcore.hosting.server.abstractions","microsoft.aspnetcore.hosting.windowsservices","microsoft.aspnetcore.html.abstractions","microsoft.aspnetcore.http","microsoft.aspnetcore.http.abstractions","microsoft.aspnetcore.http.extensions","microsoft.aspnetcore.http.features","microsoft.aspnetcore.httpoverrides","microsoft.aspnetcore.identity","microsoft.aspnetcore.identity.entityframeworkcore","microsoft.aspnetcore.jsonpatch","microsoft.aspnetcore.localization","microsoft.aspnetcore.localization.routing","microsoft.aspnetcore.middlewareanalysis","microsoft.aspnetcore.mvc","microsoft.aspnetcore.mvc.abstractions","microsoft.aspnetcore.mvc.apiexplorer","microsoft.aspnetcore.mvc.core","microsoft.aspnetcore.mvc.cors","microsoft.aspnetcore.mvc.dataannotations","microsoft.aspnetcore.mvc.formatters.json","microsoft.aspnetcore.mvc.formatters.xml","microsoft.aspnetcore.mvc.localization","microsoft.aspnetcore.mvc.razor","microsoft.aspnetcore.mvc.razor.host","microsoft.aspnetcore.mvc.razor.viewcompilation","microsoft.aspnetcore.mvc.razor.viewcompilation.design","microsoft.aspnetcore.mvc.razor.viewcompilation.tools","microsoft.aspnetcore.mvc.taghelpers","microsoft.aspnetcore.mvc.viewfeatures","microsoft.aspnetcore.mvc.webapicompatshim","microsoft.aspnetcore.owin","microsoft.aspnetcore.razor","microsoft.aspnetcore.razor.design","microsoft.aspnetcore.razor.runtime","microsoft.aspnetcore.razor.tools","microsoft.aspnetcore.responsecaching","microsoft.aspnetcore.responsecaching.abstractions","microsoft.aspnetcore.responsecompression","microsoft.aspnetcore.rewrite","microsoft.aspnetcore.routing","microsoft.aspnetcore.routing.abstractions","microsoft.aspnetcore.server.iisintegration","microsoft.aspnetcore.server.iisintegration.tools","microsoft.aspnetcore.server.kestrel","microsoft.aspnetcore.server.kestrel.https","microsoft.aspnetcore.session","microsoft.aspnetcore.staticfiles","microsoft.aspnetcore.websockets","microsoft.aspnetcore.webutilities","microsoft.azure.keyvault","microsoft.azure.keyvault.core","microsoft.azure.keyvault.webkey","microsoft.bcl","microsoft.bcl.build","microsoft.build","microsoft.build.framework","microsoft.build.runtime","microsoft.build.tasks.core","microsoft.build.utilities.core","microsoft.codeanalysis.analyzers","microsoft.codeanalysis.common","microsoft.codeanalysis.csharp","microsoft.codeanalysis.csharp.workspaces","microsoft.codeanalysis.visualbasic","microsoft.codeanalysis.workspaces.common","microsoft.codecoverage","microsoft.composition","microsoft.csharp","microsoft.data.edm","microsoft.data.odata","microsoft.data.services.client","microsoft.data.sqlite","microsoft.diasymreader.native","microsoft.dotnet.cli.utils","microsoft.dotnet.compiler.common","microsoft.dotnet.files","microsoft.dotnet.internalabstractions","microsoft.dotnet.platformabstractions","microsoft.dotnet.projectmodel","microsoft.dotnet.projectmodel.loader","microsoft.dotnet.projectmodel.workspaces","microsoft.dotnet.watcher.core","microsoft.dotnet.watcher.tools","microsoft.entityframeworkcore","microsoft.entityframeworkcore.design","microsoft.entityframeworkcore.design.core","microsoft.entityframeworkcore.inmemory","microsoft.entityframeworkcore.relational","microsoft.entityframeworkcore.relational.design","microsoft.entityframeworkcore.sqlite","microsoft.entityframeworkcore.sqlite.design","microsoft.entityframeworkcore.sqlserver","microsoft.entityframeworkcore.sqlserver.design","microsoft.entityframeworkcore.tools","microsoft.entityframeworkcore.tools.dotnet","microsoft.extensions.caching.abstractions","microsoft.extensions.caching.memory","microsoft.extensions.caching.redis","microsoft.extensions.caching.sqlconfig.tools","microsoft.extensions.caching.sqlserver","microsoft.extensions.commandlineutils","microsoft.extensions.configuration","microsoft.extensions.configuration.abstractions","microsoft.extensions.configuration.azurekeyvault","microsoft.extensions.configuration.binder","microsoft.extensions.configuration.commandline","microsoft.extensions.configuration.environmentvariables","microsoft.extensions.configuration.fileextensions","microsoft.extensions.configuration.ini","microsoft.extensions.configuration.json","microsoft.extensions.configuration.usersecrets","microsoft.extensions.configuration.xml","microsoft.extensions.dependencyinjection","microsoft.extensions.dependencyinjection.abstractions","microsoft.extensions.dependencyinjection.specification.tests","microsoft.extensions.dependencymodel","microsoft.extensions.diagnosticadapter","microsoft.extensions.fileproviders.abstractions","microsoft.extensions.fileproviders.composite","microsoft.extensions.fileproviders.embedded","microsoft.extensions.fileproviders.physical","microsoft.extensions.filesystemglobbing","microsoft.extensions.globalization.cultureinfocache","microsoft.extensions.localization","microsoft.extensions.localization.abstractions","microsoft.extensions.logging","microsoft.extensions.logging.abstractions","microsoft.extensions.logging.azureappservices","microsoft.extensions.logging.console","microsoft.extensions.logging.debug","microsoft.extensions.logging.eventlog","microsoft.extensions.logging.eventsource","microsoft.extensions.logging.filter","microsoft.extensions.logging.tracesource","microsoft.extensions.objectpool","microsoft.extensions.options","microsoft.extensions.options.configurationextensions","microsoft.extensions.optionsmodel","microsoft.extensions.platformabstractions","microsoft.extensions.primitives","microsoft.extensions.secretmanager.tools","microsoft.extensions.webencoders","microsoft.identitymodel.clients.activedirectory","microsoft.identitymodel.logging","microsoft.identitymodel.protocols","microsoft.identitymodel.protocols.openidconnect","microsoft.identitymodel.tokens","microsoft.net.http","microsoft.net.http.headers","microsoft.net.http.server","microsoft.net.test.sdk","microsoft.netcore","microsoft.netcore.app","microsoft.netcore.dotnethost","microsoft.netcore.dotnethostpolicy","microsoft.netcore.dotnethostresolver","microsoft.netcore.jit","microsoft.netcore.platforms","microsoft.netcore.portable.compatibility","microsoft.netcore.runtime","microsoft.netcore.runtime.coreclr","microsoft.netcore.runtime.coreclr-arm","microsoft.netcore.runtime.coreclr-x64","microsoft.netcore.runtime.coreclr-x86","microsoft.netcore.runtime.native","microsoft.netcore.targets","microsoft.netcore.targets.dnxcore","microsoft.netcore.targets.netframework","microsoft.netcore.targets.universalwindowsplatform","microsoft.netcore.windows.apisets","microsoft.netcore.windows.apisets-x64","microsoft.netcore.windows.apisets-x86","microsoft.owin","microsoft.owin.security","microsoft.owin.security.cookies","microsoft.owin.security.interop","microsoft.rest.clientruntime","microsoft.rest.clientruntime.azure","microsoft.testplatform.objectmodel","microsoft.testplatform.testhost","microsoft.visualbasic","microsoft.visualstudio.web.browserlink","microsoft.visualstudio.web.browserlink.loader","microsoft.visualstudio.web.codegeneration","microsoft.visualstudio.web.codegeneration.core","microsoft.visualstudio.web.codegeneration.design","microsoft.visualstudio.web.codegeneration.entityframeworkcore","microsoft.visualstudio.web.codegeneration.templating","microsoft.visualstudio.web.codegeneration.tools","microsoft.visualstudio.web.codegeneration.utils","microsoft.visualstudio.web.codegenerators.mvc","microsoft.win32.primitives","microsoft.win32.registry","mstest.testadapter","mstest.testframework","netstandard.library","newtonsoft.json","nuget.common","nuget.configuration","nuget.dependencyresolver.core","nuget.frameworks","nuget.librarymodel","nuget.packaging","nuget.packaging.core","nuget.packaging.core.types","nuget.projectmodel","nuget.protocol.core.types","nuget.protocol.core.v3","nuget.repositories","nuget.runtimemodel","nuget.versioning","nuglify","owin","remotion.linq","runtime.any.system.collections","runtime.any.system.diagnostics.tools","runtime.any.system.diagnostics.tracing","runtime.any.system.globalization","runtime.any.system.globalization.calendars","runtime.any.system.io","runtime.any.system.reflection","runtime.any.system.reflection.extensions","runtime.any.system.reflection.primitives","runtime.any.system.resources.resourcemanager","runtime.any.system.runtime","runtime.any.system.runtime.handles","runtime.any.system.runtime.interopservices","runtime.any.system.text.encoding","runtime.any.system.text.encoding.extensions","runtime.any.system.threading.tasks","runtime.any.system.threading.timer","runtime.debian.8-x64.runtime.native.system.security.cryptography.openssl","runtime.fedora.23-x64.runtime.native.system.security.cryptography.openssl","runtime.fedora.24-x64.runtime.native.system.security.cryptography.openssl","runtime.native.system","runtime.native.system.data.sqlclient.sni","runtime.native.system.io.compression","runtime.native.system.net.http","runtime.native.system.net.security","runtime.native.system.security.cryptography","runtime.native.system.security.cryptography.apple","runtime.native.system.security.cryptography.openssl","runtime.opensuse.13.2-x64.runtime.native.system.security.cryptography.openssl","runtime.opensuse.42.1-x64.runtime.native.system.security.cryptography.openssl","runtime.osx.10.10-x64.runtime.native.system.security.cryptography.apple","runtime.osx.10.10-x64.runtime.native.system.security.cryptography.openssl","runtime.rhel.7-x64.runtime.native.system.security.cryptography.openssl","runtime.ubuntu.14.04-x64.runtime.native.system.security.cryptography.openssl","runtime.ubuntu.16.04-x64.runtime.native.system.security.cryptography.openssl","runtime.ubuntu.16.10-x64.runtime.native.system.security.cryptography.openssl","runtime.win.microsoft.win32.primitives","runtime.win.system.console","runtime.win.system.diagnostics.debug","runtime.win.system.io.filesystem","runtime.win.system.net.primitives","runtime.win.system.net.sockets","runtime.win.system.runtime.extensions","runtime.win7-x64.microsoft.netcore.dotnethost","runtime.win7-x64.microsoft.netcore.dotnethostpolicy","runtime.win7-x64.microsoft.netcore.dotnethostresolver","runtime.win7-x64.microsoft.netcore.jit","runtime.win7-x64.microsoft.netcore.runtime.coreclr","runtime.win7-x64.microsoft.netcore.windows.apisets","runtime.win7-x64.runtime.native.system.data.sqlclient.sni","runtime.win7-x64.runtime.native.system.io.compression","runtime.win7-x86.microsoft.netcore.dotnethost","runtime.win7-x86.microsoft.netcore.dotnethostpolicy","runtime.win7-x86.microsoft.netcore.dotnethostresolver","runtime.win7-x86.microsoft.netcore.jit","runtime.win7-x86.microsoft.netcore.runtime.coreclr","runtime.win7-x86.microsoft.netcore.windows.apisets","runtime.win7-x86.runtime.native.system.data.sqlclient.sni","runtime.win7-x86.runtime.native.system.io.compression","runtime.win7.system.private.uri","serilog","serilog.extensions.logging","serilog.sinks.file","serilog.sinks.periodicbatching","serilog.sinks.rollingfile","sqlite","sqlite.native","stackexchange.redis.strongname","system.appcontext","system.buffers","system.collections","system.collections.concurrent","system.collections.immutable","system.collections.nongeneric","system.collections.specialized","system.componentmodel","system.componentmodel.annotations","system.componentmodel.eventbasedasync","system.componentmodel.primitives","system.componentmodel.typeconverter","system.console","system.data.common","system.data.sqlclient","system.diagnostics.contracts","system.diagnostics.debug","system.diagnostics.diagnosticsource","system.diagnostics.fileversioninfo","system.diagnostics.process","system.diagnostics.stacktrace","system.diagnostics.textwritertracelistener","system.diagnostics.tools","system.diagnostics.tracesource","system.diagnostics.tracing","system.dynamic.runtime","system.globalization","system.globalization.calendars","system.globalization.extensions","system.identitymodel.tokens.jwt","system.interactive.async","system.io","system.io.compression","system.io.compression.clrcompression-arm","system.io.compression.clrcompression-x64","system.io.compression.clrcompression-x86","system.io.compression.zipfile","system.io.filesystem","system.io.filesystem.primitives","system.io.filesystem.watcher","system.io.memorymappedfiles","system.io.pipes","system.io.unmanagedmemorystream","system.linq","system.linq.expressions","system.linq.parallel","system.linq.queryable","system.net.http","system.net.nameresolution","system.net.networkinformation","system.net.primitives","system.net.requests","system.net.security","system.net.sockets","system.net.webheadercollection","system.net.websockets","system.numerics.vectors","system.objectmodel","system.private.datacontractserialization","system.private.networking","system.private.uri","system.reflection","system.reflection.dispatchproxy","system.reflection.emit","system.reflection.emit.ilgeneration","system.reflection.emit.lightweight","system.reflection.extensions","system.reflection.metadata","system.reflection.primitives","system.reflection.typeextensions","system.resources.reader","system.resources.resourcemanager","system.resources.writer","system.runtime","system.runtime.compilerservices.unsafe","system.runtime.extensions","system.runtime.handles","system.runtime.interopservices","system.runtime.interopservices.runtimeinformation","system.runtime.interopservices.windowsruntime","system.runtime.loader","system.runtime.numerics","system.runtime.serialization.json","system.runtime.serialization.primitives","system.runtime.serialization.xml","system.runtime.windowsruntime","system.security.claims","system.security.cryptography.algorithms","system.security.cryptography.cng","system.security.cryptography.csp","system.security.cryptography.encoding","system.security.cryptography.openssl","system.security.cryptography.primitives","system.security.cryptography.x509certificates","system.security.principal","system.security.principal.windows","system.spatial","system.text.encoding","system.text.encoding.codepages","system.text.encoding.extensions","system.text.encodings.web","system.text.regularexpressions","system.threading","system.threading.overlapped","system.threading.tasks","system.threading.tasks.dataflow","system.threading.tasks.extensions","system.threading.tasks.parallel","system.threading.thread","system.threading.threadpool","system.threading.timer","system.xml.readerwriter","system.xml.xdocument","system.xml.xmldocument","system.xml.xmlserializer","system.xml.xpath","system.xml.xpath.xdocument","system.xml.xpath.xmldocument","windowsazure.storage","messagepack","microsoft.aspnetcore.all","microsoft.aspnetcore.app","microsoft.aspnetcore.applicationinsights.hostingstartup","microsoft.aspnetcore.authentication.abstractions","microsoft.aspnetcore.authentication.core","microsoft.aspnetcore.authentication.wsfederation","microsoft.aspnetcore.authorization.policy","microsoft.aspnetcore.azureappservices.hostingstartup","microsoft.aspnetcore.connections.abstractions","microsoft.aspnetcore.dataprotection.azurekeyvault","microsoft.aspnetcore.hostfiltering","microsoft.aspnetcore.http.connections","microsoft.aspnetcore.http.connections.common","microsoft.aspnetcore.httpspolicy","microsoft.aspnetcore.identity.ui","microsoft.aspnetcore.mvc.analyzers","microsoft.aspnetcore.mvc.razor.extensions","microsoft.aspnetcore.mvc.razorpages","microsoft.aspnetcore.nodeservices","microsoft.aspnetcore.razor.language","microsoft.aspnetcore.server.httpsys","microsoft.aspnetcore.server.kestrel.core","microsoft.aspnetcore.server.kestrel.transport.abstractions","microsoft.aspnetcore.server.kestrel.transport.libuv","microsoft.aspnetcore.server.kestrel.transport.sockets","microsoft.aspnetcore.signalr","microsoft.aspnetcore.signalr.common","microsoft.aspnetcore.signalr.core","microsoft.aspnetcore.signalr.protocols.json","microsoft.aspnetcore.signalr.redis","microsoft.aspnetcore.spaservices","microsoft.aspnetcore.spaservices.extensions","microsoft.azure.services.appauthentication","microsoft.codeanalysis.razor","microsoft.data.sqlite.core","microsoft.entityframeworkcore.abstractions","microsoft.entityframeworkcore.analyzers","microsoft.entityframeworkcore.sqlite.core","microsoft.extensions.configuration.keyperfile","microsoft.extensions.hosting","microsoft.extensions.hosting.abstractions","microsoft.extensions.http","microsoft.extensions.identity.core","microsoft.extensions.identity.stores","microsoft.extensions.logging.configuration","microsoft.identitymodel.protocols.wsfederation","microsoft.identitymodel.tokens.saml","microsoft.identitymodel.xml","microsoft.netcore.dotnetapphost","microsoft.visualstudio.web.codegeneration.contracts","newtonsoft.json.bson","runtime.win-arm64.runtime.native.system.data.sqlclient.sni","runtime.win-x64.runtime.native.system.data.sqlclient.sni","runtime.win-x86.runtime.native.system.data.sqlclient.sni","sqlitepclraw.bundle_green","sqlitepclraw.core","sqlitepclraw.lib.e_sqlite3.linux","sqlitepclraw.lib.e_sqlite3.osx","sqlitepclraw.lib.e_sqlite3.v110_xp","sqlitepclraw.provider.e_sqlite3.netstandard11","system.composition","system.composition.attributedmodel","system.composition.convention","system.composition.hosting","system.composition.runtime","system.composition.typedparts","system.io.pipelines","system.memory","system.net.websockets.websocketprotocol","system.runtime.serialization.formatters","system.security.accesscontrol","system.security.cryptography.pkcs","system.security.cryptography.xml","system.security.permissions","system.threading.channels","system.valuetuple"] \ No newline at end of file diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 4bfb3ec8b..0dd289326 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -4,12 +4,15 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.IO; +using System.Linq; using System.Threading.Tasks; using Autofac; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json; using NuGet.Jobs; using NuGet.Jobs.Configuration; using NuGet.Jobs.Validation; @@ -25,12 +28,14 @@ namespace NuGet.Services.Revalidate public class Job : JsonConfigurationJob { + private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet"; private const string InitializeArgumentName = "Initialize"; private const string VerifyInitializationArgumentName = "VerifyInitialization"; private const string JobConfigurationSectionName = "RevalidateJob"; private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5); + private string _preinstalledSetPath; private bool _initialize; private bool _verifyInitialization; @@ -38,6 +43,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); + var preinstalledPackagesNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var path in config.PreinstalledPaths) + { + var expandedPath = Environment.ExpandEnvironmentVariables(path); + var packagesInPath = Directory.GetDirectories(expandedPath) + .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant()) + .Where(d => !d.StartsWith(".")); + + preinstalledPackagesNames.UnionWith(packagesInPath); + } + + File.WriteAllText(_preinstalledSetPath, JsonConvert.SerializeObject(preinstalledPackagesNames)); + + Logger.LogInformation("Rebuilt the preinstalled package set. Found {PreinstalledPackages} package ids", preinstalledPackagesNames.Count); + } + else if (_initialize || _verifyInitialization) { var initializer = scope.ServiceProvider.GetRequiredService(); @@ -97,6 +124,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddSingleton(provider => provider.GetRequiredService>().Value); services.AddSingleton(provider => provider.GetRequiredService>().Value.Initialization); services.AddSingleton(provider => provider.GetRequiredService>().Value.Health); + services.AddSingleton(provider => provider.GetRequiredService>().Value.AppInsights); services.AddSingleton(provider => provider.GetRequiredService>().Value.Queue); services.AddScoped(provider => @@ -112,13 +140,14 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddTransient(); services.AddTransient(); - services.AddTransient(); + services.AddTransient(); // Initialization services.AddTransient(); services.AddTransient(); // Revalidation + services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 3e5e4f4b9..15bd808c4 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -36,6 +36,7 @@ + @@ -44,6 +45,7 @@ + @@ -52,7 +54,9 @@ + + @@ -89,6 +93,9 @@ + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} diff --git a/src/NuGet.Services.Revalidate/Program.cs b/src/NuGet.Services.Revalidate/Program.cs index 9ad233ad7..d6cff1a8f 100644 --- a/src/NuGet.Services.Revalidate/Program.cs +++ b/src/NuGet.Services.Revalidate/Program.cs @@ -10,7 +10,7 @@ class Program static void Main(string[] args) { var job = new Job(); - JobRunner.Run(job, args).GetAwaiter().GetResult(); + JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); } } } diff --git a/src/NuGet.Services.Revalidate/Services/GalleryService.cs b/src/NuGet.Services.Revalidate/Services/GalleryService.cs new file mode 100644 index 000000000..c7cd52329 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/GalleryService.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using System.Web; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace NuGet.Services.Revalidate +{ + public class GalleryService : IGalleryService + { + private static readonly string GalleryEventsQuery = HttpUtility.UrlPathEncode( + "customMetrics | " + + "where name == \"PackagePush\" or name == \"PackageUnlisted\" or name == \"PackageListed\" | " + + "summarize sum(value)"); + + private readonly HttpClient _httpClient; + private readonly ApplicationInsightsConfiguration _appInsightsConfig; + private readonly ILogger _logger; + + public GalleryService( + HttpClient httpClient, + ApplicationInsightsConfiguration appInsightsConfig, + ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _appInsightsConfig = appInsightsConfig ?? throw new ArgumentNullException(nameof(appInsightsConfig)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CountEventsInPastHourAsync() + { + try + { + using (var request = new HttpRequestMessage()) + { + request.RequestUri = new Uri($"https://api.applicationinsights.io/v1/apps/{_appInsightsConfig.AppId}/query?timespan=PT1H&query={GalleryEventsQuery}"); + request.Method = HttpMethod.Get; + + request.Headers.Add("x-api-key", _appInsightsConfig.ApiKey); + + using (var response = await _httpClient.SendAsync(request)) + { + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var data = JsonConvert.DeserializeObject(json); + + if (data?.Tables?.Length != 1 || + data.Tables[0]?.Rows?.Length != 1 || + data.Tables[0].Rows[0]?.Length != 1) + { + throw new InvalidOperationException("Malformed response content"); + } + + // Get the first row's first column's value. + return data.Tables[0].Rows[0][0]; + } + } + } + catch (Exception e) + { + _logger.LogError(0, e, "Exception thrown when getting the Gallery's package event rate."); + + throw new InvalidOperationException("Exception thrown when getting the Gallery's package event rate.", e); + } + } + + private class QueryResult + { + public QueryTable[] Tables { get; set; } + } + + private class QueryTable + { + public int[][] Rows { get; set; } + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/IGalleryService.cs b/src/NuGet.Services.Revalidate/Services/IGalleryService.cs new file mode 100644 index 000000000..035398d97 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IGalleryService.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + public interface IGalleryService + { + /// + /// Count the number of gallery events (package pushes, listing, and unlisting) in the past hour. + /// + /// The number of gallery events in the past hour. + Task CountEventsInPastHourAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs index 4041832f7..c0afcfba5 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs @@ -11,17 +11,20 @@ public class RevalidationThrottler : IRevalidationThrottler { private readonly IRevalidationJobStateService _jobState; private readonly IPackageRevalidationStateService _packageState; + private readonly IGalleryService _gallery; private readonly RevalidationConfiguration _config; private readonly ILogger _logger; public RevalidationThrottler( IRevalidationJobStateService jobState, IPackageRevalidationStateService packageState, + IGalleryService gallery, RevalidationConfiguration config, ILogger logger) { _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); + _gallery = gallery ?? throw new ArgumentNullException(nameof(gallery)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -29,12 +32,31 @@ public RevalidationThrottler( public async Task IsThrottledAsync() { var desiredRate = await _jobState.GetDesiredPackageEventRateAsync(); - var recentGalleryEvents = await CountGalleryEventsInPastHourAsync(); + var recentGalleryEvents = await _gallery.CountEventsInPastHourAsync(); var recentRevalidations = await _packageState.CountRevalidationsEnqueuedInPastHourAsync(); var revalidationQuota = desiredRate - recentRevalidations - recentGalleryEvents; - return (revalidationQuota <= 0); + if (revalidationQuota <= 0) + { + _logger.LogInformation( + "Throttling revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}", + desiredRate, + recentGalleryEvents, + recentRevalidations); + + return true; + } + else + { + _logger.LogInformation( + "Allowing revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}", + desiredRate, + recentGalleryEvents, + recentRevalidations); + + return false; + } } public async Task DelayUntilNextRevalidationAsync() @@ -55,22 +77,5 @@ public async Task DelayUntilRevalidationRetryAsync() await Task.Delay(_config.RetryLaterSleep); } - - private Task CountGalleryEventsInPastHourAsync() - { - // TODO: Count the number of package pushes, lists, and unlists. - // Run this AI query: - // - // customMetrics | where name == "PackagePush" or name == "PackageUnlisted" or name == "PackageListed" | summarize sum(value) - // - // Using this HTTP request: - // - // GET /v1/apps/46f13c7d-635f-42c3-8120-593edeaad426/query?timespan=P1D&query=customMetrics%20%7C%20where%20name%20%3D%3D%20%22PackagePush%22%20or%20name%20%3D%3D%20%22PackageUnlisted%22%20or%20name%20%3D%3D%20%22PackageListed%22%20%7C%20summarize%20sum(value)%20 HTTP/1.1 - // Host: api.applicationinsights.io - // x-api-key: my-super-secret-api-key - // - // See: https://dev.applicationinsights.io/quickstart - return Task.FromResult(0); - } } } diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index 60d0d52aa..2612db0dc 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -3,11 +3,11 @@ "Initialization": { "PreinstalledPaths": [ "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools" + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled - "SleepDurationBetweenBatches": "00:00:01" + "SleepDurationBetweenBatches": "00:00:30" }, "Health": { @@ -19,17 +19,17 @@ "MinPackageEventRate": 120, "MaxPackageEventRate": 500, + "AppInsights": { + "AppId": "46f13c7d-635f-42c3-8120-593edeaad426", + "ApiKey": "$$Dev-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$" + }, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" } }, - "Storage": { - "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", - "Container": "revalidations" - }, - "GalleryDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, @@ -44,6 +44,8 @@ "TopicPath": "validation" }, + "PackageDownloadTimeout": "00:10:00", + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index 0c56b4956..e721eb379 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -3,8 +3,8 @@ "Initialization": { "PreinstalledPaths": [ "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools" + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" @@ -19,6 +19,11 @@ "MinPackageEventRate": 120, "MaxPackageEventRate": 500, + "AppInsights": { + "AppId": "718e0c81-9132-4bf2-b24b-aa625dafd800", + "ApiKey": "$$Int-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$" + }, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -39,6 +44,8 @@ "TopicPath": "validation" }, + "PackageDownloadTimeout": "00:10:00", + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index 354c1d7b8..7a97b2039 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -3,8 +3,8 @@ "Initialization": { "PreinstalledPaths": [ "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools" + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" @@ -19,6 +19,11 @@ "MinPackageEventRate": 120, "MaxPackageEventRate": 500, + "AppInsights": { + "AppId": "338f6804-b1a9-4fe3-bba7-c93064e7ae7b", + "ApiKey": "$$Prod-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$" + }, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -39,6 +44,8 @@ "TopicPath": "validation" }, + "PackageDownloadTimeout": "00:10:00", + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", diff --git a/src/NuGet.Services.Revalidate/readme.md b/src/NuGet.Services.Revalidate/readme.md index c9a28c051..99f97a821 100644 --- a/src/NuGet.Services.Revalidate/readme.md +++ b/src/NuGet.Services.Revalidate/readme.md @@ -3,10 +3,23 @@ This job enqueues packages revalidation as fast as possible without affecting the health of NuGet's ingestion pipeline. It does so in two phases: -1. Initialization phase - the job determines which packages should be revalidated. -2. Revalidation phase - packages are enqueued for revalidations +1. Build Preinstalled Packages phase - the job builds a JSON file of packages that +are installed by .NET SDK and Visual Studio +2. Initialization phase - the job determines which packages should be revalidated. +3. Revalidation phase - packages are enqueued for revalidations -The initialization phase MUST complete before the revalidation phase is started. +These phases MUST be completed in order. + +# The Build Preinstalled Packages phase + +This phase should run be at development time before the job is deployed: + +``` +NuGet.Services.Revalidate.exe ^ + -Configuration "C:\Path\to\job\Settings\dev.json" ^ + -RebuildPreinstalledSet "C:\Path\to\job\Initialization\PreinstalledPackages.json" ^ + -Once +``` # The Initialization Phase @@ -14,7 +27,7 @@ To initialize the job, run: ``` NuGet.Services.Revalidate.exe ^ - -Configuration "C:\Path\to\config.json" ^ + -Configuration "C:\Path\to\job\Settings\dev.json" ^ -Initialize -VerifyInitialization -Once @@ -36,6 +49,6 @@ To enqueue revalidations, run: ``` NuGet.Services.Revalidate.exe ^ - -Configuration "C:\Path\to\config.json" ^ + -Configuration "C:\Path\to\job\Settings\dev.json" ^ -Initialize ``` \ No newline at end of file diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs index ffc2bf8dd..626891557 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs @@ -246,6 +246,8 @@ private List FindPackageOwners(IValidationRequest request) return registration .Owners .Select(o => o.Username) + .ToList() + .OrderBy(u => u, StringComparer.InvariantCultureIgnoreCase) .ToList(); } } diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index c1b8a4c9c..0b0dd9617 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -61,6 +61,7 @@ private static void AddParsing(IServiceCollection serviceCollection) serviceCollection.AddTransient(); serviceCollection.AddTransient(); serviceCollection.AddTransient(); + serviceCollection.AddTransient(); serviceCollection.AddTransient(); } diff --git a/src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs b/src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs new file mode 100644 index 000000000..d44047c84 --- /dev/null +++ b/src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs @@ -0,0 +1,178 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; +using NuGet.Services.Incidents; +using NuGet.Services.Status; +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace StatusAggregator.Parse +{ + public class TrafficManagerEndpointStatusIncidentParser : EnvironmentPrefixIncidentParser + { + private const string DomainGroupName = "Domain"; + private const string TargetGroupName = "Target"; + private static string SubtitleRegEx = $"Traffic Manager for (?<{DomainGroupName}>.*) is reporting (?<{TargetGroupName}>.*) as not Online!"; + + private readonly ILogger _logger; + + public TrafficManagerEndpointStatusIncidentParser( + IEnumerable filters, + ILogger logger) + : base(SubtitleRegEx, filters, logger) + { + _logger = logger; + } + + protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) + { + affectedComponentPath = null; + + var domain = groups[DomainGroupName].Value; + var target = groups[TargetGroupName].Value; + var environment = groups[EnvironmentFilter.EnvironmentGroupName].Value; + _logger.LogInformation("Domain is {Domain}, target is {Target}, environment is {Environment}.", domain, target, environment); + + if (EnvironmentToDomainToTargetToPath.TryGetValue(environment, out var domainToTargetToPath) && + domainToTargetToPath.TryGetValue(domain, out var targetToPath) && + targetToPath.TryGetValue(target, out var path)) + { + affectedComponentPath = path; + } + + return affectedComponentPath != null; + } + + protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) + { + affectedComponentStatus = ComponentStatus.Down; + return true; + } + + private static readonly string GalleryUsncPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.GalleryName, + NuGetServiceComponentFactory.UsncInstanceName); + + private static readonly string GalleryUsscPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.GalleryName, + NuGetServiceComponentFactory.UsscInstanceName); + + private static readonly string RestoreV3GlobalPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.RestoreName, + NuGetServiceComponentFactory.V3ProtocolName, + NuGetServiceComponentFactory.GlobalRegionName); + + private static readonly string RestoreV3ChinaPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.RestoreName, + NuGetServiceComponentFactory.V3ProtocolName, + NuGetServiceComponentFactory.ChinaRegionName); + + private static readonly IDictionary> DevDomainToTargetToPath = + new Dictionary> + { + { + "devnugettest.trafficmanager.net", + new Dictionary + { + { + "nuget-dev-use2-gallery.cloudapp.net", + GalleryUsncPath + }, + + { + "nuget-dev-ussc-gallery.cloudapp.net", + GalleryUsncPath + } + } + }, + + { + "nugetapidev.trafficmanager.net", + new Dictionary + { + { + "az635243.vo.msecnd.net", + RestoreV3GlobalPath + }, + { + "nugetdevcnredirect.trafficmanager.net", + RestoreV3ChinaPath + } + } + } + }; + + private static readonly IDictionary> IntDomainToTargetToPath = + new Dictionary> + { + { + "nuget-int-test-failover.trafficmanager.net", + new Dictionary + { + { + "nuget-int-0-v2gallery.cloudapp.net", + GalleryUsncPath + }, + + { + "nuget-int-ussc-gallery.cloudapp.net", + GalleryUsncPath + } + } + } + }; + + private static readonly IDictionary> ProdDomainToTargetToPath = + new Dictionary> + { + { + "nuget-prod-v2gallery.trafficmanager.net", + new Dictionary + { + { + "nuget-prod-0-v2gallery.cloudapp.net", + GalleryUsncPath + }, + + { + "nuget-prod-ussc-gallery.cloudapp.net", + GalleryUsncPath + } + } + }, + + { + "nugetapiprod.trafficmanager.net", + new Dictionary + { + { + "az320820.vo.msecnd.net", + RestoreV3GlobalPath + }, + { + "nugetprodcnredirect.trafficmanager.net", + RestoreV3ChinaPath + } + } + } + }; + + private static readonly IDictionary>> EnvironmentToDomainToTargetToPath = + new Dictionary>> + { + { "dev", DevDomainToTargetToPath }, + { "test", DevDomainToTargetToPath }, + { "int", IntDomainToTargetToPath }, + { "prod", ProdDomainToTargetToPath } + }; + } +} diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 00eba2850..6908130c1 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -48,6 +48,7 @@ + diff --git a/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs index 9deb48c17..7127d9d14 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs @@ -74,8 +74,8 @@ public void FindsPreinstalledPackages() { _context.Mock(packageRegistrations: new[] { - new PackageRegistration { Key = 1, Id = "System.Linq" }, - new PackageRegistration { Key = 2, Id = "Newtonsoft.Json" } + new PackageRegistration { Key = 1, Id = "system.linq" }, + new PackageRegistration { Key = 2, Id = "newtonsoft.json" } }); var actual = _target.FindPreinstalledPackages(except: new HashSet()); @@ -89,8 +89,8 @@ public void SkipsPackagesInExceptSet() { _context.Mock(packageRegistrations: new[] { - new PackageRegistration { Key = 1, Id = "System.Linq" }, - new PackageRegistration { Key = 2, Id = "Newtonsoft.Json" } + new PackageRegistration { Key = 1, Id = "system.linq" }, + new PackageRegistration { Key = 2, Id = "newtonsoft.json" } }); var actual = _target.FindPreinstalledPackages(except: new HashSet { 1 }); @@ -104,7 +104,7 @@ public void SkipsPackagesWithNoRegistration() { _context.Mock(packageRegistrations: new[] { - new PackageRegistration { Key = 1, Id = "System.Linq" }, + new PackageRegistration { Key = 1, Id = "system.linq" }, }); var actual = _target.FindPreinstalledPackages(except: new HashSet()); diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs index 7b5da8254..86f6c930c 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs @@ -14,6 +14,7 @@ public class TheIsThrottledAsyncMethod { private readonly Mock _settings; private readonly Mock _state; + private readonly Mock _gallery; private readonly RevalidationConfiguration _config; private readonly IRevalidationThrottler _target; @@ -22,28 +23,35 @@ public TheIsThrottledAsyncMethod() { _settings = new Mock(); _state = new Mock(); + _gallery = new Mock(); _config = new RevalidationConfiguration(); _target = new RevalidationThrottler( _settings.Object, _state.Object, + _gallery.Object, _config, Mock.Of>()); } - [Fact] - public async Task ReturnsTrueIfRecentRevalidationsMoreThanDesiredRate() + [Theory] + [InlineData(100, 0)] + [InlineData(0, 100)] + [InlineData(40, 40)] + public async Task ReturnsTrueIfRecentRevalidationsMoreThanDesiredRate(int enqueuedRevalidations, int galleryEvents) { // Arrange _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(50); - _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(100); + _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(enqueuedRevalidations); + _gallery.Setup(g => g.CountEventsInPastHourAsync()).ReturnsAsync(galleryEvents); // Act & Assert Assert.True(await _target.IsThrottledAsync()); _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + _gallery.Verify(g => g.CountEventsInPastHourAsync(), Times.Once); } [Fact] @@ -52,12 +60,14 @@ public async Task ReturnsFalseIfRecentRevalidationsLessThanDesiredRate() // Arrange _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(100); _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(50); + _gallery.Setup(g => g.CountEventsInPastHourAsync()).ReturnsAsync(40); // Act & Assert Assert.False(await _target.IsThrottledAsync()); _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + _gallery.Verify(g => g.CountEventsInPastHourAsync(), Times.Once); } } } diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs index 978035e1c..0c3bdd2f0 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs @@ -251,9 +251,12 @@ public async Task EnqueuesScanAndSignIfPackageHasNoRepositorySignature() _request.NupkgUrl, _config.V3ServiceIndexUrl, It.Is>(l => - l.Count() == 2 && - l.Contains("Billy") && - l.Contains("Bob"))), + // Ensure that the owners are lexicographically ordered. + l.Count() == 4 && + l[0] == "Annie" && + l[1] == "Bob" && + l[2] == "zack" && + l[3] == "Zorro")), Times.Once); _validatorStateServiceMock @@ -441,8 +444,10 @@ public async Task WhenPackageFitsCriteriaAndIsNotRepositorySigned_DoesNotSkipSca { Owners = new List { - new User("Billy"), + new User("Zorro"), new User("Bob"), + new User("Annie"), + new User("zack") } };