From 6de44bb7264e88a04b6311997d28159b77a62891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Thu, 2 Aug 2018 17:07:18 -0700 Subject: [PATCH 1/4] [Revalidation] Improve initialization (#509) The revalidation job prioritizes packages that are installed by Visual Studio or .NET SDK. Before, the revalidation job required that Visual Studio and the .NET SDK were installed on the VM so that it could find these packages. This change removes that requirement by embedding a list of known preinstalled packages. Addresses https://github.com/NuGet/Engineering/issues/1545 --- .../Initialization/PackageFinder.cs | 16 +++++----- .../Initialization/PreinstalledPackages.json | 1 + src/NuGet.Services.Revalidate/Job.cs | 29 ++++++++++++++++++- .../NuGet.Services.Revalidate.csproj | 3 ++ src/NuGet.Services.Revalidate/Program.cs | 2 +- .../Settings/dev.json | 6 ++-- .../Settings/int.json | 4 +-- .../Settings/prod.json | 4 +-- src/NuGet.Services.Revalidate/readme.md | 23 +++++++++++---- .../Initializer/PackageFinderFacts.cs | 10 +++---- 10 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json diff --git a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs index 8768f2d12..b6cff5662 100644 --- a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs +++ b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs @@ -8,6 +8,7 @@ using System.Linq.Expressions; using System.Threading; using Microsoft.Extensions.Logging; +using Newtonsoft.Json; using NuGet.Versioning; using NuGetGallery; @@ -22,6 +23,8 @@ public class PackageFinder : IPackageFinder public const string DependencySetName = "Dependency"; public const string RemainingSetName = "Remaining"; + private const string PreinstalledPackagesResource = "NuGet.Services.Revalidate.Initialization.PreinstalledPackages.json"; + private static int BatchSize = 1000; private static string MicrosoftAccountName = "Microsoft"; @@ -46,16 +49,13 @@ public HashSet FindMicrosoftPackages() public HashSet FindPreinstalledPackages(HashSet except) { - var preinstalledPackagesNames = new HashSet(StringComparer.OrdinalIgnoreCase); + List preinstalledPackagesNames; + var assembly = typeof(PackageFinder).Assembly; - foreach (var path in _config.PreinstalledPaths) + using (var resource = assembly.GetManifestResourceStream(PreinstalledPackagesResource)) + using (var reader = new StreamReader(resource)) { - var expandedPath = Environment.ExpandEnvironmentVariables(path); - var packagesInPath = Directory.GetDirectories(expandedPath) - .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant()) - .Where(d => !d.StartsWith(".")); - - preinstalledPackagesNames.UnionWith(packagesInPath); + preinstalledPackagesNames = JsonConvert.DeserializeObject>(reader.ReadToEnd()); } var preinstalledPackages = FindRegistrationKeys(PreinstalledSetName, r => preinstalledPackagesNames.Contains(r.Id)); diff --git a/src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json b/src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json new file mode 100644 index 000000000..5a778201c --- /dev/null +++ b/src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json @@ -0,0 +1 @@ +["bundlerminifier.core","entityframework","libuv","microsoft.applicationinsights","microsoft.applicationinsights.agent.intercept","microsoft.applicationinsights.aspnetcore","microsoft.applicationinsights.dependencycollector","microsoft.applicationinsights.perfcountercollector","microsoft.applicationinsights.windowsserver.telemetrychannel","microsoft.aspnet.cryptography.internal","microsoft.aspnet.dataprotection","microsoft.aspnet.dataprotection.abstractions","microsoft.aspnet.dataprotection.extensions","microsoft.aspnet.identity.core","microsoft.aspnet.identity.entityframework","microsoft.aspnet.webapi.client","microsoft.aspnetcore","microsoft.aspnetcore.antiforgery","microsoft.aspnetcore.authentication","microsoft.aspnetcore.authentication.cookies","microsoft.aspnetcore.authentication.facebook","microsoft.aspnetcore.authentication.google","microsoft.aspnetcore.authentication.jwtbearer","microsoft.aspnetcore.authentication.microsoftaccount","microsoft.aspnetcore.authentication.oauth","microsoft.aspnetcore.authentication.openidconnect","microsoft.aspnetcore.authentication.twitter","microsoft.aspnetcore.authorization","microsoft.aspnetcore.azureappservicesintegration","microsoft.aspnetcore.cookiepolicy","microsoft.aspnetcore.cors","microsoft.aspnetcore.cryptography.internal","microsoft.aspnetcore.cryptography.keyderivation","microsoft.aspnetcore.dataprotection","microsoft.aspnetcore.dataprotection.abstractions","microsoft.aspnetcore.dataprotection.azurestorage","microsoft.aspnetcore.dataprotection.extensions","microsoft.aspnetcore.dataprotection.systemweb","microsoft.aspnetcore.diagnostics","microsoft.aspnetcore.diagnostics.abstractions","microsoft.aspnetcore.diagnostics.entityframeworkcore","microsoft.aspnetcore.hosting","microsoft.aspnetcore.hosting.abstractions","microsoft.aspnetcore.hosting.server.abstractions","microsoft.aspnetcore.hosting.windowsservices","microsoft.aspnetcore.html.abstractions","microsoft.aspnetcore.http","microsoft.aspnetcore.http.abstractions","microsoft.aspnetcore.http.extensions","microsoft.aspnetcore.http.features","microsoft.aspnetcore.httpoverrides","microsoft.aspnetcore.identity","microsoft.aspnetcore.identity.entityframeworkcore","microsoft.aspnetcore.jsonpatch","microsoft.aspnetcore.localization","microsoft.aspnetcore.localization.routing","microsoft.aspnetcore.middlewareanalysis","microsoft.aspnetcore.mvc","microsoft.aspnetcore.mvc.abstractions","microsoft.aspnetcore.mvc.apiexplorer","microsoft.aspnetcore.mvc.core","microsoft.aspnetcore.mvc.cors","microsoft.aspnetcore.mvc.dataannotations","microsoft.aspnetcore.mvc.formatters.json","microsoft.aspnetcore.mvc.formatters.xml","microsoft.aspnetcore.mvc.localization","microsoft.aspnetcore.mvc.razor","microsoft.aspnetcore.mvc.razor.host","microsoft.aspnetcore.mvc.razor.viewcompilation","microsoft.aspnetcore.mvc.razor.viewcompilation.design","microsoft.aspnetcore.mvc.razor.viewcompilation.tools","microsoft.aspnetcore.mvc.taghelpers","microsoft.aspnetcore.mvc.viewfeatures","microsoft.aspnetcore.mvc.webapicompatshim","microsoft.aspnetcore.owin","microsoft.aspnetcore.razor","microsoft.aspnetcore.razor.design","microsoft.aspnetcore.razor.runtime","microsoft.aspnetcore.razor.tools","microsoft.aspnetcore.responsecaching","microsoft.aspnetcore.responsecaching.abstractions","microsoft.aspnetcore.responsecompression","microsoft.aspnetcore.rewrite","microsoft.aspnetcore.routing","microsoft.aspnetcore.routing.abstractions","microsoft.aspnetcore.server.iisintegration","microsoft.aspnetcore.server.iisintegration.tools","microsoft.aspnetcore.server.kestrel","microsoft.aspnetcore.server.kestrel.https","microsoft.aspnetcore.session","microsoft.aspnetcore.staticfiles","microsoft.aspnetcore.websockets","microsoft.aspnetcore.webutilities","microsoft.azure.keyvault","microsoft.azure.keyvault.core","microsoft.azure.keyvault.webkey","microsoft.bcl","microsoft.bcl.build","microsoft.build","microsoft.build.framework","microsoft.build.runtime","microsoft.build.tasks.core","microsoft.build.utilities.core","microsoft.codeanalysis.analyzers","microsoft.codeanalysis.common","microsoft.codeanalysis.csharp","microsoft.codeanalysis.csharp.workspaces","microsoft.codeanalysis.visualbasic","microsoft.codeanalysis.workspaces.common","microsoft.codecoverage","microsoft.composition","microsoft.csharp","microsoft.data.edm","microsoft.data.odata","microsoft.data.services.client","microsoft.data.sqlite","microsoft.diasymreader.native","microsoft.dotnet.cli.utils","microsoft.dotnet.compiler.common","microsoft.dotnet.files","microsoft.dotnet.internalabstractions","microsoft.dotnet.platformabstractions","microsoft.dotnet.projectmodel","microsoft.dotnet.projectmodel.loader","microsoft.dotnet.projectmodel.workspaces","microsoft.dotnet.watcher.core","microsoft.dotnet.watcher.tools","microsoft.entityframeworkcore","microsoft.entityframeworkcore.design","microsoft.entityframeworkcore.design.core","microsoft.entityframeworkcore.inmemory","microsoft.entityframeworkcore.relational","microsoft.entityframeworkcore.relational.design","microsoft.entityframeworkcore.sqlite","microsoft.entityframeworkcore.sqlite.design","microsoft.entityframeworkcore.sqlserver","microsoft.entityframeworkcore.sqlserver.design","microsoft.entityframeworkcore.tools","microsoft.entityframeworkcore.tools.dotnet","microsoft.extensions.caching.abstractions","microsoft.extensions.caching.memory","microsoft.extensions.caching.redis","microsoft.extensions.caching.sqlconfig.tools","microsoft.extensions.caching.sqlserver","microsoft.extensions.commandlineutils","microsoft.extensions.configuration","microsoft.extensions.configuration.abstractions","microsoft.extensions.configuration.azurekeyvault","microsoft.extensions.configuration.binder","microsoft.extensions.configuration.commandline","microsoft.extensions.configuration.environmentvariables","microsoft.extensions.configuration.fileextensions","microsoft.extensions.configuration.ini","microsoft.extensions.configuration.json","microsoft.extensions.configuration.usersecrets","microsoft.extensions.configuration.xml","microsoft.extensions.dependencyinjection","microsoft.extensions.dependencyinjection.abstractions","microsoft.extensions.dependencyinjection.specification.tests","microsoft.extensions.dependencymodel","microsoft.extensions.diagnosticadapter","microsoft.extensions.fileproviders.abstractions","microsoft.extensions.fileproviders.composite","microsoft.extensions.fileproviders.embedded","microsoft.extensions.fileproviders.physical","microsoft.extensions.filesystemglobbing","microsoft.extensions.globalization.cultureinfocache","microsoft.extensions.localization","microsoft.extensions.localization.abstractions","microsoft.extensions.logging","microsoft.extensions.logging.abstractions","microsoft.extensions.logging.azureappservices","microsoft.extensions.logging.console","microsoft.extensions.logging.debug","microsoft.extensions.logging.eventlog","microsoft.extensions.logging.eventsource","microsoft.extensions.logging.filter","microsoft.extensions.logging.tracesource","microsoft.extensions.objectpool","microsoft.extensions.options","microsoft.extensions.options.configurationextensions","microsoft.extensions.optionsmodel","microsoft.extensions.platformabstractions","microsoft.extensions.primitives","microsoft.extensions.secretmanager.tools","microsoft.extensions.webencoders","microsoft.identitymodel.clients.activedirectory","microsoft.identitymodel.logging","microsoft.identitymodel.protocols","microsoft.identitymodel.protocols.openidconnect","microsoft.identitymodel.tokens","microsoft.net.http","microsoft.net.http.headers","microsoft.net.http.server","microsoft.net.test.sdk","microsoft.netcore","microsoft.netcore.app","microsoft.netcore.dotnethost","microsoft.netcore.dotnethostpolicy","microsoft.netcore.dotnethostresolver","microsoft.netcore.jit","microsoft.netcore.platforms","microsoft.netcore.portable.compatibility","microsoft.netcore.runtime","microsoft.netcore.runtime.coreclr","microsoft.netcore.runtime.coreclr-arm","microsoft.netcore.runtime.coreclr-x64","microsoft.netcore.runtime.coreclr-x86","microsoft.netcore.runtime.native","microsoft.netcore.targets","microsoft.netcore.targets.dnxcore","microsoft.netcore.targets.netframework","microsoft.netcore.targets.universalwindowsplatform","microsoft.netcore.windows.apisets","microsoft.netcore.windows.apisets-x64","microsoft.netcore.windows.apisets-x86","microsoft.owin","microsoft.owin.security","microsoft.owin.security.cookies","microsoft.owin.security.interop","microsoft.rest.clientruntime","microsoft.rest.clientruntime.azure","microsoft.testplatform.objectmodel","microsoft.testplatform.testhost","microsoft.visualbasic","microsoft.visualstudio.web.browserlink","microsoft.visualstudio.web.browserlink.loader","microsoft.visualstudio.web.codegeneration","microsoft.visualstudio.web.codegeneration.core","microsoft.visualstudio.web.codegeneration.design","microsoft.visualstudio.web.codegeneration.entityframeworkcore","microsoft.visualstudio.web.codegeneration.templating","microsoft.visualstudio.web.codegeneration.tools","microsoft.visualstudio.web.codegeneration.utils","microsoft.visualstudio.web.codegenerators.mvc","microsoft.win32.primitives","microsoft.win32.registry","mstest.testadapter","mstest.testframework","netstandard.library","newtonsoft.json","nuget.common","nuget.configuration","nuget.dependencyresolver.core","nuget.frameworks","nuget.librarymodel","nuget.packaging","nuget.packaging.core","nuget.packaging.core.types","nuget.projectmodel","nuget.protocol.core.types","nuget.protocol.core.v3","nuget.repositories","nuget.runtimemodel","nuget.versioning","nuglify","owin","remotion.linq","runtime.any.system.collections","runtime.any.system.diagnostics.tools","runtime.any.system.diagnostics.tracing","runtime.any.system.globalization","runtime.any.system.globalization.calendars","runtime.any.system.io","runtime.any.system.reflection","runtime.any.system.reflection.extensions","runtime.any.system.reflection.primitives","runtime.any.system.resources.resourcemanager","runtime.any.system.runtime","runtime.any.system.runtime.handles","runtime.any.system.runtime.interopservices","runtime.any.system.text.encoding","runtime.any.system.text.encoding.extensions","runtime.any.system.threading.tasks","runtime.any.system.threading.timer","runtime.debian.8-x64.runtime.native.system.security.cryptography.openssl","runtime.fedora.23-x64.runtime.native.system.security.cryptography.openssl","runtime.fedora.24-x64.runtime.native.system.security.cryptography.openssl","runtime.native.system","runtime.native.system.data.sqlclient.sni","runtime.native.system.io.compression","runtime.native.system.net.http","runtime.native.system.net.security","runtime.native.system.security.cryptography","runtime.native.system.security.cryptography.apple","runtime.native.system.security.cryptography.openssl","runtime.opensuse.13.2-x64.runtime.native.system.security.cryptography.openssl","runtime.opensuse.42.1-x64.runtime.native.system.security.cryptography.openssl","runtime.osx.10.10-x64.runtime.native.system.security.cryptography.apple","runtime.osx.10.10-x64.runtime.native.system.security.cryptography.openssl","runtime.rhel.7-x64.runtime.native.system.security.cryptography.openssl","runtime.ubuntu.14.04-x64.runtime.native.system.security.cryptography.openssl","runtime.ubuntu.16.04-x64.runtime.native.system.security.cryptography.openssl","runtime.ubuntu.16.10-x64.runtime.native.system.security.cryptography.openssl","runtime.win.microsoft.win32.primitives","runtime.win.system.console","runtime.win.system.diagnostics.debug","runtime.win.system.io.filesystem","runtime.win.system.net.primitives","runtime.win.system.net.sockets","runtime.win.system.runtime.extensions","runtime.win7-x64.microsoft.netcore.dotnethost","runtime.win7-x64.microsoft.netcore.dotnethostpolicy","runtime.win7-x64.microsoft.netcore.dotnethostresolver","runtime.win7-x64.microsoft.netcore.jit","runtime.win7-x64.microsoft.netcore.runtime.coreclr","runtime.win7-x64.microsoft.netcore.windows.apisets","runtime.win7-x64.runtime.native.system.data.sqlclient.sni","runtime.win7-x64.runtime.native.system.io.compression","runtime.win7-x86.microsoft.netcore.dotnethost","runtime.win7-x86.microsoft.netcore.dotnethostpolicy","runtime.win7-x86.microsoft.netcore.dotnethostresolver","runtime.win7-x86.microsoft.netcore.jit","runtime.win7-x86.microsoft.netcore.runtime.coreclr","runtime.win7-x86.microsoft.netcore.windows.apisets","runtime.win7-x86.runtime.native.system.data.sqlclient.sni","runtime.win7-x86.runtime.native.system.io.compression","runtime.win7.system.private.uri","serilog","serilog.extensions.logging","serilog.sinks.file","serilog.sinks.periodicbatching","serilog.sinks.rollingfile","sqlite","sqlite.native","stackexchange.redis.strongname","system.appcontext","system.buffers","system.collections","system.collections.concurrent","system.collections.immutable","system.collections.nongeneric","system.collections.specialized","system.componentmodel","system.componentmodel.annotations","system.componentmodel.eventbasedasync","system.componentmodel.primitives","system.componentmodel.typeconverter","system.console","system.data.common","system.data.sqlclient","system.diagnostics.contracts","system.diagnostics.debug","system.diagnostics.diagnosticsource","system.diagnostics.fileversioninfo","system.diagnostics.process","system.diagnostics.stacktrace","system.diagnostics.textwritertracelistener","system.diagnostics.tools","system.diagnostics.tracesource","system.diagnostics.tracing","system.dynamic.runtime","system.globalization","system.globalization.calendars","system.globalization.extensions","system.identitymodel.tokens.jwt","system.interactive.async","system.io","system.io.compression","system.io.compression.clrcompression-arm","system.io.compression.clrcompression-x64","system.io.compression.clrcompression-x86","system.io.compression.zipfile","system.io.filesystem","system.io.filesystem.primitives","system.io.filesystem.watcher","system.io.memorymappedfiles","system.io.pipes","system.io.unmanagedmemorystream","system.linq","system.linq.expressions","system.linq.parallel","system.linq.queryable","system.net.http","system.net.nameresolution","system.net.networkinformation","system.net.primitives","system.net.requests","system.net.security","system.net.sockets","system.net.webheadercollection","system.net.websockets","system.numerics.vectors","system.objectmodel","system.private.datacontractserialization","system.private.networking","system.private.uri","system.reflection","system.reflection.dispatchproxy","system.reflection.emit","system.reflection.emit.ilgeneration","system.reflection.emit.lightweight","system.reflection.extensions","system.reflection.metadata","system.reflection.primitives","system.reflection.typeextensions","system.resources.reader","system.resources.resourcemanager","system.resources.writer","system.runtime","system.runtime.compilerservices.unsafe","system.runtime.extensions","system.runtime.handles","system.runtime.interopservices","system.runtime.interopservices.runtimeinformation","system.runtime.interopservices.windowsruntime","system.runtime.loader","system.runtime.numerics","system.runtime.serialization.json","system.runtime.serialization.primitives","system.runtime.serialization.xml","system.runtime.windowsruntime","system.security.claims","system.security.cryptography.algorithms","system.security.cryptography.cng","system.security.cryptography.csp","system.security.cryptography.encoding","system.security.cryptography.openssl","system.security.cryptography.primitives","system.security.cryptography.x509certificates","system.security.principal","system.security.principal.windows","system.spatial","system.text.encoding","system.text.encoding.codepages","system.text.encoding.extensions","system.text.encodings.web","system.text.regularexpressions","system.threading","system.threading.overlapped","system.threading.tasks","system.threading.tasks.dataflow","system.threading.tasks.extensions","system.threading.tasks.parallel","system.threading.thread","system.threading.threadpool","system.threading.timer","system.xml.readerwriter","system.xml.xdocument","system.xml.xmldocument","system.xml.xmlserializer","system.xml.xpath","system.xml.xpath.xdocument","system.xml.xpath.xmldocument","windowsazure.storage","messagepack","microsoft.aspnetcore.all","microsoft.aspnetcore.app","microsoft.aspnetcore.applicationinsights.hostingstartup","microsoft.aspnetcore.authentication.abstractions","microsoft.aspnetcore.authentication.core","microsoft.aspnetcore.authentication.wsfederation","microsoft.aspnetcore.authorization.policy","microsoft.aspnetcore.azureappservices.hostingstartup","microsoft.aspnetcore.connections.abstractions","microsoft.aspnetcore.dataprotection.azurekeyvault","microsoft.aspnetcore.hostfiltering","microsoft.aspnetcore.http.connections","microsoft.aspnetcore.http.connections.common","microsoft.aspnetcore.httpspolicy","microsoft.aspnetcore.identity.ui","microsoft.aspnetcore.mvc.analyzers","microsoft.aspnetcore.mvc.razor.extensions","microsoft.aspnetcore.mvc.razorpages","microsoft.aspnetcore.nodeservices","microsoft.aspnetcore.razor.language","microsoft.aspnetcore.server.httpsys","microsoft.aspnetcore.server.kestrel.core","microsoft.aspnetcore.server.kestrel.transport.abstractions","microsoft.aspnetcore.server.kestrel.transport.libuv","microsoft.aspnetcore.server.kestrel.transport.sockets","microsoft.aspnetcore.signalr","microsoft.aspnetcore.signalr.common","microsoft.aspnetcore.signalr.core","microsoft.aspnetcore.signalr.protocols.json","microsoft.aspnetcore.signalr.redis","microsoft.aspnetcore.spaservices","microsoft.aspnetcore.spaservices.extensions","microsoft.azure.services.appauthentication","microsoft.codeanalysis.razor","microsoft.data.sqlite.core","microsoft.entityframeworkcore.abstractions","microsoft.entityframeworkcore.analyzers","microsoft.entityframeworkcore.sqlite.core","microsoft.extensions.configuration.keyperfile","microsoft.extensions.hosting","microsoft.extensions.hosting.abstractions","microsoft.extensions.http","microsoft.extensions.identity.core","microsoft.extensions.identity.stores","microsoft.extensions.logging.configuration","microsoft.identitymodel.protocols.wsfederation","microsoft.identitymodel.tokens.saml","microsoft.identitymodel.xml","microsoft.netcore.dotnetapphost","microsoft.visualstudio.web.codegeneration.contracts","newtonsoft.json.bson","runtime.win-arm64.runtime.native.system.data.sqlclient.sni","runtime.win-x64.runtime.native.system.data.sqlclient.sni","runtime.win-x86.runtime.native.system.data.sqlclient.sni","sqlitepclraw.bundle_green","sqlitepclraw.core","sqlitepclraw.lib.e_sqlite3.linux","sqlitepclraw.lib.e_sqlite3.osx","sqlitepclraw.lib.e_sqlite3.v110_xp","sqlitepclraw.provider.e_sqlite3.netstandard11","system.composition","system.composition.attributedmodel","system.composition.convention","system.composition.hosting","system.composition.runtime","system.composition.typedparts","system.io.pipelines","system.memory","system.net.websockets.websocketprotocol","system.runtime.serialization.formatters","system.security.accesscontrol","system.security.cryptography.pkcs","system.security.cryptography.xml","system.security.permissions","system.threading.channels","system.valuetuple"] \ No newline at end of file diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 4bfb3ec8b..126677998 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -4,12 +4,15 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.IO; +using System.Linq; using System.Threading.Tasks; using Autofac; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json; using NuGet.Jobs; using NuGet.Jobs.Configuration; using NuGet.Jobs.Validation; @@ -25,12 +28,14 @@ namespace NuGet.Services.Revalidate public class Job : JsonConfigurationJob { + private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet"; private const string InitializeArgumentName = "Initialize"; private const string VerifyInitializationArgumentName = "VerifyInitialization"; private const string JobConfigurationSectionName = "RevalidateJob"; private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5); + private string _preinstalledSetPath; private bool _initialize; private bool _verifyInitialization; @@ -38,6 +43,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); + var preinstalledPackagesNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var path in config.PreinstalledPaths) + { + var expandedPath = Environment.ExpandEnvironmentVariables(path); + var packagesInPath = Directory.GetDirectories(expandedPath) + .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant()) + .Where(d => !d.StartsWith(".")); + + preinstalledPackagesNames.UnionWith(packagesInPath); + } + + File.WriteAllText(_preinstalledSetPath, JsonConvert.SerializeObject(preinstalledPackagesNames)); + + Logger.LogInformation("Rebuilt the preinstalled package set. Found {PreinstalledPackages} package ids", preinstalledPackagesNames.Count); + } + else if (_initialize || _verifyInitialization) { var initializer = scope.ServiceProvider.GetRequiredService(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 3e5e4f4b9..e32a5766e 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -89,6 +89,9 @@ + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} diff --git a/src/NuGet.Services.Revalidate/Program.cs b/src/NuGet.Services.Revalidate/Program.cs index 9ad233ad7..d6cff1a8f 100644 --- a/src/NuGet.Services.Revalidate/Program.cs +++ b/src/NuGet.Services.Revalidate/Program.cs @@ -10,7 +10,7 @@ class Program static void Main(string[] args) { var job = new Job(); - JobRunner.Run(job, args).GetAwaiter().GetResult(); + JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); } } } diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index 60d0d52aa..f40ff6266 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -3,11 +3,11 @@ "Initialization": { "PreinstalledPaths": [ "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools" + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled - "SleepDurationBetweenBatches": "00:00:01" + "SleepDurationBetweenBatches": "00:00:30" }, "Health": { diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index 0c56b4956..8eb5c9597 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -3,8 +3,8 @@ "Initialization": { "PreinstalledPaths": [ "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools" + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index 354c1d7b8..f299f81ce 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -3,8 +3,8 @@ "Initialization": { "PreinstalledPaths": [ "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder", - "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools" + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder", + "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools" ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" diff --git a/src/NuGet.Services.Revalidate/readme.md b/src/NuGet.Services.Revalidate/readme.md index c9a28c051..61978fd25 100644 --- a/src/NuGet.Services.Revalidate/readme.md +++ b/src/NuGet.Services.Revalidate/readme.md @@ -3,10 +3,23 @@ This job enqueues packages revalidation as fast as possible without affecting the health of NuGet's ingestion pipeline. It does so in two phases: -1. Initialization phase - the job determines which packages should be revalidated. -2. Revalidation phase - packages are enqueued for revalidations +1. Build Preinstalled Packages phase - the job builds a JSON file of packages that +are installed by .NET SDK and Visual Studio +2. Initialization phase - the job determines which packages should be revalidated. +3. Revalidation phase - packages are enqueued for revalidations -The initialization phase MUST complete before the revalidation phase is started. +These phase MUST be completed in order. + +# The Build Preinstalled Packages phase + +This phase should run be at development time before the job is deployed: + +``` +NuGet.Services.Revalidate.exe ^ + -Configuration "C:\Path\to\job\Settings\dev.json" ^ + -RebuildPreinstalledSet "C:\Path\to\job\Initialization\PreinstalledPackages.json" ^ + -Once +``` # The Initialization Phase @@ -14,7 +27,7 @@ To initialize the job, run: ``` NuGet.Services.Revalidate.exe ^ - -Configuration "C:\Path\to\config.json" ^ + -Configuration "C:\Path\to\job\Settings\dev.json" ^ -Initialize -VerifyInitialization -Once @@ -36,6 +49,6 @@ To enqueue revalidations, run: ``` NuGet.Services.Revalidate.exe ^ - -Configuration "C:\Path\to\config.json" ^ + -Configuration "C:\Path\to\job\Settings\dev.json" ^ -Initialize ``` \ No newline at end of file diff --git a/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs index 9deb48c17..7127d9d14 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs @@ -74,8 +74,8 @@ public void FindsPreinstalledPackages() { _context.Mock(packageRegistrations: new[] { - new PackageRegistration { Key = 1, Id = "System.Linq" }, - new PackageRegistration { Key = 2, Id = "Newtonsoft.Json" } + new PackageRegistration { Key = 1, Id = "system.linq" }, + new PackageRegistration { Key = 2, Id = "newtonsoft.json" } }); var actual = _target.FindPreinstalledPackages(except: new HashSet()); @@ -89,8 +89,8 @@ public void SkipsPackagesInExceptSet() { _context.Mock(packageRegistrations: new[] { - new PackageRegistration { Key = 1, Id = "System.Linq" }, - new PackageRegistration { Key = 2, Id = "Newtonsoft.Json" } + new PackageRegistration { Key = 1, Id = "system.linq" }, + new PackageRegistration { Key = 2, Id = "newtonsoft.json" } }); var actual = _target.FindPreinstalledPackages(except: new HashSet { 1 }); @@ -104,7 +104,7 @@ public void SkipsPackagesWithNoRegistration() { _context.Mock(packageRegistrations: new[] { - new PackageRegistration { Key = 1, Id = "System.Linq" }, + new PackageRegistration { Key = 1, Id = "system.linq" }, }); var actual = _target.FindPreinstalledPackages(except: new HashSet()); From ff0e2cbb1e14b6f70efb17fce4ceeade902da7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Fri, 3 Aug 2018 13:36:19 -0700 Subject: [PATCH 2/4] [Revalidate] Slow down based off the Gallery's event rate (#499) This makes the revalidate job slow down if it detects package events on the Gallery. Part of: https://github.com/NuGet/Engineering/issues/1441 --- .../ApplicationInsightsConfiguration.cs | 22 +++++ .../RevalidationConfiguration.cs | 5 ++ src/NuGet.Services.Revalidate/Job.cs | 4 +- .../NuGet.Services.Revalidate.csproj | 4 + .../Services/GalleryService.cs | 82 +++++++++++++++++++ .../Services/IGalleryService.cs | 16 ++++ .../Services/RevalidationThrottler.cs | 43 +++++----- .../Settings/dev.json | 12 +-- .../Settings/int.json | 7 ++ .../Settings/prod.json | 7 ++ .../Services/RevalidationThrottlerFacts.cs | 16 +++- 11 files changed, 190 insertions(+), 28 deletions(-) create mode 100644 src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs create mode 100644 src/NuGet.Services.Revalidate/Services/GalleryService.cs create mode 100644 src/NuGet.Services.Revalidate/Services/IGalleryService.cs diff --git a/src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs new file mode 100644 index 000000000..9331f0dec --- /dev/null +++ b/src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Revalidate +{ + /// + /// The configuration needed to query an Application Insights account using + /// the REST endpoints. + /// + public class ApplicationInsightsConfiguration + { + /// + /// The Application Insights account identifier. + /// + public string AppId { get; set; } + + /// + /// The API Key used to access the Application Insights account. + /// + public string ApiKey { get; set; } + } +} diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs index d91d42d1d..252771914 100644 --- a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs +++ b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs @@ -40,6 +40,11 @@ public class RevalidationConfiguration /// public HealthConfiguration Health { get; set; } + /// + /// The configurations to authenticate to Application Insight's REST endpoints. + /// + public ApplicationInsightsConfiguration AppInsights { get; set; } + /// /// The configurations used by the in-memory queue of revalidations to start. /// diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 126677998..0dd289326 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -124,6 +124,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddSingleton(provider => provider.GetRequiredService>().Value); services.AddSingleton(provider => provider.GetRequiredService>().Value.Initialization); services.AddSingleton(provider => provider.GetRequiredService>().Value.Health); + services.AddSingleton(provider => provider.GetRequiredService>().Value.AppInsights); services.AddSingleton(provider => provider.GetRequiredService>().Value.Queue); services.AddScoped(provider => @@ -139,13 +140,14 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddTransient(); services.AddTransient(); - services.AddTransient(); + services.AddTransient(); // Initialization services.AddTransient(); services.AddTransient(); // Revalidation + services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index e32a5766e..15bd808c4 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -36,6 +36,7 @@ + @@ -44,6 +45,7 @@ + @@ -52,7 +54,9 @@ + + diff --git a/src/NuGet.Services.Revalidate/Services/GalleryService.cs b/src/NuGet.Services.Revalidate/Services/GalleryService.cs new file mode 100644 index 000000000..c7cd52329 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/GalleryService.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using System.Web; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace NuGet.Services.Revalidate +{ + public class GalleryService : IGalleryService + { + private static readonly string GalleryEventsQuery = HttpUtility.UrlPathEncode( + "customMetrics | " + + "where name == \"PackagePush\" or name == \"PackageUnlisted\" or name == \"PackageListed\" | " + + "summarize sum(value)"); + + private readonly HttpClient _httpClient; + private readonly ApplicationInsightsConfiguration _appInsightsConfig; + private readonly ILogger _logger; + + public GalleryService( + HttpClient httpClient, + ApplicationInsightsConfiguration appInsightsConfig, + ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _appInsightsConfig = appInsightsConfig ?? throw new ArgumentNullException(nameof(appInsightsConfig)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CountEventsInPastHourAsync() + { + try + { + using (var request = new HttpRequestMessage()) + { + request.RequestUri = new Uri($"https://api.applicationinsights.io/v1/apps/{_appInsightsConfig.AppId}/query?timespan=PT1H&query={GalleryEventsQuery}"); + request.Method = HttpMethod.Get; + + request.Headers.Add("x-api-key", _appInsightsConfig.ApiKey); + + using (var response = await _httpClient.SendAsync(request)) + { + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var data = JsonConvert.DeserializeObject(json); + + if (data?.Tables?.Length != 1 || + data.Tables[0]?.Rows?.Length != 1 || + data.Tables[0].Rows[0]?.Length != 1) + { + throw new InvalidOperationException("Malformed response content"); + } + + // Get the first row's first column's value. + return data.Tables[0].Rows[0][0]; + } + } + } + catch (Exception e) + { + _logger.LogError(0, e, "Exception thrown when getting the Gallery's package event rate."); + + throw new InvalidOperationException("Exception thrown when getting the Gallery's package event rate.", e); + } + } + + private class QueryResult + { + public QueryTable[] Tables { get; set; } + } + + private class QueryTable + { + public int[][] Rows { get; set; } + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/IGalleryService.cs b/src/NuGet.Services.Revalidate/Services/IGalleryService.cs new file mode 100644 index 000000000..035398d97 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IGalleryService.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + public interface IGalleryService + { + /// + /// Count the number of gallery events (package pushes, listing, and unlisting) in the past hour. + /// + /// The number of gallery events in the past hour. + Task CountEventsInPastHourAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs index 4041832f7..c0afcfba5 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs @@ -11,17 +11,20 @@ public class RevalidationThrottler : IRevalidationThrottler { private readonly IRevalidationJobStateService _jobState; private readonly IPackageRevalidationStateService _packageState; + private readonly IGalleryService _gallery; private readonly RevalidationConfiguration _config; private readonly ILogger _logger; public RevalidationThrottler( IRevalidationJobStateService jobState, IPackageRevalidationStateService packageState, + IGalleryService gallery, RevalidationConfiguration config, ILogger logger) { _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); + _gallery = gallery ?? throw new ArgumentNullException(nameof(gallery)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -29,12 +32,31 @@ public RevalidationThrottler( public async Task IsThrottledAsync() { var desiredRate = await _jobState.GetDesiredPackageEventRateAsync(); - var recentGalleryEvents = await CountGalleryEventsInPastHourAsync(); + var recentGalleryEvents = await _gallery.CountEventsInPastHourAsync(); var recentRevalidations = await _packageState.CountRevalidationsEnqueuedInPastHourAsync(); var revalidationQuota = desiredRate - recentRevalidations - recentGalleryEvents; - return (revalidationQuota <= 0); + if (revalidationQuota <= 0) + { + _logger.LogInformation( + "Throttling revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}", + desiredRate, + recentGalleryEvents, + recentRevalidations); + + return true; + } + else + { + _logger.LogInformation( + "Allowing revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}", + desiredRate, + recentGalleryEvents, + recentRevalidations); + + return false; + } } public async Task DelayUntilNextRevalidationAsync() @@ -55,22 +77,5 @@ public async Task DelayUntilRevalidationRetryAsync() await Task.Delay(_config.RetryLaterSleep); } - - private Task CountGalleryEventsInPastHourAsync() - { - // TODO: Count the number of package pushes, lists, and unlists. - // Run this AI query: - // - // customMetrics | where name == "PackagePush" or name == "PackageUnlisted" or name == "PackageListed" | summarize sum(value) - // - // Using this HTTP request: - // - // GET /v1/apps/46f13c7d-635f-42c3-8120-593edeaad426/query?timespan=P1D&query=customMetrics%20%7C%20where%20name%20%3D%3D%20%22PackagePush%22%20or%20name%20%3D%3D%20%22PackageUnlisted%22%20or%20name%20%3D%3D%20%22PackageListed%22%20%7C%20summarize%20sum(value)%20 HTTP/1.1 - // Host: api.applicationinsights.io - // x-api-key: my-super-secret-api-key - // - // See: https://dev.applicationinsights.io/quickstart - return Task.FromResult(0); - } } } diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index f40ff6266..2612db0dc 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -19,17 +19,17 @@ "MinPackageEventRate": 120, "MaxPackageEventRate": 500, + "AppInsights": { + "AppId": "46f13c7d-635f-42c3-8120-593edeaad426", + "ApiKey": "$$Dev-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$" + }, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" } }, - "Storage": { - "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", - "Container": "revalidations" - }, - "GalleryDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, @@ -44,6 +44,8 @@ "TopicPath": "validation" }, + "PackageDownloadTimeout": "00:10:00", + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index 8eb5c9597..e721eb379 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -19,6 +19,11 @@ "MinPackageEventRate": 120, "MaxPackageEventRate": 500, + "AppInsights": { + "AppId": "718e0c81-9132-4bf2-b24b-aa625dafd800", + "ApiKey": "$$Int-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$" + }, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -39,6 +44,8 @@ "TopicPath": "validation" }, + "PackageDownloadTimeout": "00:10:00", + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index f299f81ce..7a97b2039 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -19,6 +19,11 @@ "MinPackageEventRate": 120, "MaxPackageEventRate": 500, + "AppInsights": { + "AppId": "338f6804-b1a9-4fe3-bba7-c93064e7ae7b", + "ApiKey": "$$Prod-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$" + }, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -39,6 +44,8 @@ "TopicPath": "validation" }, + "PackageDownloadTimeout": "00:10:00", + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs index 7b5da8254..86f6c930c 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs @@ -14,6 +14,7 @@ public class TheIsThrottledAsyncMethod { private readonly Mock _settings; private readonly Mock _state; + private readonly Mock _gallery; private readonly RevalidationConfiguration _config; private readonly IRevalidationThrottler _target; @@ -22,28 +23,35 @@ public TheIsThrottledAsyncMethod() { _settings = new Mock(); _state = new Mock(); + _gallery = new Mock(); _config = new RevalidationConfiguration(); _target = new RevalidationThrottler( _settings.Object, _state.Object, + _gallery.Object, _config, Mock.Of>()); } - [Fact] - public async Task ReturnsTrueIfRecentRevalidationsMoreThanDesiredRate() + [Theory] + [InlineData(100, 0)] + [InlineData(0, 100)] + [InlineData(40, 40)] + public async Task ReturnsTrueIfRecentRevalidationsMoreThanDesiredRate(int enqueuedRevalidations, int galleryEvents) { // Arrange _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(50); - _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(100); + _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(enqueuedRevalidations); + _gallery.Setup(g => g.CountEventsInPastHourAsync()).ReturnsAsync(galleryEvents); // Act & Assert Assert.True(await _target.IsThrottledAsync()); _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + _gallery.Verify(g => g.CountEventsInPastHourAsync(), Times.Once); } [Fact] @@ -52,12 +60,14 @@ public async Task ReturnsFalseIfRecentRevalidationsLessThanDesiredRate() // Arrange _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(100); _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(50); + _gallery.Setup(g => g.CountEventsInPastHourAsync()).ReturnsAsync(40); // Act & Assert Assert.False(await _target.IsThrottledAsync()); _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + _gallery.Verify(g => g.CountEventsInPastHourAsync(), Times.Once); } } } From cb186cddaaaa0c49851e49bc54d937ac82b5c563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Fri, 3 Aug 2018 16:49:04 -0700 Subject: [PATCH 3/4] [Repository Signing] Order owners metadata lexicographically (#513) Part of https://github.com/NuGet/NuGetGallery/issues/6258 --- src/NuGet.Services.Revalidate/readme.md | 2 +- .../ScanAndSign/ScanAndSignProcessor.cs | 2 ++ .../ScanAndSignProcessorFacts.cs | 13 +++++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/NuGet.Services.Revalidate/readme.md b/src/NuGet.Services.Revalidate/readme.md index 61978fd25..99f97a821 100644 --- a/src/NuGet.Services.Revalidate/readme.md +++ b/src/NuGet.Services.Revalidate/readme.md @@ -8,7 +8,7 @@ are installed by .NET SDK and Visual Studio 2. Initialization phase - the job determines which packages should be revalidated. 3. Revalidation phase - packages are enqueued for revalidations -These phase MUST be completed in order. +These phases MUST be completed in order. # The Build Preinstalled Packages phase diff --git a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs index ffc2bf8dd..626891557 100644 --- a/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs +++ b/src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs @@ -246,6 +246,8 @@ private List FindPackageOwners(IValidationRequest request) return registration .Owners .Select(o => o.Username) + .ToList() + .OrderBy(u => u, StringComparer.InvariantCultureIgnoreCase) .ToList(); } } diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs index 978035e1c..0c3bdd2f0 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs @@ -251,9 +251,12 @@ public async Task EnqueuesScanAndSignIfPackageHasNoRepositorySignature() _request.NupkgUrl, _config.V3ServiceIndexUrl, It.Is>(l => - l.Count() == 2 && - l.Contains("Billy") && - l.Contains("Bob"))), + // Ensure that the owners are lexicographically ordered. + l.Count() == 4 && + l[0] == "Annie" && + l[1] == "Bob" && + l[2] == "zack" && + l[3] == "Zorro")), Times.Once); _validatorStateServiceMock @@ -441,8 +444,10 @@ public async Task WhenPackageFitsCriteriaAndIsNotRepositorySigned_DoesNotSkipSca { Owners = new List { - new User("Billy"), + new User("Zorro"), new User("Bob"), + new User("Annie"), + new User("zack") } }; From 32182eea3e0f4c04d5274a019c9e6d37c406f11e Mon Sep 17 00:00:00 2001 From: Scott Bommarito Date: Mon, 6 Aug 2018 13:27:23 -0700 Subject: [PATCH 4/4] Status Aggregator - Add traffic manager incident support (#514) --- src/StatusAggregator/Job.cs | 1 + ...fficManagerEndpointStatusIncidentParser.cs | 178 ++++++++++++++++++ src/StatusAggregator/StatusAggregator.csproj | 1 + 3 files changed, 180 insertions(+) create mode 100644 src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs diff --git a/src/StatusAggregator/Job.cs b/src/StatusAggregator/Job.cs index c1b8a4c9c..0b0dd9617 100644 --- a/src/StatusAggregator/Job.cs +++ b/src/StatusAggregator/Job.cs @@ -61,6 +61,7 @@ private static void AddParsing(IServiceCollection serviceCollection) serviceCollection.AddTransient(); serviceCollection.AddTransient(); serviceCollection.AddTransient(); + serviceCollection.AddTransient(); serviceCollection.AddTransient(); } diff --git a/src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs b/src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs new file mode 100644 index 000000000..d44047c84 --- /dev/null +++ b/src/StatusAggregator/Parse/TrafficManagerEndpointStatusIncidentParser.cs @@ -0,0 +1,178 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; +using NuGet.Services.Incidents; +using NuGet.Services.Status; +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace StatusAggregator.Parse +{ + public class TrafficManagerEndpointStatusIncidentParser : EnvironmentPrefixIncidentParser + { + private const string DomainGroupName = "Domain"; + private const string TargetGroupName = "Target"; + private static string SubtitleRegEx = $"Traffic Manager for (?<{DomainGroupName}>.*) is reporting (?<{TargetGroupName}>.*) as not Online!"; + + private readonly ILogger _logger; + + public TrafficManagerEndpointStatusIncidentParser( + IEnumerable filters, + ILogger logger) + : base(SubtitleRegEx, filters, logger) + { + _logger = logger; + } + + protected override bool TryParseAffectedComponentPath(Incident incident, GroupCollection groups, out string affectedComponentPath) + { + affectedComponentPath = null; + + var domain = groups[DomainGroupName].Value; + var target = groups[TargetGroupName].Value; + var environment = groups[EnvironmentFilter.EnvironmentGroupName].Value; + _logger.LogInformation("Domain is {Domain}, target is {Target}, environment is {Environment}.", domain, target, environment); + + if (EnvironmentToDomainToTargetToPath.TryGetValue(environment, out var domainToTargetToPath) && + domainToTargetToPath.TryGetValue(domain, out var targetToPath) && + targetToPath.TryGetValue(target, out var path)) + { + affectedComponentPath = path; + } + + return affectedComponentPath != null; + } + + protected override bool TryParseAffectedComponentStatus(Incident incident, GroupCollection groups, out ComponentStatus affectedComponentStatus) + { + affectedComponentStatus = ComponentStatus.Down; + return true; + } + + private static readonly string GalleryUsncPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.GalleryName, + NuGetServiceComponentFactory.UsncInstanceName); + + private static readonly string GalleryUsscPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.GalleryName, + NuGetServiceComponentFactory.UsscInstanceName); + + private static readonly string RestoreV3GlobalPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.RestoreName, + NuGetServiceComponentFactory.V3ProtocolName, + NuGetServiceComponentFactory.GlobalRegionName); + + private static readonly string RestoreV3ChinaPath = + ComponentUtility.GetPath( + NuGetServiceComponentFactory.RootName, + NuGetServiceComponentFactory.RestoreName, + NuGetServiceComponentFactory.V3ProtocolName, + NuGetServiceComponentFactory.ChinaRegionName); + + private static readonly IDictionary> DevDomainToTargetToPath = + new Dictionary> + { + { + "devnugettest.trafficmanager.net", + new Dictionary + { + { + "nuget-dev-use2-gallery.cloudapp.net", + GalleryUsncPath + }, + + { + "nuget-dev-ussc-gallery.cloudapp.net", + GalleryUsncPath + } + } + }, + + { + "nugetapidev.trafficmanager.net", + new Dictionary + { + { + "az635243.vo.msecnd.net", + RestoreV3GlobalPath + }, + { + "nugetdevcnredirect.trafficmanager.net", + RestoreV3ChinaPath + } + } + } + }; + + private static readonly IDictionary> IntDomainToTargetToPath = + new Dictionary> + { + { + "nuget-int-test-failover.trafficmanager.net", + new Dictionary + { + { + "nuget-int-0-v2gallery.cloudapp.net", + GalleryUsncPath + }, + + { + "nuget-int-ussc-gallery.cloudapp.net", + GalleryUsncPath + } + } + } + }; + + private static readonly IDictionary> ProdDomainToTargetToPath = + new Dictionary> + { + { + "nuget-prod-v2gallery.trafficmanager.net", + new Dictionary + { + { + "nuget-prod-0-v2gallery.cloudapp.net", + GalleryUsncPath + }, + + { + "nuget-prod-ussc-gallery.cloudapp.net", + GalleryUsncPath + } + } + }, + + { + "nugetapiprod.trafficmanager.net", + new Dictionary + { + { + "az320820.vo.msecnd.net", + RestoreV3GlobalPath + }, + { + "nugetprodcnredirect.trafficmanager.net", + RestoreV3ChinaPath + } + } + } + }; + + private static readonly IDictionary>> EnvironmentToDomainToTargetToPath = + new Dictionary>> + { + { "dev", DevDomainToTargetToPath }, + { "test", DevDomainToTargetToPath }, + { "int", IntDomainToTargetToPath }, + { "prod", ProdDomainToTargetToPath } + }; + } +} diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 00eba2850..6908130c1 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -48,6 +48,7 @@ +