[ReleasePrep][2018.08.06] RI of dev into master

src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.Revalidate
+{
+    /// <summary>
+    /// The configuration needed to query an Application Insights account using
+    /// the REST endpoints.
+    /// </summary>
+    public class ApplicationInsightsConfiguration
+    {
+        /// <summary>
+        /// The Application Insights account identifier.
+        /// </summary>
+        public string AppId { get; set; }
+
+        /// <summary>
+        /// The API Key used to access the Application Insights account.
+        /// </summary>
+        public string ApiKey { get; set; }
+    }
+}

src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs

@@ -40,6 +40,11 @@ public class RevalidationConfiguration
         /// </summary>
         public HealthConfiguration Health { get; set; }
 
+        /// <summary>
+        /// The configurations to authenticate to Application Insight's REST endpoints.
+        /// </summary>
+        public ApplicationInsightsConfiguration AppInsights { get; set; }
+
         /// <summary>
         /// The configurations used by the in-memory queue of revalidations to start.
         /// </summary>

src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs

@@ -8,6 +8,7 @@
 using System.Linq.Expressions;
 using System.Threading;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 using NuGet.Versioning;
 using NuGetGallery;
 
@@ -22,6 +23,8 @@ public class PackageFinder : IPackageFinder
         public const string DependencySetName = "Dependency";
         public const string RemainingSetName = "Remaining";
 
+        private const string PreinstalledPackagesResource = "NuGet.Services.Revalidate.Initialization.PreinstalledPackages.json";
+
         private static int BatchSize = 1000;
         private static string MicrosoftAccountName = "Microsoft";
 
@@ -46,16 +49,13 @@ public HashSet<int> FindMicrosoftPackages()
 
         public HashSet<int> FindPreinstalledPackages(HashSet<int> except)
         {
-            var preinstalledPackagesNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            List<string> preinstalledPackagesNames;
+            var assembly = typeof(PackageFinder).Assembly;
 
-            foreach (var path in _config.PreinstalledPaths)
+            using (var resource = assembly.GetManifestResourceStream(PreinstalledPackagesResource))
+            using (var reader = new StreamReader(resource))
             {
-                var expandedPath = Environment.ExpandEnvironmentVariables(path);
-                var packagesInPath = Directory.GetDirectories(expandedPath)
-                    .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant())
-                    .Where(d => !d.StartsWith("."));
-
-                preinstalledPackagesNames.UnionWith(packagesInPath);
+                preinstalledPackagesNames = JsonConvert.DeserializeObject<List<string>>(reader.ReadToEnd());
             }
 
             var preinstalledPackages = FindRegistrationKeys(PreinstalledSetName, r => preinstalledPackagesNames.Contains(r.Id));

src/NuGet.Services.Revalidate/Job.cs

@@ -4,12 +4,15 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Design;
+using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Autofac;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Newtonsoft.Json;
 using NuGet.Jobs;
 using NuGet.Jobs.Configuration;
 using NuGet.Jobs.Validation;
@@ -25,19 +28,22 @@ namespace NuGet.Services.Revalidate
 
     public class Job : JsonConfigurationJob
     {
+        private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet";
         private const string InitializeArgumentName = "Initialize";
         private const string VerifyInitializationArgumentName = "VerifyInitialization";
         private const string JobConfigurationSectionName = "RevalidateJob";
 
         private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5);
 
+        private string _preinstalledSetPath;
         private bool _initialize;
         private bool _verifyInitialization;
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
             base.Init(serviceContainer, jobArgsDictionary);
 
+            _preinstalledSetPath = JobConfigurationManager.TryGetArgument(jobArgsDictionary, RebuildPreinstalledSetArgumentName);
             _initialize = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, InitializeArgumentName);
             _verifyInitialization = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, VerifyInitializationArgumentName);
 
@@ -56,7 +62,28 @@ public override async Task Run()
         {
             using (var scope = _serviceProvider.CreateScope())
             {
-                if (_initialize || _verifyInitialization)
+                if (!string.IsNullOrEmpty(_preinstalledSetPath))
+                {
+                    Logger.LogInformation("Rebuilding the preinstalled packages set...");
+
+                    var config = scope.ServiceProvider.GetRequiredService<InitializationConfiguration>();
+                    var preinstalledPackagesNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+                    foreach (var path in config.PreinstalledPaths)
+                    {
+                        var expandedPath = Environment.ExpandEnvironmentVariables(path);
+                        var packagesInPath = Directory.GetDirectories(expandedPath)
+                            .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant())
+                            .Where(d => !d.StartsWith("."));
+
+                        preinstalledPackagesNames.UnionWith(packagesInPath);
+                    }
+
+                    File.WriteAllText(_preinstalledSetPath, JsonConvert.SerializeObject(preinstalledPackagesNames));
+
+                    Logger.LogInformation("Rebuilt the preinstalled package set. Found {PreinstalledPackages} package ids", preinstalledPackagesNames.Count);
+                }
+                else if (_initialize || _verifyInitialization)
                 {
                     var initializer = scope.ServiceProvider.GetRequiredService<InitializationManager>();
 
@@ -97,6 +124,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Health);
+            services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.AppInsights);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Queue);
 
             services.AddScoped<IGalleryContext>(provider =>
@@ -112,13 +140,14 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
 
             services.AddTransient<IPackageRevalidationStateService, PackageRevalidationStateService>();
             services.AddTransient<IRevalidationJobStateService, RevalidationJobStateService>();
-            services.AddTransient<NuGetGallery.IRevalidationStateService, NuGetGallery.RevalidationStateService>();
+            services.AddTransient<IRevalidationStateService, RevalidationStateService>();
 
             // Initialization
             services.AddTransient<IPackageFinder, PackageFinder>();
             services.AddTransient<InitializationManager>();
 
             // Revalidation
+            services.AddTransient<IGalleryService, GalleryService>();
             services.AddTransient<IHealthService, HealthService>();
             services.AddTransient<IRevalidationQueue, RevalidationQueue>();
             services.AddTransient<IRevalidationService, RevalidationService>();

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -36,6 +36,7 @@
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Web" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -44,6 +45,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Configuration\HealthConfiguration.cs" />
+    <Compile Include="Configuration\ApplicationInsightsConfiguration.cs" />
     <Compile Include="Configuration\InitializationConfiguration.cs" />
     <Compile Include="Configuration\RevalidationQueueConfiguration.cs" />
     <Compile Include="Extensions\IEnumerableExtensions.cs" />
@@ -52,7 +54,9 @@
     <Compile Include="Initialization\IPackageFinder.cs" />
     <Compile Include="Initialization\PackageFinder.cs" />
     <Compile Include="Initialization\PackageRegistrationInformation.cs" />
+    <Compile Include="Services\GalleryService.cs" />
     <Compile Include="Services\HealthService.cs" />
+    <Compile Include="Services\IGalleryService.cs" />
     <Compile Include="Services\IHealthService.cs" />
     <Compile Include="Services\IRevalidationQueue.cs" />
     <Compile Include="Services\IRevalidationJobStateService.cs" />
@@ -89,6 +93,9 @@
   <ItemGroup>
     <Content Include="Scripts\nssm.exe" />
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Initialization\PreinstalledPackages.json" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj">
       <Project>{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}</Project>

src/NuGet.Services.Revalidate/Program.cs

@@ -10,7 +10,7 @@ class Program
         static void Main(string[] args)
         {
             var job = new Job();
-            JobRunner.Run(job, args).GetAwaiter().GetResult();
+            JobRunner.RunOnce(job, args).GetAwaiter().GetResult();
         }
     }
 }

src/NuGet.Services.Revalidate/Services/GalleryService.cs

@@ -0,0 +1,82 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Web;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+
+namespace NuGet.Services.Revalidate
+{
+    public class GalleryService : IGalleryService
+    {
+        private static readonly string GalleryEventsQuery = HttpUtility.UrlPathEncode(
+            "customMetrics | " +
+            "where name == \"PackagePush\" or name == \"PackageUnlisted\" or name == \"PackageListed\" | " +
+            "summarize sum(value)");
+
+        private readonly HttpClient _httpClient;
+        private readonly ApplicationInsightsConfiguration _appInsightsConfig;
+        private readonly ILogger<GalleryService> _logger;
+
+        public GalleryService(
+            HttpClient httpClient,
+            ApplicationInsightsConfiguration appInsightsConfig,
+            ILogger<GalleryService> logger)
+        {
+            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+            _appInsightsConfig = appInsightsConfig ?? throw new ArgumentNullException(nameof(appInsightsConfig));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task<int> CountEventsInPastHourAsync()
+        {
+            try
+            {
+                using (var request = new HttpRequestMessage())
+                {
+                    request.RequestUri = new Uri($"https://api.applicationinsights.io/v1/apps/{_appInsightsConfig.AppId}/query?timespan=PT1H&query={GalleryEventsQuery}");
+                    request.Method = HttpMethod.Get;
+
+                    request.Headers.Add("x-api-key", _appInsightsConfig.ApiKey);
+
+                    using (var response = await _httpClient.SendAsync(request))
+                    {
+                        response.EnsureSuccessStatusCode();
+
+                        var json = await response.Content.ReadAsStringAsync();
+                        var data = JsonConvert.DeserializeObject<QueryResult>(json);
+
+                        if (data?.Tables?.Length != 1 ||
+                            data.Tables[0]?.Rows?.Length != 1 ||
+                            data.Tables[0].Rows[0]?.Length != 1)
+                        {
+                            throw new InvalidOperationException("Malformed response content");
+                        }
+
+                        // Get the first row's first column's value.
+                        return data.Tables[0].Rows[0][0];
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(0, e, "Exception thrown when getting the Gallery's package event rate.");
+
+                throw new InvalidOperationException("Exception thrown when getting the Gallery's package event rate.", e);
+            }
+        }
+
+        private class QueryResult
+        {
+            public QueryTable[] Tables { get; set; }
+        }
+
+        private class QueryTable
+        {
+            public int[][] Rows { get; set; }
+        }
+    }
+}

src/NuGet.Services.Revalidate/Services/IGalleryService.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Revalidate
+{
+    public interface IGalleryService
+    {
+        /// <summary>
+        /// Count the number of gallery events (package pushes, listing, and unlisting) in the past hour.
+        /// </summary>
+        /// <returns>The number of gallery events in the past hour.</returns>
+        Task<int> CountEventsInPastHourAsync();
+    }
+}

src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs

@@ -11,30 +11,52 @@ public class RevalidationThrottler : IRevalidationThrottler
     {
         private readonly IRevalidationJobStateService _jobState;
         private readonly IPackageRevalidationStateService _packageState;
+        private readonly IGalleryService _gallery;
         private readonly RevalidationConfiguration _config;
         private readonly ILogger<RevalidationThrottler> _logger;
 
         public RevalidationThrottler(
             IRevalidationJobStateService jobState,
             IPackageRevalidationStateService packageState,
+            IGalleryService gallery,
             RevalidationConfiguration config,
             ILogger<RevalidationThrottler> logger)
         {
             _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState));
             _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState));
+            _gallery = gallery ?? throw new ArgumentNullException(nameof(gallery));
             _config = config ?? throw new ArgumentNullException(nameof(config));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
         public async Task<bool> IsThrottledAsync()
         {
             var desiredRate = await _jobState.GetDesiredPackageEventRateAsync();
-            var recentGalleryEvents = await CountGalleryEventsInPastHourAsync();
+            var recentGalleryEvents = await _gallery.CountEventsInPastHourAsync();
             var recentRevalidations = await _packageState.CountRevalidationsEnqueuedInPastHourAsync();
 
             var revalidationQuota = desiredRate - recentRevalidations - recentGalleryEvents;
 
-            return (revalidationQuota <= 0);
+            if (revalidationQuota <= 0)
+            {
+                _logger.LogInformation(
+                    "Throttling revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}",
+                    desiredRate,
+                    recentGalleryEvents,
+                    recentRevalidations);
+
+                return true;
+            }
+            else
+            {
+                _logger.LogInformation(
+                    "Allowing revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}",
+                    desiredRate,
+                    recentGalleryEvents,
+                    recentRevalidations);
+
+                return false;
+            }
         }
 
         public async Task DelayUntilNextRevalidationAsync()
@@ -55,22 +77,5 @@ public async Task DelayUntilRevalidationRetryAsync()
 
             await Task.Delay(_config.RetryLaterSleep);
         }
-
-        private Task<int> CountGalleryEventsInPastHourAsync()
-        {
-            // TODO: Count the number of package pushes, lists, and unlists.
-            // Run this AI query:
-            //
-            //   customMetrics | where name == "PackagePush" or name == "PackageUnlisted" or name == "PackageListed" | summarize sum(value)
-            //
-            // Using this HTTP request:
-            //
-            // GET /v1/apps/46f13c7d-635f-42c3-8120-593edeaad426/query?timespan=P1D&query=customMetrics%20%7C%20where%20name%20%3D%3D%20%22PackagePush%22%20or%20name%20%3D%3D%20%22PackageUnlisted%22%20or%20name%20%3D%3D%20%22PackageListed%22%20%7C%20summarize%20sum(value)%20 HTTP/1.1
-            // Host: api.applicationinsights.io
-            // x-api-key: my-super-secret-api-key
-            //
-            // See: https://dev.applicationinsights.io/quickstart
-            return Task.FromResult(0);
-        }
     }
 }