From 90872d37678efab7b58cfe5330e2e7241fd94b06 Mon Sep 17 00:00:00 2001
From: Joel Verhagen <jver@microsoft.com>
Date: Fri, 24 Jan 2020 11:51:13 -0800
Subject: [PATCH] Add configurable parallelism for RegistrationComparer

Progress on https://github.com/nuget/nugetgallery/issues/7741
---
 src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs        | 7 ++++++-
 .../RegistrationComparerCollectorLogic.cs                  | 2 +-
 .../RegistrationComparerConfiguration.cs                   | 2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs b/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs
index acb027aed..0e9780d35 100644
--- a/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs
+++ b/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs
@@ -10,6 +10,7 @@
 using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NuGet.Protocol.Catalog;
@@ -22,15 +23,18 @@ public class HiveComparer
     {
         private readonly HttpClient _httpClient;
         private readonly JsonComparer _comparer;
+        private readonly IOptionsSnapshot<RegistrationComparerConfiguration> _options;
         private readonly ILogger<HiveComparer> _logger;
 
         public HiveComparer(
             HttpClient httpClient,
             JsonComparer comparer,
+            IOptionsSnapshot<RegistrationComparerConfiguration> options,
             ILogger<HiveComparer> logger)
         {
             _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
             _comparer = comparer ?? throw new ArgumentNullException(nameof(comparer));
+            _options = options ?? throw new ArgumentNullException(nameof(options));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
@@ -113,7 +117,8 @@ await ParallelAsync.Repeat(
                         var json = await GetJObjectOrNullAsync(pageUrl);
                         urlToJson.TryAdd(pageUrl, json.Data);
                     }
-                });
+                },
+                _options.Value.MaxConcurrentPageAndLeafDownloadsPerId);
 
             // Compare the pages.
             for (var i = 1; i < baseUrls.Count; i++)
diff --git a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs
index 2de68904e..b02456f59 100644
--- a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs
+++ b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs
@@ -103,7 +103,7 @@ await ParallelAsync
                         await work();
                     }
                 },
-                degreeOfParallelism: 32);
+                degreeOfParallelism: _options.Value.MaxConcurrentComparisons);
 
             if (failures > 0)
             {
diff --git a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs
index 48f3a9632..8eda1d562 100644
--- a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs
+++ b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs
@@ -14,6 +14,8 @@ public class RegistrationComparerConfiguration : ICommitCollectorConfiguration
         public List<HivesConfiguration> Registrations { get; set; }
         public string Source { get; set; }
         public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64;
+        public int MaxConcurrentComparisons { get; set; } = 32;
+        public int MaxConcurrentPageAndLeafDownloadsPerId { get; set; } = 32;
         public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10);
     }
 }