Add configurable parallelism for RegistrationComparer

Progress on NuGet/NuGetGallery#7741

src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs

@@ -10,6 +10,7 @@
 using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NuGet.Protocol.Catalog;
@@ -22,15 +23,18 @@ public class HiveComparer
     {
         private readonly HttpClient _httpClient;
         private readonly JsonComparer _comparer;
+        private readonly IOptionsSnapshot<RegistrationComparerConfiguration> _options;
         private readonly ILogger<HiveComparer> _logger;
 
         public HiveComparer(
             HttpClient httpClient,
             JsonComparer comparer,
+            IOptionsSnapshot<RegistrationComparerConfiguration> options,
             ILogger<HiveComparer> logger)
         {
             _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
             _comparer = comparer ?? throw new ArgumentNullException(nameof(comparer));
+            _options = options ?? throw new ArgumentNullException(nameof(options));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
@@ -113,7 +117,8 @@ await ParallelAsync.Repeat(
                         var json = await GetJObjectOrNullAsync(pageUrl);
                         urlToJson.TryAdd(pageUrl, json.Data);
                     }
-                });
+                },
+                _options.Value.MaxConcurrentPageAndLeafDownloadsPerId);
 
             // Compare the pages.
             for (var i = 1; i < baseUrls.Count; i++)

src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs

@@ -103,7 +103,7 @@ await ParallelAsync
                         await work();
                     }
                 },
-                degreeOfParallelism: 32);
+                degreeOfParallelism: _options.Value.MaxConcurrentComparisons);
 
             if (failures > 0)
             {

src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs

@@ -14,6 +14,8 @@ public class RegistrationComparerConfiguration : ICommitCollectorConfiguration
         public List<HivesConfiguration> Registrations { get; set; }
         public string Source { get; set; }
         public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64;
+        public int MaxConcurrentComparisons { get; set; } = 32;
+        public int MaxConcurrentPageAndLeafDownloadsPerId { get; set; } = 32;
         public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10);
     }
 }