diff --git a/NuGet.Services.Metadata.sln b/NuGet.Services.Metadata.sln index 276abadfa..81e84139d 100644 --- a/NuGet.Services.Metadata.sln +++ b/NuGet.Services.Metadata.sln @@ -61,10 +61,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2Registra EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3.Tests", "tests\NuGet.Services.V3.Tests\NuGet.Services.V3.Tests.csproj", "{CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.RegistrationComparer", "src\NuGet.Jobs.RegistrationComparer\NuGet.Jobs.RegistrationComparer.csproj", "{4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.RegistrationComparer.Tests", "tests\NuGet.Jobs.RegistrationComparer.Tests\NuGet.Jobs.RegistrationComparer.Tests.csproj", "{A0E0698A-1161-4DEA-81A9-06D30FB16538}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -315,30 +311,6 @@ Global {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Mixed Platforms.Build.0 = Release|Any CPU {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|x64.ActiveCfg = Release|Any CPU {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|x64.Build.0 = Release|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Debug|x64.ActiveCfg = Debug|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Debug|x64.Build.0 = Debug|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Release|Any CPU.Build.0 = Release|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Release|x64.ActiveCfg = Release|Any CPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C}.Release|x64.Build.0 = Release|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Debug|x64.ActiveCfg = Debug|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Debug|x64.Build.0 = Debug|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Release|Any CPU.Build.0 = Release|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Release|x64.ActiveCfg = Release|Any CPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -364,8 +336,6 @@ Global {C3F9A738-9759-4B2B-A50D-6507B28A659B} = {5DE01C58-D5F7-482F-8256-A8333064384C} {296703A3-67BA-4876-8C1D-ACE13DF901EF} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} - {A0E0698A-1161-4DEA-81A9-06D30FB16538} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D3AB83E9-02B4-4FFA-A2D0-637F0B97E626} diff --git a/build.ps1 b/build.ps1 index 1baf81bbb..97b63212e 100644 --- a/build.ps1 +++ b/build.ps1 @@ -85,8 +85,7 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { "src\NuGet.Jobs.Catalog2AzureSearch\Properties\AssemblyInfo.g.cs", ` "src\NuGet.Services.SearchService\Properties\AssemblyInfo.g.cs", ` "src\NuGet.Jobs.Auxiliary2AzureSearch\Properties\AssemblyInfo.g.cs", ` - "src\NuGet.Jobs.Catalog2Registration\Properties\AssemblyInfo.g.cs", ` - "src\NuGet.Jobs.RegistrationComparer\Properties\AssemblyInfo.g.cs" + "src\NuGet.Jobs.Catalog2Registration\Properties\AssemblyInfo.g.cs" Foreach ($assemblyInfo in $assemblyInfos) { Set-VersionInfo -Path (Join-Path $PSScriptRoot $assemblyInfo) -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA @@ -135,8 +134,7 @@ Invoke-BuildStep 'Creating artifacts' { "src\NuGet.Jobs.Db2AzureSearch\NuGet.Jobs.Db2AzureSearch.nuspec", ` "src\NuGet.Jobs.Catalog2AzureSearch\NuGet.Jobs.Catalog2AzureSearch.nuspec", ` "src\NuGet.Jobs.Auxiliary2AzureSearch\NuGet.Jobs.Auxiliary2AzureSearch.nuspec", ` - "src\NuGet.Jobs.Catalog2Registration\NuGet.Jobs.Catalog2Registration.nuspec", ` - "src\NuGet.Jobs.RegistrationComparer\NuGet.Jobs.RegistrationComparer.nuspec" + "src\NuGet.Jobs.Catalog2Registration\NuGet.Jobs.Catalog2Registration.nuspec" $nuspecPackages | ForEach-Object { New-Package (Join-Path $PSScriptRoot $_) -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch diff --git a/src/NuGet.Jobs.RegistrationComparer/App.config b/src/NuGet.Jobs.RegistrationComparer/App.config deleted file mode 100644 index 56efbc7b5..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/App.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/src/NuGet.Jobs.RegistrationComparer/ComparisonContext.cs b/src/NuGet.Jobs.RegistrationComparer/ComparisonContext.cs deleted file mode 100644 index e24bfe39f..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/ComparisonContext.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace NuGet.Jobs.RegistrationComparer -{ - public class ComparisonContext - { - public ComparisonContext( - string packageId, - string leftBaseUrl, - string rightBaseUrl, - string leftUrl, - string rightUrl, - Normalizers normalizers) - { - PackageId = packageId; - LeftBaseUrl = leftBaseUrl; - RightBaseUrl = rightBaseUrl; - LeftUrl = leftUrl; - RightUrl = rightUrl; - Normalizers = normalizers; - } - - public string PackageId { get; } - public string LeftBaseUrl { get; } - public string RightBaseUrl { get; } - public string LeftUrl { get; } - public string RightUrl { get; } - public Normalizers Normalizers { get; } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/CursorUtility.cs b/src/NuGet.Jobs.RegistrationComparer/CursorUtility.cs deleted file mode 100644 index 20eb14819..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/CursorUtility.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Net.Http; -using Microsoft.Extensions.Options; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Persistence; - -namespace NuGet.Jobs.RegistrationComparer -{ - public static class CursorUtility - { - public static Dictionary GetRegistrationCursors( - Func handlerFunc, - IOptionsSnapshot options) - { - var hiveCursors = new Dictionary(); - foreach (var hives in options.Value.Registrations) - { - var cursorUrl = new Uri(hives.Legacy.StorageBaseUrl.TrimEnd('/') + "/cursor.json"); - hiveCursors.Add(cursorUrl.AbsoluteUri, new HttpReadCursor(cursorUrl, DateTime.MinValue, handlerFunc)); - } - - return hiveCursors; - } - - public static KeyValuePair GetComparerCursor(IStorageFactory storageFactory) - { - return GetDurableCursor(storageFactory, "comparer-cursor.json"); - } - - private static KeyValuePair GetDurableCursor(IStorageFactory storageFactory, string name) - { - var cursorStorage = storageFactory.Create(); - var cursorUri = cursorStorage.ResolveUri(name); - return new KeyValuePair( - cursorUri.AbsoluteUri, - new DurableCursor(cursorUri, cursorStorage, DateTime.MinValue)); - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs b/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs deleted file mode 100644 index 419adf0b2..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/HiveComparer.cs +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NuGet.Protocol.Catalog; -using NuGet.Protocol.Registration; -using NuGet.Services.Metadata.Catalog.Helpers; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class HiveComparer - { - private static long _requestId = 0; - - private readonly HttpClient _httpClient; - private readonly JsonComparer _comparer; - private readonly IOptionsSnapshot _options; - private readonly ILogger _logger; - - public HiveComparer( - HttpClient httpClient, - JsonComparer comparer, - IOptionsSnapshot options, - ILogger logger) - { - _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); - _comparer = comparer ?? throw new ArgumentNullException(nameof(comparer)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task CompareAsync( - IReadOnlyList hives, - string id, - IReadOnlyList versions) - { - if (hives.Count <= 1) - { - throw new ArgumentException("At least two hive configurations must be provided.", nameof(hives)); - } - - // Compare the indexes. - var rawIndexes = await Task.WhenAll(hives.Select(x => GetIndexAsync(x, id))); - var areBothMissing = false; - for (var i = 1; i < hives.Count; i++) - { - if (AreBothMissing( - rawIndexes[i - 1].Url, - rawIndexes[i].Url, - rawIndexes[i - 1].Data, - rawIndexes[i].Data)) - { - areBothMissing = true; - continue; - } - - var comparisonContext = new ComparisonContext( - id, - hives[i - 1].BaseUrl, - hives[i].BaseUrl, - rawIndexes[i - 1].Url, - rawIndexes[i].Url, - Normalizers.Index); - - _comparer.Compare( - rawIndexes[i - 1].Data, - rawIndexes[i].Data, - comparisonContext); - } - - if (areBothMissing) - { - return; - } - - // Deserialize the indexes so we can get the page URLs. - var indexes = new List>(); - foreach (var rawIndex in rawIndexes) - { - indexes.Add(new DownloadedData( - rawIndex.Hive, - rawIndex.Url, - rawIndex.StorageUrl, - rawIndex.Data.ToObject(NuGetJsonSerialization.Serializer))); - } - - // Download the pages (if any) and leaves. - var pageUrlGroups = indexes - .Select((x, i) => x - .Data - .Items - .Where(p => p.Items == null) - .Select(p => new { x.Hive, p.Url }) - .ToList()) - .ToList(); - var leafUrlGroups = hives - .Select(x => versions - .Select(v => $"{x.BaseUrl}{id}/{v}.json") - .Select(u => new { Hive = x, Url = u }) - .ToList()) - .ToList(); - - var pairs = new ConcurrentBag>(pageUrlGroups - .SelectMany(x => x) - .Concat(leafUrlGroups.SelectMany(x => x)) - .Select(x => new KeyValuePair(x.Url, x.Hive))); - var urlToJson = new ConcurrentDictionary(); - await ParallelAsync.Repeat( - async () => - { - await Task.Yield(); - while (pairs.TryTake(out var pair)) - { - var data = await GetJObjectOrNullAsync(pair.Value, pair.Key); - urlToJson.TryAdd(pair.Key, data.Data); - } - }, - _options.Value.MaxConcurrentPageAndLeafDownloadsPerId); - - // Compare the pages. - for (var i = 1; i < hives.Count; i++) - { - for (var pageIndex = 0; pageIndex < pageUrlGroups[i].Count; pageIndex++) - { - var left = pageUrlGroups[i - 1][pageIndex]; - var right = pageUrlGroups[i][pageIndex]; - - var comparisonContext = new ComparisonContext( - id, - hives[i - 1].BaseUrl, - hives[i].BaseUrl, - left.Url, - right.Url, - Normalizers.Page); - - _comparer.Compare( - urlToJson[left.Url], - urlToJson[right.Url], - comparisonContext); - } - } - - // Compare the affected leaves. - for (var i = 1; i < hives.Count; i++) - { - for (var leafIndex = 0; leafIndex < leafUrlGroups[i].Count; leafIndex++) - { - var left = leafUrlGroups[i - 1][leafIndex]; - var right = leafUrlGroups[i][leafIndex]; - - try - { - if (AreBothMissing( - left.Url, - right.Url, - urlToJson[left.Url], - urlToJson[right.Url])) - { - continue; - } - } - catch (InvalidOperationException ex) - { - _logger.LogWarning(ex, "A comparison warning was found."); - continue; - } - - var comparisonContext = new ComparisonContext( - id, - hives[i - 1].BaseUrl, - hives[i].BaseUrl, - left.Url, - right.Url, - Normalizers.Leaf); - - _comparer.Compare( - urlToJson[left.Url], - urlToJson[right.Url], - comparisonContext); - } - } - } - - private bool AreBothMissing(string leftUrl, string rightUrl, JObject left, JObject right) - { - if ((left == null) != (right == null)) - { - throw new InvalidOperationException(Environment.NewLine + - $"One of the URLs exists, the other does not." + Environment.NewLine + - $"| Left URL: {leftUrl}" + Environment.NewLine + - $"| Right URL: {rightUrl}" + Environment.NewLine + - $"| Left is 404: {left == null}" + Environment.NewLine + - $"| Right is 404: {right == null}" + Environment.NewLine); - } - - return left == null; - } - - private async Task> GetIndexAsync(HiveConfiguration hive, string id) - { - var url = $"{hive.BaseUrl}{id}/index.json"; - return await GetJObjectOrNullAsync(hive, url); - } - - private async Task> GetJObjectOrNullAsync(HiveConfiguration hive, string url) - { - if (!url.StartsWith(hive.BaseUrl)) - { - throw new ArgumentException("The provided URL must start with the hive base URL."); - } - - var storageUrl = hive.StorageBaseUrl + url.Substring(hive.BaseUrl.Length); - var requestId = Interlocked.Increment(ref _requestId); - _logger.LogInformation("[Request {RequestId}] Fetching {Url}", requestId, storageUrl); - var stopwatch = Stopwatch.StartNew(); - using (var response = await _httpClient.GetAsync(storageUrl, HttpCompletionOption.ResponseContentRead)) - { - _logger.LogInformation( - "[Request {RequestId}] Got {StatusCode} {ReasonPhrase} after {DurationMs}ms", - requestId, - (int)response.StatusCode, - response.ReasonPhrase, - (int)stopwatch.Elapsed.TotalMilliseconds); - - if (response.StatusCode == HttpStatusCode.NotFound) - { - return new DownloadedData(hive, url, storageUrl, null); - } - - response.EnsureSuccessStatusCode(); - - using (var stream = await response.Content.ReadAsStreamAsync()) - using (var streamReader = new StreamReader(stream)) - using (var jsonTextReader = new JsonTextReader(streamReader)) - { - jsonTextReader.DateParseHandling = DateParseHandling.None; - - var data = JObject.Load(jsonTextReader); - return new DownloadedData(hive, url, storageUrl, data); - } - } - } - - private class DownloadedData - { - public DownloadedData(HiveConfiguration hive, string url, string storageUrl, T data) - { - Hive = hive; - Url = url; - StorageUrl = storageUrl; - Data = data; - } - - public HiveConfiguration Hive { get; } - public string Url { get; } - public string StorageUrl { get; } - public T Data { get; } - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/HiveConfiguration.cs b/src/NuGet.Jobs.RegistrationComparer/HiveConfiguration.cs deleted file mode 100644 index 73d33cad1..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/HiveConfiguration.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace NuGet.Jobs.RegistrationComparer -{ - public class HiveConfiguration - { - public string StorageBaseUrl { get; set; } - public string BaseUrl { get; set; } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/HivesConfiguration.cs b/src/NuGet.Jobs.RegistrationComparer/HivesConfiguration.cs deleted file mode 100644 index dd08b657f..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/HivesConfiguration.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace NuGet.Jobs.RegistrationComparer -{ - public class HivesConfiguration - { - public HiveConfiguration Legacy { get; set; } - public HiveConfiguration Gzipped { get; set; } - public HiveConfiguration SemVer2 { get; set; } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/Job.cs b/src/NuGet.Jobs.RegistrationComparer/Job.cs deleted file mode 100644 index bffce9688..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Job.cs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.ComponentModel.Design; -using System.Net; -using System.Threading; -using System.Threading.Tasks; -using Autofac; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; -using NuGet.Protocol; -using NuGet.Services.Configuration; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Services.V3; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class Job : JsonConfigurationJob - { - private const string ConfigurationSectionName = "RegistrationComparer"; - private const string RegistrationComparerMode = "compare"; - private string _mode; - - public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) - { - _mode = jobArgsDictionary.GetOrThrow("mode"); - base.Init(serviceContainer, jobArgsDictionary); - } - - public override async Task Run() - { - ServicePointManager.DefaultConnectionLimit = 64; - ServicePointManager.MaxServicePointIdleTime = 10000; - - switch (_mode) - { - case RegistrationComparerMode: - await _serviceProvider - .GetRequiredService() - .ExecuteAsync(CancellationToken.None); - break; - default: - throw new InvalidOperationException("Unknown mode."); - } - } - - protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) - { - containerBuilder - .Register(c => - { - var options = c.Resolve>(); - return CloudStorageAccount.Parse(options.Value.StorageConnectionString); - }) - .AsSelf(); - - containerBuilder - .Register(c => - { - var options = c.Resolve>(); - return new AzureStorageFactory( - c.Resolve(), - options.Value.StorageContainer, - maxExecutionTime: AzureStorage.DefaultMaxExecutionTime, - serverTimeout: AzureStorage.DefaultServerTimeout, - path: string.Empty, - baseAddress: null, - useServerSideCopy: true, - compressContent: false, - verbose: true, - initializeContainer: false, - throttle: NullThrottle.Instance); - }) - .As(); - } - - protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) - { - services.AddV3(GlobalTelemetryDimensions); - - switch (_mode) - { - case RegistrationComparerMode: - services.AddTransient(); - break; - } - - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); - services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/JsonComparer.cs b/src/NuGet.Jobs.RegistrationComparer/JsonComparer.cs deleted file mode 100644 index b17dc9248..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/JsonComparer.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Linq; -using Newtonsoft.Json.Linq; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class JsonComparer - { - public void Compare(JToken left, JToken right, ComparisonContext context) - { - var normalized = false; - if (left.Type != right.Type) - { - Type leftType = null; - Type rightType = null; - try - { - foreach (var pair in context.Normalizers.ScalarNormalizers) - { - if (pair.Key(left.Path)) - { - leftType = pair.Value(left, true, context)?.GetType(); - rightType = pair.Value(right, false, context)?.GetType(); - normalized = true; - break; - } - } - } - catch - { - // Swallow exceptions encountered during normalization. The comparison will just consider the tokens - // as different, which is correct. - } - - if (!normalized || leftType != rightType) - { - throw new InvalidOperationException(Environment.NewLine + - $"The type of the JSON value is different." + Environment.NewLine + - $"| Left URL: {context.LeftUrl}" + Environment.NewLine + - $"| Right URL: {context.RightUrl}" + Environment.NewLine + - $"| Left path: {left.Path}" + Environment.NewLine + - $"| Right path: {right.Path}" + Environment.NewLine + - $"| Left type: {left.Type}" + Environment.NewLine + - $"| Right type: {right.Type}" + Environment.NewLine); - } - } - - if (!normalized && left.Type == JTokenType.Object) - { - Compare((JObject)left, (JObject)right, context); - } - else if (!normalized && left.Type == JTokenType.Array) - { - Compare((JArray)left, (JArray)right, context); - } - else - { - var leftJson = left.ToString(); - var rightJson = right.ToString(); - if (leftJson != rightJson) - { - var leftString = leftJson; - var rightString = rightJson; - foreach (var pair in context.Normalizers.ScalarNormalizers) - { - if (pair.Key(left.Path)) - { - leftString = pair.Value(left, true, context) ?? leftJson; - rightString = pair.Value(right, false, context) ?? rightJson; - break; - } - } - - if (leftString != rightString) - { - throw new InvalidOperationException(Environment.NewLine + - $"The value of the JSON scalar is different." + Environment.NewLine + - $"| Left URL: {context.LeftUrl}" + Environment.NewLine + - $"| Right URL: {context.RightUrl}" + Environment.NewLine + - $"| Left path: {left.Path}" + Environment.NewLine + - $"| Right path: {right.Path}" + Environment.NewLine + - $"| Left value: {leftJson}" + Environment.NewLine + - $"| Right value: {rightJson}" + Environment.NewLine); - } - } - } - } - - private void Compare(JArray left, JArray right, ComparisonContext context) - { - if (left.Count != right.Count) - { - throw new InvalidOperationException(Environment.NewLine + - $"The JSON array item count is different." + Environment.NewLine + - $"| Left URL: {context.LeftUrl}" + Environment.NewLine + - $"| Right URL: {context.RightUrl}" + Environment.NewLine + - $"| Left path: {left.Path}" + Environment.NewLine + - $"| Right path: {right.Path}" + Environment.NewLine + - $"| Left count: {left.Count}" + Environment.NewLine + - $"| Right count: {right.Count}" + Environment.NewLine); - } - - var leftItems = left.ToList(); - var rightItems = right.ToList(); - if (leftItems.Count > 1) - { - foreach (var pair in context.Normalizers.UnsortedArrays) - { - if (pair.Key(left)) - { - leftItems.Sort(pair.Value); - rightItems.Sort(pair.Value); - break; - } - } - } - - for (var i = 0; i < leftItems.Count; i++) - { - Compare(leftItems[i], rightItems[i], context); - } - } - - private void Compare(JObject left, JObject right, ComparisonContext context) - { - var leftPropertyNames = left.Properties().Select(x => x.Name); - var rightPropertyNames = right.Properties().Select(x => x.Name); - - var onlyLeft = leftPropertyNames.Except(rightPropertyNames); - var onlyRight = rightPropertyNames.Except(leftPropertyNames); - if (onlyLeft.Any() || onlyRight.Any()) - { - throw new InvalidOperationException(Environment.NewLine + - $"The JSON object property names are disjoint." + Environment.NewLine + - $"| Left URL: {context.LeftUrl}" + Environment.NewLine + - $"| Right URL: {context.RightUrl}" + Environment.NewLine + - $"| Left path: {left.Path}" + Environment.NewLine + - $"| Right path: {right.Path}" + Environment.NewLine + - $"| Only left: {string.Join(", ", onlyLeft)}" + Environment.NewLine + - $"| Only right: {string.Join(", ", onlyRight)}" + Environment.NewLine); - } - - var leftProperties = left.Properties(); - var rightProperties = right.Properties(); - - if (!leftPropertyNames.SequenceEqual(rightPropertyNames)) - { - if (context.Normalizers.UnsortedObjects.Any(x => x(left.Path))) - { - leftProperties = leftProperties.OrderBy(x => x.Name); - rightProperties = rightProperties.OrderBy(x => x.Name); - } - else - { - throw new InvalidOperationException(Environment.NewLine + - $"The JSON object property names are in a different order." + Environment.NewLine + - $"| Left URL: {context.LeftUrl}" + Environment.NewLine + - $"| Right URL: {context.RightUrl}" + Environment.NewLine + - $"| Left path: {left.Path}" + Environment.NewLine + - $"| Right path: {right.Path}" + Environment.NewLine + - $"| Left order: {string.Join(", ", leftPropertyNames)}" + Environment.NewLine + - $"| Right order: {string.Join(", ", rightPropertyNames)}" + Environment.NewLine); - } - } - - var pairs = leftProperties.Zip(rightProperties, (l, r) => new { Left = l.Value, Right = r.Value }); - foreach (var pair in pairs) - { - Compare(pair.Left, pair.Right, context); - } - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/Normalizers.cs b/src/NuGet.Jobs.RegistrationComparer/Normalizers.cs deleted file mode 100644 index 6c5dbe25a..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Normalizers.cs +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using Newtonsoft.Json.Linq; -using NuGet.Versioning; -using ValueNormalizer = System.Collections.Generic.KeyValuePair; -using ArrayNormalizer = System.Collections.Generic.KeyValuePair>; - -namespace NuGet.Jobs.RegistrationComparer -{ - public delegate bool ShouldNormalizeByPath(string jsonPath); - public delegate bool ShouldNormalizeByArray(JArray array); - public delegate string NormalizeToken(JToken value, bool isLeft, ComparisonContext context); - - public class Normalizers - { - private readonly static HashSet PackageIdsWithDotDotDependencies = new HashSet(StringComparer.OrdinalIgnoreCase) - { - "angularjs.TypeScript.DefinitelyTyped", - "devextreme.TypeScript.DefinitelyTyped", - }; - - private static readonly IReadOnlyList DefaultScalarNormalizers = new List - { - new ValueNormalizer( - path => IsPropertyName(path, "@id") || IsPropertyName(path, "registration") || IsPropertyName(path, "parent"), - (value, isLeft, context) => - { - var url = (string)value; - var baseUrl = isLeft ? context.LeftBaseUrl : context.RightBaseUrl; - - // Ignore a quirky dependency package IDs: - // - "../jquery.TypeScript.DefinitelyTyped" - // https://api.nuget.org/v3/catalog0/data/2018.10.18.22.40.48/angularjs.typescript.definitelytyped.0.6.6.json - // - "../jquery.TypeScript.DefinitelyTyped" - // https://api.nuget.org/v3/catalog0/data/2018.12.15.06.50.14/devextreme.typescript.definitelytyped.0.0.4.json - if (PackageIdsWithDotDotDependencies.Contains(context.PackageId) - && IsPropertyName(value.Path, "registration") - && url.Contains("../")) - { - var otherBaseUrl = isLeft ? context.RightBaseUrl : context.LeftBaseUrl; - url = TrySetBaseUrl(url, baseUrl, otherBaseUrl); - url = new Uri(url).AbsoluteUri; - } - - url = TrySetBaseUrl(url, baseUrl, "{base URL}/"); - - return url; - }), - }; - - private static string TrySetBaseUrl(string url, string currentBaseUrl, string newBaseUrl) - { - if (url.StartsWith(currentBaseUrl)) - { - url = newBaseUrl + url.Substring(currentBaseUrl.Length); - } - - return url; - } - - private static readonly IReadOnlyList IndexAndPageScalarNormalizers = new List(DefaultScalarNormalizers) - { - new ValueNormalizer( - path => IsPropertyName(path, "commitId"), - (value, isLeft, context) => - { - // Each iteration, the writer will come up with a different commit ID. - var commitId = (string)value; - if (commitId != null - && Regex.IsMatch(commitId, "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")) - { - return Guid.Empty.ToString(); - } - - return commitId; - }), - new ValueNormalizer( - path => IsPropertyName(path, "commitTimeStamp"), - (value, isLeft, context) => - { - // Each iteration, the writer will come up with a different commit timestamp. - var commitTimestamp = (string)value; - if (commitTimestamp != null - && Regex.IsMatch(commitTimestamp, @"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,})?(Z|\+00:00)")) - { - return DateTimeOffset.MinValue.ToString("o"); - } - - return commitTimestamp; - }), - new ValueNormalizer( - path => IsPropertyName(path, "summary"), - (value, isLeft, context) => - { - // The NuGet.Protocol.Catalog library deserializes properties in a case-insensitive way. - // This is due to the following Newtonsoft.Json issue: - // https://github.com/JamesNK/Newtonsoft.Json/issues/815 - - // This package has "Summary" instead of "summary" - // https://api.nuget.org/v3/catalog0/data/2019.10.02.07.14.49/staticproxy.fody.1.0.146-genericmethods.json - if (context.PackageId.Equals("StaticProxy.Fody", StringComparison.OrdinalIgnoreCase) - && (string)value.Parent.Parent["version"] == "1.0.146-GenericMethods") - { - return ""; - } - - return (string)value; - }), - new ValueNormalizer( - path => IsPropertyName(path, "licenseUrl"), - (value, isLeft, context) => - { - // This package has "licenseurl" instead of "licenseUrl" - // https://api.nuget.org/v3/catalog0/data/2019.01.02.03.56.37/boolli.1.0.0.json - if (context.PackageId.Equals("Boolli", StringComparison.OrdinalIgnoreCase) - && (string)value.Parent.Parent["version"] == "1.0.0") - { - return ""; - } - - return (string)value; - }), - new ValueNormalizer( - path => IsPropertyName(path, "projectUrl"), - (value, isLeft, context) => - { - // This package has "ProjectUrl" instead of "projectUrl" - // https://api.nuget.org/v3/catalog0/data/2019.01.02.13.49.28/blogml.core.1.0.0.json - if (context.PackageId.Equals("BlogML.Core", StringComparison.OrdinalIgnoreCase) - && (string)value.Parent.Parent["version"] == "1.0.0") - { - return ""; - } - - return (string)value; - }), - new ValueNormalizer( - path => IsPropertyName(path, "iconUrl"), - (value, isLeft, context) => - { - // This package has "iconURL" instead of "iconUrl" - // https://api.nuget.org/v3/catalog0/data/2019.04.18.10.29.30/confuser.msbuild.2.0.0-alpha-0191.json - if (context.PackageId.Equals("Confuser.MSBuild", StringComparison.OrdinalIgnoreCase) - && (string)value.Parent.Parent["version"] == "2.0.0-alpha-0191") - { - return ""; - } - - return (string)value; - }), - new ValueNormalizer( - path => IsPropertyName(path, "language"), - (value, isLeft, context) => - { - // This package has "Language" instead of "language" - // https://api.nuget.org/v3/catalog0/data/2018.12.19.07.07.41/smartseeder.0.0.1.json - // https://api.nuget.org/v3/catalog0/data/2018.12.19.07.07.41/smartseeder.1.0.0-rc1-beta.json - // https://api.nuget.org/v3/catalog0/data/2018.12.19.07.07.31/smartseeder.1.0.0-rc1-preview.json - if (context.PackageId.Equals("SmartSeeder", StringComparison.OrdinalIgnoreCase) - && new[] { "0.0.1", "1.0.0-rc1-beta", "1.0.0-rc1-preview" }.Contains((string)value.Parent.Parent["version"])) - { - return ""; - } - - return (string)value; - }), - new ValueNormalizer( - path => IsPropertyName(path, "range"), - (value, isLeft, context) => - { - // This package has duplicate dependencies, meaning the dependency range is an array instead of string. - // https://api.nuget.org/v3/catalog0/data/2019.11.26.10.59.56/paket.core.5.237.1.json - // https://api.nuget.org/v3/catalog0/data/2019.11.26.11.13.24/paket.core.5.237.2.json - if (context.PackageId.Equals("Paket.Core", StringComparison.OrdinalIgnoreCase) - && new[] { "5.237.1", "5.237.2" }.Contains((string)value.Parent.Parent.Parent.Parent.Parent.Parent.Parent.Parent["version"]) - && value.Type == JTokenType.Array) - { - return (string)value.OrderBy(x => x).First(); - } - - // This package has duplicate dependencies, meaning the dependency range is an array instead of string. - // https://api.nuget.org/v3/catalog0/data/2016.02.21.10.24.50/dingu.generic.repo.ef7.1.0.0.json - // https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json - if (context.PackageId.Equals("Dingu.Generic.Repo.EF7", StringComparison.OrdinalIgnoreCase) - && new[] { "1.0.0", "1.0.0-beta2" }.Contains((string)value.Parent.Parent.Parent.Parent.Parent.Parent.Parent.Parent["version"]) - && value.Type == JTokenType.Array) - { - return (string)value.OrderBy(x => x).First(); - } - - return (string)value; - }), - }; - - private static readonly IReadOnlyList DefaultUnsortedObjects = new List - { - path => path.StartsWith("@context."), - }; - - private static readonly IReadOnlyList PageUnsortedObjects = new List(DefaultUnsortedObjects) - { - path => path == string.Empty, - }; - - private static readonly IReadOnlyList DefaultUnsortedArrays = new List - { - new ArrayNormalizer( - a => IsPropertyName(a.Path, "@type"), - (a, b) => StringComparer.Ordinal.Compare((string)a, (string)b)), - }; - - private static readonly IReadOnlyList IndexAndPageUnsortedArrays = new List(DefaultUnsortedArrays) - { - new ArrayNormalizer( - a => IsPropertyName(a.Path, "dependencies"), - (a, b) => StringComparer.OrdinalIgnoreCase.Compare((string)a["id"], (string)b["id"])), - new ArrayNormalizer( - a => IsPropertyName(a.Path, "tags"), - (a, b) => StringComparer.Ordinal.Compare((string)a, (string)b)), - new ArrayNormalizer( - a => IsPropertyName(a.Path, "reasons"), - (a, b) => StringComparer.Ordinal.Compare((string)a, (string)b)), - new ArrayNormalizer( - a => IsPropertyName(a.Path, "dependencyGroups"), - (a, b) => StringComparer.Ordinal.Compare((string)a["targetFramework"], (string)b["targetFramework"])), - new ArrayNormalizer( - a => IsPropertyName(a.Path, "items") - && a.Parent?.Parent != null - && a.Parent.Parent["@type"]?.Type == JTokenType.String - && (string)a.Parent.Parent["@type"] == "catalog:CatalogPage", - (a, b) => NuGetVersion.Parse((string)a["catalogEntry"]["version"]).CompareTo(NuGetVersion.Parse((string)b["catalogEntry"]["version"]))) - }; - - private static bool IsPropertyName(string path, string name) - { - return path == name || path.EndsWith("." + name); - } - - public Normalizers( - IReadOnlyList scalarNormalizers, - IReadOnlyList unsortedObjects, - IReadOnlyList unsortedArrays) - { - ScalarNormalizers = scalarNormalizers; - UnsortedObjects = unsortedObjects; - UnsortedArrays = unsortedArrays; - } - - public static readonly Normalizers Index = new Normalizers( - IndexAndPageScalarNormalizers, - DefaultUnsortedObjects, - IndexAndPageUnsortedArrays); - - public static readonly Normalizers Page = new Normalizers( - IndexAndPageScalarNormalizers, - PageUnsortedObjects, - IndexAndPageUnsortedArrays); - - public static readonly Normalizers Leaf = new Normalizers( - DefaultScalarNormalizers, - DefaultUnsortedObjects, - DefaultUnsortedArrays); - - public IReadOnlyList ScalarNormalizers { get; } - public IReadOnlyList UnsortedObjects { get; } - public IReadOnlyList UnsortedArrays { get; } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/NuGet.Jobs.RegistrationComparer.csproj b/src/NuGet.Jobs.RegistrationComparer/NuGet.Jobs.RegistrationComparer.csproj deleted file mode 100644 index 15c778479..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/NuGet.Jobs.RegistrationComparer.csproj +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - Debug - AnyCPU - {4CE6C864-DB4D-4262-A2DD-80BB932F6E8C} - Exe - NuGet.Jobs.RegistrationComparer - NuGet.Jobs.RegistrationComparer - v4.7.2 - 512 - true - true - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - false - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0.3.0 - runtime; build; native; contentfiles; analyzers - all - - - - - {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} - NuGet.Services.Metadata.Catalog - - - {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} - NuGet.Protocol.Catalog - - - {c3f9a738-9759-4b2b-a50d-6507b28a659b} - NuGet.Services.V3 - - - - - - - - ..\..\build - $(BUILD_SOURCESDIRECTORY)\build - $(NuGetBuildPath) - none - - - - \ No newline at end of file diff --git a/src/NuGet.Jobs.RegistrationComparer/NuGet.Jobs.RegistrationComparer.nuspec b/src/NuGet.Jobs.RegistrationComparer/NuGet.Jobs.RegistrationComparer.nuspec deleted file mode 100644 index 0503a06e4..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/NuGet.Jobs.RegistrationComparer.nuspec +++ /dev/null @@ -1,16 +0,0 @@ - - - - RegistrationComparer - $version$ - .NET Foundation - .NET Foundation - RegistrationComparer - Copyright .NET Foundation - - - - - - - \ No newline at end of file diff --git a/src/NuGet.Jobs.RegistrationComparer/Program.cs b/src/NuGet.Jobs.RegistrationComparer/Program.cs deleted file mode 100644 index eba0ec2af..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Program.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace NuGet.Jobs.RegistrationComparer -{ - class Program - { - static int Main(string[] args) - { - var job = new Job(); - return JobRunner.Run(job, args).GetAwaiter().GetResult(); - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/Properties/AssemblyInfo.cs b/src/NuGet.Jobs.RegistrationComparer/Properties/AssemblyInfo.cs deleted file mode 100644 index b10e9f388..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Reflection; -using System.Runtime.InteropServices; - -[assembly: AssemblyTitle("NuGet.Jobs.RegistrationComparer")] -[assembly: ComVisible(false)] -[assembly: Guid("4ce6c864-db4d-4262-a2dd-80bb932f6e8c")] diff --git a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs deleted file mode 100644 index ff03bae2a..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCollectorLogic.cs +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Helpers; -using NuGet.Services.V3; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class RegistrationComparerCollectorLogic : ICommitCollectorLogic - { - private readonly CommitCollectorUtility _utility; - private readonly HiveComparer _comparer; - private readonly IOptionsSnapshot _options; - private readonly ILogger _logger; - - public RegistrationComparerCollectorLogic( - CommitCollectorUtility utility, - HiveComparer comparer, - IOptionsSnapshot options, - ILogger logger) - { - _utility = utility ?? throw new ArgumentNullException(nameof(utility)); - _comparer = comparer ?? throw new ArgumentNullException(nameof(comparer)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public Task> CreateBatchesAsync(IEnumerable catalogItems) - { - return Task.FromResult(_utility.CreateSingleBatch(catalogItems)); - } - - public async Task OnProcessBatchAsync(IEnumerable items) - { - var packageIdGroups = items - .GroupBy(x => x.PackageIdentity.Id, StringComparer.OrdinalIgnoreCase) - .Select(g => new - { - Id = g.Key.ToLowerInvariant(), - Versions = g - .Select(x => x.PackageIdentity.Version) - .Distinct() - .OrderBy(x => x) - .Select(x => x.ToNormalizedString().ToLowerInvariant()) - .ToList(), - }) - .ToList(); - - _logger.LogInformation("Comparing {Count} package IDs.", packageIdGroups.Count); - - var hiveGroups = _options - .Value - .Registrations - .SelectMany(x => new[] - { - new - { - Hive = nameof(x.Legacy), - BaseUrl = x.Legacy.BaseUrl.TrimEnd('/') + '/', - StorageBaseUrl = x.Legacy.StorageBaseUrl.TrimEnd('/') + '/', - }, - new - { - Hive = nameof(x.Gzipped), - BaseUrl = x.Gzipped.BaseUrl.TrimEnd('/') + '/', - StorageBaseUrl = x.Gzipped.StorageBaseUrl.TrimEnd('/') + '/', - }, - new - { - Hive = nameof(x.SemVer2), - BaseUrl = x.SemVer2.BaseUrl.TrimEnd('/') + '/', - StorageBaseUrl = x.SemVer2.StorageBaseUrl.TrimEnd('/') + '/' - }, - }) - .GroupBy(x => x.Hive, x => new HiveConfiguration { BaseUrl = x.BaseUrl, StorageBaseUrl = x.StorageBaseUrl }); - - var allWork = new ConcurrentBag>(); - var failures = 0; - foreach (var group in packageIdGroups) - { - foreach (var hiveGroup in hiveGroups) - { - var hives = hiveGroup.ToList(); - var hive = hiveGroup.Key; - var id = group.Id; - var versions = group.Versions; - allWork.Add(async () => - { - _logger.LogInformation("Verifying hive {Hive} for {PackageId}.", hive, id); - try - { - await _comparer.CompareAsync(hives, id, versions); - } - catch (Exception ex) - { - Interlocked.Increment(ref failures); - _logger.LogError(ex, "The comparison failed."); - } - }); - } - } - - await ParallelAsync - .Repeat(async () => - { - await Task.Yield(); - while (allWork.TryTake(out var work)) - { - await work(); - } - }, - degreeOfParallelism: _options.Value.MaxConcurrentComparisons); - - if (failures > 0) - { - throw new InvalidOperationException($"{failures} hives failed the comparison."); - } - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCommand.cs b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCommand.cs deleted file mode 100644 index b16b11f87..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerCommand.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Persistence; -using NuGet.Services.V3; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class RegistrationComparerCommand - { - private readonly ICollector _collector; - private readonly CloudStorageAccount _storageAccount; - private readonly IStorageFactory _storageFactory; - private readonly Func _handlerFunc; - private readonly IOptionsSnapshot _options; - private readonly ILogger _logger; - - public RegistrationComparerCommand( - ICollector collector, - CloudStorageAccount storageAccount, - IStorageFactory storageFactory, - Func handlerFunc, - IOptionsSnapshot options, - ILogger logger) - { - _collector = collector ?? throw new ArgumentNullException(nameof(collector)); - _storageAccount = storageAccount ?? throw new ArgumentNullException(nameof(storageAccount)); - _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); - _handlerFunc = handlerFunc ?? throw new ArgumentNullException(nameof(handlerFunc)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task ExecuteAsync(CancellationToken token) - { - var registrationCursors = CursorUtility.GetRegistrationCursors(_handlerFunc, _options); - var backCursor = new AggregateCursor(registrationCursors.Values); - - var frontCursorPair = CursorUtility.GetComparerCursor(_storageFactory); - _logger.LogInformation("Using cursor: {CursurUrl}", frontCursorPair.Key); - var frontCursor = frontCursorPair.Value; - - var container = _storageAccount - .CreateCloudBlobClient() - .GetContainerReference(_options.Value.StorageContainer); - await container.CreateIfNotExistsAsync(); - await container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); - - await frontCursor.LoadAsync(token); - await backCursor.LoadAsync(token); - _logger.LogInformation( - "The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.", - frontCursor.Value, - backCursor.Value); - - // Run the collector. - await _collector.RunAsync( - frontCursor, - backCursor, - token); - } - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs b/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs deleted file mode 100644 index 8eda1d562..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/RegistrationComparerConfiguration.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using NuGet.Services.V3; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class RegistrationComparerConfiguration : ICommitCollectorConfiguration - { - public string StorageConnectionString { get; set; } - public string StorageContainer { get; set; } - public List Registrations { get; set; } - public string Source { get; set; } - public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64; - public int MaxConcurrentComparisons { get; set; } = 32; - public int MaxConcurrentPageAndLeafDownloadsPerId { get; set; } = 32; - public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10); - } -} diff --git a/src/NuGet.Jobs.RegistrationComparer/Scripts/Functions.ps1 b/src/NuGet.Jobs.RegistrationComparer/Scripts/Functions.ps1 deleted file mode 100644 index a8bff40fc..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Scripts/Functions.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -Function Uninstall-NuGetService() { - Param ([string]$ServiceName) - - if (Get-Service $ServiceName -ErrorAction SilentlyContinue) - { - Write-Host Removing service $ServiceName... - Stop-Service $ServiceName -Force - sc.exe delete $ServiceName - Write-Host Removed service $ServiceName. - } else { - Write-Host Skipping removal of service $ServiceName - no such service exists. - } -} - -Function Install-NuGetService() { - Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) - - Write-Host Installing service $ServiceName... - - $installService = "nssm install $ServiceName $ScriptToRun" - cmd /C $installService - - Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic - sc.exe failure $ServiceName reset= 30 actions= restart/5000 - - # Run service - net start $ServiceName - - Write-Host Installed service $ServiceName. -} \ No newline at end of file diff --git a/src/NuGet.Jobs.RegistrationComparer/Scripts/PostDeploy.ps1 b/src/NuGet.Jobs.RegistrationComparer/Scripts/PostDeploy.ps1 deleted file mode 100644 index 7d5183d5b..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Scripts/PostDeploy.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -. .\Functions.ps1 - -$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") - -Write-Host Installing services... - -$currentDirectory = [string](Get-Location) - -$jobsToInstall.Split("{;}") | %{ - $serviceName = $_ - $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] - $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] - $scriptToRun = "$currentDirectory\$scriptToRun" - - Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun -} - -Write-Host Installed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.RegistrationComparer/Scripts/PreDeploy.ps1 b/src/NuGet.Jobs.RegistrationComparer/Scripts/PreDeploy.ps1 deleted file mode 100644 index ef711a912..000000000 --- a/src/NuGet.Jobs.RegistrationComparer/Scripts/PreDeploy.ps1 +++ /dev/null @@ -1,11 +0,0 @@ -. .\Functions.ps1 - -$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") - -Write-Host Removing services... - -$jobsToInstall.Split("{;}") | %{ - Uninstall-NuGetService -ServiceName $_ -} - -Write-Host Removed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.RegistrationComparer/Scripts/nssm.exe b/src/NuGet.Jobs.RegistrationComparer/Scripts/nssm.exe deleted file mode 100644 index 6ccfe3cfb..000000000 Binary files a/src/NuGet.Jobs.RegistrationComparer/Scripts/nssm.exe and /dev/null differ diff --git a/test.ps1 b/test.ps1 index 3fcd57755..c255783c8 100644 --- a/test.ps1 +++ b/test.ps1 @@ -31,8 +31,7 @@ Function Run-Tests { "tests\NuGet.Protocol.Catalog.Tests\bin\$Configuration\NuGet.Protocol.Catalog.Tests.dll", ` "tests\NuGet.Services.AzureSearch.Tests\bin\$Configuration\NuGet.Services.AzureSearch.Tests.dll", ` "tests\NuGet.Services.SearchService.Tests\bin\$Configuration\NuGet.Services.SearchService.Tests.dll", ` - "tests\NuGet.Jobs.Catalog2Registration.Tests\bin\$Configuration\NuGet.Jobs.Catalog2Registration.Tests.dll", ` - "tests\NuGet.Jobs.RegistrationComparer.Tests\bin\$Configuration\NuGet.Jobs.RegistrationComparer.Tests.dll" + "tests\NuGet.Jobs.Catalog2Registration.Tests\bin\$Configuration\NuGet.Jobs.Catalog2Registration.Tests.dll" $TestCount = 0 diff --git a/tests/NuGet.Jobs.RegistrationComparer.Tests/App.config b/tests/NuGet.Jobs.RegistrationComparer.Tests/App.config deleted file mode 100644 index 2a2d44990..000000000 --- a/tests/NuGet.Jobs.RegistrationComparer.Tests/App.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/tests/NuGet.Jobs.RegistrationComparer.Tests/JsonComparerFacts.cs b/tests/NuGet.Jobs.RegistrationComparer.Tests/JsonComparerFacts.cs deleted file mode 100644 index fc9096eb9..000000000 --- a/tests/NuGet.Jobs.RegistrationComparer.Tests/JsonComparerFacts.cs +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Xunit; -using ArrayNormalizer = System.Collections.Generic.KeyValuePair>; -using ValueNormalizer = System.Collections.Generic.KeyValuePair; - -namespace NuGet.Jobs.RegistrationComparer -{ - public class JsonComparerFacts - { - [Fact] - public void AcceptsSameObject() - { - var a = Json(new { array = new[] { 0, 1, 3 } }); - var b = Json(new { array = new[] { 0, 1, 3 } }); - - Target.Compare(a, b, Context); - - Assert.NotSame(a, b); - } - - [Fact] - public void DetectsMissingItemInArray() - { - var a = Json(new { array = new[] { 0, 1, 3 } }); - var b = Json(new { array = new[] { 0, 1, 2, 3 } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The JSON array item count is different.", ex.Message); - } - - [Fact] - public void DetectsDifferentItemsInArray() - { - var a = Json(new { array = new[] { 0, 1, 3 } }); - var b = Json(new { array = new[] { 0, 1, 2 } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The value of the JSON scalar is different.", ex.Message); - } - - [Fact] - public void DetectsOutOfOrderItemsInArray() - { - var a = Json(new { array = new[] { 0, 2, 1 } }); - var b = Json(new { array = new[] { 0, 1, 2 } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The value of the JSON scalar is different.", ex.Message); - } - - [Theory] - [InlineData("2", 2)] - [InlineData(true, 1)] - [InlineData("false", false)] - [InlineData("null", null)] - [InlineData(0, null)] - public void DetectsDifferentTypesInArray(object valueA, object valueB) - { - var a = Json(new { array = new object[] { 0, 1, valueA } }); - var b = Json(new { array = new object[] { 0, 1, valueB } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The type of the JSON value is different.", ex.Message); - } - - [Fact] - public void DetectsDifferentProperties() - { - var a = Json(new { arrayA = new[] { 0, 1, 2 } }); - var b = Json(new { arrayB = new[] { 0, 1, 2 } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The JSON object property names are disjoint.", ex.Message); - } - - [Fact] - public void DetectsOutOfOrderProperties() - { - var a = Json(new { inner = new { a = "a", b = "b" } }); - var b = Json(new { inner = new { b = "b", a = "a" } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The JSON object property names are in a different order.", ex.Message); - } - - [Fact] - public void DetectsDifferentCaseOfPropertyNames() - { - var a = Json(new { array = new[] { 0, 1, 2 } }); - var b = Json(new { Array = new[] { 0, 1, 2 } }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The JSON object property names are disjoint.", ex.Message); - } - - [Fact] - public void DetectsExtraProperty() - { - var a = Json(new { array = new[] { 0, 1, 2 } }); - var b = Json(new { array = new[] { 0, 1, 2 }, somethingElse = 2 }); - - var ex = Assert.Throws(() => Target.Compare(a, b, Context)); - - Assert.Contains("The JSON object property names are disjoint.", ex.Message); - } - - [Fact] - public void AllowsValueToBeNormalized() - { - var normalizers = new Normalizers( - scalarNormalizers: new List - { - new ValueNormalizer( - (path) => path == "random", - (token, isLeft, context) => "999"), - }, - unsortedObjects: new List(), - unsortedArrays: new List()); - var a = Json(new { array = new[] { 0, 1, 2 }, random = 23 }); - var b = Json(new { array = new[] { 0, 1, 2 }, random = 42 }); - - Target.Compare(a, b, GetContext(normalizers)); - } - - [Fact] - public void AllowsObjectPropertyOrderToBeIgnored() - { - var normalizers = new Normalizers( - scalarNormalizers: new List(), - unsortedObjects: new List - { - (path) => path == "inner", - }, - unsortedArrays: new List()); - var a = Json(new { inner = new { a = "a", b = "b" } }); - var b = Json(new { inner = new { b = "b", a = "a" } }); - - Target.Compare(a, b, GetContext(normalizers)); - } - - [Fact] - public void AllowsArrayItemOrderToBeIgnored() - { - var normalizers = new Normalizers( - scalarNormalizers: new List(), - unsortedObjects: new List(), - unsortedArrays: new List - { - new ArrayNormalizer( - array => array.Path == "array", - (x, y) => Comparer.Default.Compare(x, y)), - }); - var a = Json(new { array = new[] { 0, 2, 1 } }); - var b = Json(new { array = new[] { 0, 1, 2 } }); - - Target.Compare(a, b, GetContext(normalizers)); - } - - [Fact] - public void AllowsPropertyNameWithAtToBeNormalized() - { - var normalizers = new Normalizers( - scalarNormalizers: new List(), - unsortedObjects: new List(), - unsortedArrays: new List - { - new ArrayNormalizer( - array => array.Path == "@type", - (x, y) => StringComparer.Ordinal.Compare((string)x, (string)y)), - }); - var a = Json(new Dictionary { { "@type", new[] { "foo", "bar" } } }); - var b = Json(new Dictionary { { "@type", new[] { "bar", "foo" } } }); - - Target.Compare(a, b, GetContext(normalizers)); - } - - [Fact] - public async Task NormalizesKnownUrl() - { - var leftBaseUrl = "https://api.nuget.org/v3/registration4-gz-semver2/"; - var rightBaseUrl = "https://api.nuget.org/v3/registration5-gz-semver2/"; - var packageId = "BaseTestPackage.SearchFilters"; - var relativePath = "/index.json"; - var context = new ComparisonContext( - packageId, - leftBaseUrl, - rightBaseUrl, - $"{leftBaseUrl}{packageId.ToLowerInvariant()}{relativePath}", - $"{rightBaseUrl}{packageId.ToLowerInvariant()}{relativePath}", - Normalizers.Index); - var a = await DownloadAsync(context.LeftUrl); - var b = await DownloadAsync(context.RightUrl); - - Target.Compare(a, b, context); - } - - public JsonComparerFacts() - { - Context = GetContext(); - Target = new JsonComparer(); - } - - private ComparisonContext GetContext(Normalizers normalizers) - { - return new ComparisonContext( - "NuGet.Versioning", - "https://example/api/a", - "https://example/api/b", - "https://example/api/a/index.json", - "https://example/api/b/index.json", - normalizers); - } - - private ComparisonContext GetContext() - { - return GetContext( - new Normalizers( - new List(), - new List(), - new List())); - } - - public ComparisonContext Context { get; } - public JsonComparer Target { get; } - - private JToken Json(T obj) - { - return JsonConvert.DeserializeObject(JsonConvert.SerializeObject(obj)); - } - - private async Task DownloadAsync(string url) - { - using (var httpClientHandler = new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip }) - using (var httpClient = new HttpClient(httpClientHandler)) - using (var stream = await httpClient.GetStreamAsync(url)) - using (var streamReader = new StreamReader(stream)) - using (var jsonTextReader = new JsonTextReader(streamReader)) - { - jsonTextReader.DateParseHandling = DateParseHandling.None; - - return await JObject.LoadAsync(jsonTextReader); - } - } - } -} diff --git a/tests/NuGet.Jobs.RegistrationComparer.Tests/NuGet.Jobs.RegistrationComparer.Tests.csproj b/tests/NuGet.Jobs.RegistrationComparer.Tests/NuGet.Jobs.RegistrationComparer.Tests.csproj deleted file mode 100644 index 5ae96dfba..000000000 --- a/tests/NuGet.Jobs.RegistrationComparer.Tests/NuGet.Jobs.RegistrationComparer.Tests.csproj +++ /dev/null @@ -1,79 +0,0 @@ - - - - - Debug - AnyCPU - {A0E0698A-1161-4DEA-81A9-06D30FB16538} - Library - Properties - NuGet.Jobs.RegistrationComparer - NuGet.Jobs.RegistrationComparer.Tests - v4.7.2 - 512 - true - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - - - - - 4.10.1 - - - 2.4.1 - - - 2.4.1 - runtime; build; native; contentfiles; analyzers - all - - - - - - - - - - - - {4ce6c864-db4d-4262-a2dd-80bb932f6e8c} - NuGet.Jobs.RegistrationComparer - - - - - ..\..\build - $(BUILD_SOURCESDIRECTORY)\build - $(NuGetBuildPath) - - - \ No newline at end of file diff --git a/tests/NuGet.Jobs.RegistrationComparer.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Jobs.RegistrationComparer.Tests/Properties/AssemblyInfo.cs deleted file mode 100644 index 32478c614..000000000 --- a/tests/NuGet.Jobs.RegistrationComparer.Tests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Reflection; -using System.Runtime.InteropServices; - -[assembly: AssemblyTitle("NuGet.Jobs.RegistrationComparer.Tests")] -[assembly: ComVisible(false)] -[assembly: Guid("a83b83d0-4f95-4e1e-bb46-8c4ce547bd67")]