From f981490bb36aa0d7b5f49a8156e55962b2f34dff Mon Sep 17 00:00:00 2001 From: Xavier Decoster Date: Wed, 12 Jun 2019 10:44:54 +0200 Subject: [PATCH] #2504 Remove package2catalog job and all code only used by this job (#551) --- src/Catalog/Helpers/FeedHelpers.cs | 28 ---- src/Ng/Jobs/Package2CatalogJob.cs | 141 ------------------ src/Ng/Ng.csproj | 1 - src/Ng/NgJobFactory.cs | 1 - .../CatalogTests/Helpers/FeedHelpersTests.cs | 63 -------- tests/NgTests/NgTests.csproj | 1 - tests/NgTests/Package2CatalogJobTests.cs | 120 --------------- 7 files changed, 355 deletions(-) delete mode 100644 src/Ng/Jobs/Package2CatalogJob.cs delete mode 100644 tests/NgTests/Package2CatalogJobTests.cs diff --git a/src/Catalog/Helpers/FeedHelpers.cs b/src/Catalog/Helpers/FeedHelpers.cs index aa351b373..62a48abfd 100644 --- a/src/Catalog/Helpers/FeedHelpers.cs +++ b/src/Catalog/Helpers/FeedHelpers.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net.Http; using System.Threading.Tasks; using System.Web; @@ -34,33 +33,6 @@ public static Uri MakeUriForPackage(string source, string id, string version) return UriUtils.GetNonhijackableUri(uri); } - /// - /// Returns a from the feed. - /// - /// The field to sort the on. - public static async Task>> GetPackagesInOrder(HttpClient client, Uri uri, Func keyDateFunc) - { - var result = new SortedList>(); - - var allPackages = await GetPackages(client, uri); - - foreach (var package in allPackages) - { - IList packagesWithSameKeyDate; - - var packageKeyDate = keyDateFunc(package); - if (!result.TryGetValue(packageKeyDate, out packagesWithSameKeyDate)) - { - packagesWithSameKeyDate = new List(); - result.Add(packageKeyDate, packagesWithSameKeyDate); - } - - packagesWithSameKeyDate.Add(package); - } - - return result; - } - /// /// Asynchronously gets a from the feed. /// diff --git a/src/Ng/Jobs/Package2CatalogJob.cs b/src/Ng/Jobs/Package2CatalogJob.cs deleted file mode 100644 index 08a7b81eb..000000000 --- a/src/Ng/Jobs/Package2CatalogJob.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using NuGet.Services.Configuration; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Helpers; -using NuGet.Services.Metadata.Catalog.Persistence; - -namespace Ng.Jobs -{ - public class Package2CatalogJob : NgJob - { - private string _gallery; - private bool _verbose; - private string _id; - private string _version; - private IStorage _storage; - - public Package2CatalogJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) - : base(telemetryService, loggerFactory) - { - } - - protected Package2CatalogJob( - ITelemetryService telemetryService, - ILoggerFactory loggerFactory, - IStorage storage, - string gallery, - string packageId, - string packageVersion, - bool verbose) - : this(telemetryService, loggerFactory) - { - _storage = storage; - _gallery = gallery; - _id = packageId; - _version = packageVersion; - _verbose = verbose; - } - - public override string GetUsage() - { - return "Usage: ng package2catalog " - + $"-{Arguments.Gallery} " - + $"-{Arguments.StorageBaseAddress} " - + $"-{Arguments.StorageType} file|azure " - + $"[-{Arguments.StoragePath} ]" - + "|" - + $"[-{Arguments.StorageAccountName} " - + $"-{Arguments.StorageKeyValue} " - + $"-{Arguments.StorageContainer} " - + $"-{Arguments.StoragePath} ] " - + $"[-{Arguments.Verbose} true|false] " - + $"-{Arguments.Id} " - + $"[-{Arguments.Version} ]"; - } - - protected override void Init(IDictionary arguments, CancellationToken cancellationToken) - { - _gallery = arguments.GetOrThrow(Arguments.Gallery); - _verbose = arguments.GetOrDefault(Arguments.Verbose, false); - _id = arguments.GetOrThrow(Arguments.Id); - _version = arguments.GetOrDefault(Arguments.Version); - - var storageFactory = CommandHelpers.CreateStorageFactory(arguments, _verbose); - - _storage = storageFactory.Create(); - } - - protected override async Task RunInternalAsync(CancellationToken cancellationToken) - { - var timeout = TimeSpan.FromSeconds(300); - - using (var client = CreateHttpClient()) - { - client.Timeout = timeout; - - // if the version is specified a single package is processed otherwise all the packages corresponding to that id are processed - - var uri = (_version == null) ? MakePackageUri(_gallery, _id) : MakePackageUri(_gallery, _id, _version); - - var packages = await FeedHelpers.GetPackagesInOrder(client, uri, package => package.CreatedDate); - - Logger.LogInformation($"Downloading {packages.Select(t => t.Value.Count).Sum()} packages"); - - // the idea here is to leave the lastCreated, lastEdited and lastDeleted values exactly as they were - var catalogProperties = await CatalogProperties.ReadAsync(_storage, TelemetryService, cancellationToken); - var lastCreated = catalogProperties.LastCreated ?? DateTime.MinValue.ToUniversalTime(); - var lastEdited = catalogProperties.LastEdited ?? DateTime.MinValue.ToUniversalTime(); - var lastDeleted = catalogProperties.LastDeleted ?? DateTime.MinValue.ToUniversalTime(); - var packageCatalogItemCreator = PackageCatalogItemCreator.Create( - client, - TelemetryService, - Logger, - storage: null); - - await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( - packageCatalogItemCreator, - packages, - _storage, - lastCreated, - lastEdited, - lastDeleted, - MaxDegreeOfParallelism, - createdPackages: null, - updateCreatedFromEdited: false, - cancellationToken: cancellationToken, - telemetryService: TelemetryService, - logger: Logger); - } - } - - protected virtual HttpClient CreateHttpClient() - { - return FeedHelpers.CreateHttpClient(CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose)); - } - - private static Uri MakePackageUri(string source, string id, string version) - { - var address = - $"{source.Trim('/')}/Packages?$filter=Id%20eq%20'{id}'%20and%20Version%20eq%20'{version}'&$select=Created,LastEdited,Published,LicenseNames,LicenseReportUrl"; - - return new Uri(address); - } - - private static Uri MakePackageUri(string source, string id) - { - var address = - $"{source.Trim('/')}/Packages?$filter=Id%20eq%20'{id}'&$select=Created,LastEdited,Published,LicenseNames,LicenseReportUrl"; - - return new Uri(address); - } - } -} \ No newline at end of file diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index fe5dcc328..8ae94cb67 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -78,7 +78,6 @@ - diff --git a/src/Ng/NgJobFactory.cs b/src/Ng/NgJobFactory.cs index 7fe9eda63..85e868054 100644 --- a/src/Ng/NgJobFactory.cs +++ b/src/Ng/NgJobFactory.cs @@ -13,7 +13,6 @@ public static class NgJobFactory { public static IDictionary JobMap = new Dictionary() { - { "package2catalog", typeof(Package2CatalogJob) }, { "db2catalog", typeof(Db2CatalogJob) }, { "catalog2registration", typeof(Catalog2RegistrationJob) }, { "catalog2lucene", typeof(Catalog2LuceneJob) }, diff --git a/tests/CatalogTests/Helpers/FeedHelpersTests.cs b/tests/CatalogTests/Helpers/FeedHelpersTests.cs index 94774d1d0..852f015c4 100644 --- a/tests/CatalogTests/Helpers/FeedHelpersTests.cs +++ b/tests/CatalogTests/Helpers/FeedHelpersTests.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -52,58 +51,6 @@ public async Task GetPackages_GetsAllPackages(IEnumerable oDataPac } } - public static IEnumerable GetPackagesInOrder_GetsAllPackagesInOrder_data - { - get - { - var oDataPackages = ODataPackages; - - var keyDateFuncs = new Func[] - { - package => package.CreatedDate, - package => package.LastEditedDate, - package => package.PublishedDate, - }; - - return keyDateFuncs.Select(p => new object[] { oDataPackages, p }); - } - } - - [Theory] - [MemberData(nameof(GetPackagesInOrder_GetsAllPackagesInOrder_data))] - public async Task GetPackagesInOrder_GetsAllPackagesInOrder(IEnumerable oDataPackages, Func keyDateFunc) - { - //// Act - var feedPackagesInOrder = await TestGetPackagesInOrder(oDataPackages, keyDateFunc); - - //// Assert - // All OData packages must exist in the result. - foreach (var oDataPackage in oDataPackages) - { - Assert.Contains(feedPackagesInOrder.SelectMany(p => p.Value), - (feedPackage) => - { - return ArePackagesEqual(feedPackage, oDataPackage); - }); - } - - // The packages must be in order and grouped by timestamp. - var currentTimestamp = DateTime.MinValue; - foreach (var feedPackages in feedPackagesInOrder) - { - Assert.True(currentTimestamp < feedPackages.Key); - - currentTimestamp = feedPackages.Key; - foreach (var feedPackage in feedPackages.Value) - { - var packageKeyDate = keyDateFunc(feedPackage); - Assert.Equal(packageKeyDate.Ticks, currentTimestamp.Ticks); - - VerifyDateTimesAreInUtc(feedPackage); - } - } - } - private Task> TestGetPackagesAsync(IEnumerable oDataPackages) { return FeedHelpers.GetPackages( @@ -111,16 +58,6 @@ private Task> TestGetPackagesAsync(IEnumerable>> TestGetPackagesInOrder( - IEnumerable oDataPackages, - Func keyDateFunc) - { - return FeedHelpers.GetPackagesInOrder( - new HttpClient(GetMessageHandlerForPackages(oDataPackages)), - new Uri(_baseUri + "/test"), - keyDateFunc); - } - private HttpMessageHandler GetMessageHandlerForPackages(IEnumerable oDataPackages) { var mockServer = new MockServerHttpClientHandler(); diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj index c8d8e0d8a..57f9d653d 100644 --- a/tests/NgTests/NgTests.csproj +++ b/tests/NgTests/NgTests.csproj @@ -75,7 +75,6 @@ - diff --git a/tests/NgTests/Package2CatalogJobTests.cs b/tests/NgTests/Package2CatalogJobTests.cs deleted file mode 100644 index d0825b96e..000000000 --- a/tests/NgTests/Package2CatalogJobTests.cs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Globalization; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Moq; -using Ng.Jobs; -using NgTests.Infrastructure; -using NuGet.Services.Metadata.Catalog; -using NuGet.Services.Metadata.Catalog.Persistence; -using Xunit; -using Xunit.Abstractions; - -namespace NgTests -{ - public class Package2CatalogJobTests - { - private const string _feedBaseUri = "http://unit.test"; - private const string _feedUriSuffix = "/Packages?$filter=Id%20eq%20'{0}'%20and%20Version%20eq%20'{1}'&$select=Created,LastEdited,Published,LicenseNames,LicenseReportUrl"; - private readonly ITestOutputHelper _testOutputHelper; - - public Package2CatalogJobTests(ITestOutputHelper testOutputHelper) - { - _testOutputHelper = testOutputHelper; - } - - [Fact] - public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() - { - var messageHandler = new MockServerHttpClientHandler(); - var catalogStorage = new Mock(MockBehavior.Strict); - var datetime = DateTime.MinValue.ToString("O") + "Z"; - var json = $"{{\"nuget:lastCreated\":\"{datetime}\"," + - $"\"nuget:lastDeleted\":\"{datetime}\"," + - $"\"nuget:lastEdited\":\"{datetime}\"}}"; - var packageId = "a"; - var packageVersion = "1.0.0"; - - catalogStorage.Setup(x => x.ResolveUri(It.IsNotNull())) - .Returns(new Uri(_feedBaseUri)); - catalogStorage.Setup(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny())) - .ReturnsAsync(json); - - messageHandler.SetAction("/", GetRootActionAsync); - messageHandler.SetAction(GetPath(packageId, packageVersion), GetEmptyPackages); - - var job = new TestPackage2CatalogJob( - _testOutputHelper, - messageHandler, - catalogStorage.Object, - _feedBaseUri, - packageId, - packageVersion, - verbose: true); - - await job.RunOnceAsync(CancellationToken.None); - - catalogStorage.Verify(x => x.ResolveUri(It.IsNotNull()), Times.AtLeastOnce()); - catalogStorage.Verify(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny()), Times.Once()); - } - - private static string GetPath(string packageId, string packageVersion) - { - return string.Format(CultureInfo.InvariantCulture, _feedUriSuffix, packageId, packageVersion); - } - - private static Task GetEmptyPackages(HttpRequestMessage request) - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent( - ODataFeedHelper.ToODataFeed(Enumerable.Empty(), new Uri(_feedBaseUri), "Packages")) - }); - } - - private static Task GetRootActionAsync(HttpRequestMessage request) - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); - } - - private sealed class TestPackage2CatalogJob : Package2CatalogJob - { - private readonly HttpMessageHandler _handler; - - internal TestPackage2CatalogJob( - ITestOutputHelper testOutputHelper, - HttpMessageHandler handler, - IStorage storage, - string gallery, - string packageId, - string packageVersion, - bool verbose) : base( - new Mock().Object, - new TestLoggerFactory(testOutputHelper), - storage, - gallery, - packageId, - packageVersion, - verbose) - { - _handler = handler; - } - - protected override HttpClient CreateHttpClient() - { - return new HttpClient(_handler); - } - - internal Task RunOnceAsync(CancellationToken cancellationToken) - { - return RunInternalAsync(cancellationToken); - } - } - } -} \ No newline at end of file