From f34cdc5c4486f236a8a2a4e399a2970b447c0e28 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sat, 5 Oct 2019 23:31:23 -0700 Subject: [PATCH] Extract collector DI and registration client to generic V3 project (#677) This supports building other collectors that use dependency injection like Catalog2AzureSearch. Also, the registration client code is not Azure Search specific so a more generic place is better. This is a PR to move code around that has no impact on behavior. It allows me to share code between other new collectors and Catalog2AzureSearch without pulling in all of the Azure Search dependencies. --- NuGet.Services.Metadata.sln | 30 ++++ src/NuGet.Jobs.Catalog2AzureSearch/Job.cs | 2 + .../NuGet.Jobs.Catalog2AzureSearch.csproj | 4 + .../AzureSearchTelemetryService.cs | 10 -- .../AzureSearchCollectorLogic.cs | 160 ++--------------- .../Catalog2AzureSearchCommand.cs | 1 + .../Catalog2AzureSearchConfiguration.cs | 3 +- .../DependencyInjectionExtensions.cs | 44 +---- .../IAzureSearchTelemetryService.cs | 1 - .../NuGet.Services.AzureSearch.csproj | 23 +-- src/NuGet.Services.BasicSearch/Web.config | 4 +- src/NuGet.Services.SearchService/Web.config | 4 +- .../CommitCollectorConfiguration.cs | 14 ++ .../CommitCollectorHost.cs} | 12 +- .../CommitCollectorUtility.cs | 169 ++++++++++++++++++ .../DefaultBlobRequestOptions.cs | 23 +++ .../DependencyInjectionExtensions.cs | 50 ++++++ .../ICollector.cs | 2 +- .../ICommitCollectorConfiguration.cs | 14 ++ .../ICommitCollectorLogic.cs | 2 +- src/NuGet.Services.V3/IV3TelemetryService.cs | 12 ++ .../NuGet.Services.V3.csproj | 96 ++++++++++ .../Properties/AssemblyInfo.cs | 9 + .../Registration/IRegistrationClient.cs | 0 .../Models/RegistrationCatalogEntry.cs | 0 .../Registration/Models/RegistrationIndex.cs | 0 .../Registration/Models/RegistrationLeaf.cs | 0 .../Models/RegistrationLeafItem.cs | 0 .../Registration/Models/RegistrationPage.cs | 0 .../Registration/RegistrationClient.cs | 0 .../Registration/RegistrationUrlBuilder.cs | 0 .../Support}/Guard.cs | 4 +- .../Support}/IdAndValue.cs | 4 +- .../Support}/KeyValuePair.cs | 4 +- .../Support}/ReferenceEqualityComparer.cs | 5 +- src/NuGet.Services.V3/V3TelemetryService.cs | 31 ++++ .../DownloadSetComparerFacts.cs | 1 - .../AuxiliaryFileClientFacts.cs | 1 - .../AuxiliaryFiles/DownloadDataClientFacts.cs | 1 - .../AuxiliaryFiles/OwnerDataClientFacts.cs | 1 - .../VerifiedPackagesDataClientFacts.cs | 1 - .../BatchPusherFacts.cs | 1 - .../AzureSearchCollectorLogicFacts.cs | 36 ++-- .../Catalog2AzureSearchCommandFacts.cs | 2 +- .../CatalogLeafFetcherFacts.cs | 1 - ...ureSearchCollectorLogicIntegrationTests.cs | 22 ++- .../Db2AzureSearchCommandFacts.cs | 1 - .../NewPackageRegistrationProducerFacts.cs | 1 - .../PackageEntityIndexActionBuilderFacts.cs | 1 - .../IndexBuilderFacts.cs | 1 - .../NuGet.Services.AzureSearch.Tests.csproj | 22 +-- .../OwnerSetComparerFacts.cs | 1 - .../Owners2AzureSearchCommandFacts.cs | 1 - .../AuxiliaryFileReloaderFacts.cs | 1 - .../SearchService/SearchStatusServiceFacts.cs | 1 - .../SearchService/SecretRefresherFacts.cs | 1 - .../VersionList/VersionListDataClientFacts.cs | 1 - .../DocumentOperationsWrapperFacts.cs | 1 - .../NuGet.Services.V3.Tests.csproj | 91 ++++++++++ .../Properties/AssemblyInfo.cs | 9 + .../Registration/RegistrationClientFacts.cs | 0 .../RegistrationUrlBuilderFacts.cs | 0 .../Support/Cursor.cs | 2 +- .../Support/DbSetMockFactory.cs | 2 +- .../Support/RecordingLogger.cs | 2 +- .../Support/TestCursorStorage.cs | 2 +- .../Support/TestDbAsyncQueryProvider.cs | 2 +- .../Support/TestHttpMessageHandler.cs | 2 +- .../Support/TestOutputHelperExtensions.cs | 2 +- .../Support/XunitLogger.cs | 0 .../Support/XunitLoggerFactoryExtensions.cs | 0 .../Support/XunitLoggerProvider.cs | 0 72 files changed, 662 insertions(+), 289 deletions(-) create mode 100644 src/NuGet.Services.V3/CommitCollectorConfiguration.cs rename src/{NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollector.cs => NuGet.Services.V3/CommitCollectorHost.cs} (82%) create mode 100644 src/NuGet.Services.V3/CommitCollectorUtility.cs create mode 100644 src/NuGet.Services.V3/DefaultBlobRequestOptions.cs create mode 100644 src/NuGet.Services.V3/DependencyInjectionExtensions.cs rename src/{NuGet.Services.AzureSearch/Catalog2AzureSearch => NuGet.Services.V3}/ICollector.cs (88%) create mode 100644 src/NuGet.Services.V3/ICommitCollectorConfiguration.cs rename src/{NuGet.Services.AzureSearch/Catalog2AzureSearch => NuGet.Services.V3}/ICommitCollectorLogic.cs (90%) create mode 100644 src/NuGet.Services.V3/IV3TelemetryService.cs create mode 100644 src/NuGet.Services.V3/NuGet.Services.V3.csproj create mode 100644 src/NuGet.Services.V3/Properties/AssemblyInfo.cs rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/IRegistrationClient.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/Models/RegistrationCatalogEntry.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/Models/RegistrationIndex.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/Models/RegistrationLeaf.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/Models/RegistrationLeafItem.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/Models/RegistrationPage.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/RegistrationClient.cs (100%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3}/Registration/RegistrationUrlBuilder.cs (100%) rename src/{NuGet.Services.AzureSearch/VersionList => NuGet.Services.V3/Support}/Guard.cs (94%) rename src/{NuGet.Services.AzureSearch => NuGet.Services.V3/Support}/IdAndValue.cs (86%) rename src/{NuGet.Services.AzureSearch/VersionList => NuGet.Services.V3/Support}/KeyValuePair.cs (84%) rename src/{NuGet.Services.AzureSearch/Catalog2AzureSearch => NuGet.Services.V3/Support}/ReferenceEqualityComparer.cs (88%) create mode 100644 src/NuGet.Services.V3/V3TelemetryService.cs create mode 100644 tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj create mode 100644 tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Registration/RegistrationClientFacts.cs (100%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Registration/RegistrationUrlBuilderFacts.cs (100%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/Cursor.cs (87%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/DbSetMockFactory.cs (97%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/RecordingLogger.cs (96%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/TestCursorStorage.cs (98%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/TestDbAsyncQueryProvider.cs (98%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/TestHttpMessageHandler.cs (94%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/TestOutputHelperExtensions.cs (93%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/XunitLogger.cs (100%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/XunitLoggerFactoryExtensions.cs (100%) rename tests/{NuGet.Services.AzureSearch.Tests => NuGet.Services.V3.Tests}/Support/XunitLoggerProvider.cs (100%) diff --git a/NuGet.Services.Metadata.sln b/NuGet.Services.Metadata.sln index 27b725b8e..ffc0f9358 100644 --- a/NuGet.Services.Metadata.sln +++ b/NuGet.Services.Metadata.sln @@ -63,6 +63,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution build.ps1 = build.ps1 EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3", "src\NuGet.Services.V3\NuGet.Services.V3.csproj", "{C3F9A738-9759-4B2B-A50D-6507B28A659B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3.Tests", "tests\NuGet.Services.V3.Tests\NuGet.Services.V3.Tests.csproj", "{CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -321,6 +325,30 @@ Global {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|Mixed Platforms.Build.0 = Release|Any CPU {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|x64.ActiveCfg = Release|Any CPU {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|x64.Build.0 = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|x64.Build.0 = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Any CPU.Build.0 = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|x64.ActiveCfg = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|x64.Build.0 = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|x64.ActiveCfg = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|x64.Build.0 = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Any CPU.Build.0 = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|x64.ActiveCfg = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -347,6 +375,8 @@ Global {F009209D-A663-45E1-87E8-158569A0F097} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} {F81A7CA6-97D4-4958-A9C1-FF94FCA283CB} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} + {C3F9A738-9759-4B2B-A50D-6507B28A659B} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D3AB83E9-02B4-4FFA-A2D0-637F0B97E626} diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs b/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs index 831e0b121..989469c1d 100644 --- a/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection; using NuGet.Services.AzureSearch; using NuGet.Services.AzureSearch.Catalog2AzureSearch; +using NuGet.Services.V3; namespace NuGet.Jobs { @@ -17,6 +18,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi base.ConfigureJobServices(services, configurationRoot); services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); } diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj b/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj index 1c4afce3d..a4cca2381 100644 --- a/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj +++ b/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj @@ -71,6 +71,10 @@ {1a53fe3d-8041-4773-942f-d73aef5b82b2} NuGet.Services.AzureSearch + + {C3F9A738-9759-4B2B-A50D-6507B28A659B} + NuGet.Services.V3 + diff --git a/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs b/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs index ae4e4a37e..71d7f9237 100644 --- a/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs +++ b/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs @@ -230,16 +230,6 @@ public void TrackGetSearchServiceStatus(SearchStatusOptions options, bool succes }); } - public IDisposable TrackCatalogLeafDownloadBatch(int count) - { - return _telemetryClient.TrackDuration( - Prefix + "CatalogLeafDownloadBatchSeconds", - new Dictionary - { - { "Count", count.ToString() }, - }); - } - public void TrackDocumentCountQuery(string indexName, long count, TimeSpan elapsed) { _telemetryClient.TrackMetric( diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs index e491f0efb..7ec10c220 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs @@ -10,29 +10,30 @@ using Microsoft.Extensions.Options; using NuGet.Protocol.Catalog; using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; namespace NuGet.Services.AzureSearch.Catalog2AzureSearch { public class AzureSearchCollectorLogic : ICommitCollectorLogic { - private readonly ICatalogClient _catalogClient; private readonly ICatalogIndexActionBuilder _indexActionBuilder; private readonly Func _batchPusherFactory; + private readonly CommitCollectorUtility _utility; private readonly IOptionsSnapshot _options; private readonly IAzureSearchTelemetryService _telemetryService; private readonly ILogger _logger; public AzureSearchCollectorLogic( - ICatalogClient catalogClient, ICatalogIndexActionBuilder indexActionBuilder, Func batchPusherFactory, + CommitCollectorUtility utility, IOptionsSnapshot options, IAzureSearchTelemetryService telemetryService, ILogger logger) { - _catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient)); _indexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder)); _batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory)); + _utility = utility ?? throw new ArgumentNullException(nameof(utility)); _options = options ?? throw new ArgumentNullException(nameof(options)); _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -43,51 +44,21 @@ public AzureSearchCollectorLogic( nameof(options), $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero."); } - - if (_options.Value.MaxConcurrentCatalogLeafDownloads <= 0) - { - throw new ArgumentOutOfRangeException( - nameof(options), - $"The {nameof(Catalog2AzureSearchConfiguration.MaxConcurrentCatalogLeafDownloads)} must be greater than zero."); - } } public Task> CreateBatchesAsync(IEnumerable catalogItems) { - if (!catalogItems.Any()) - { - return Task.FromResult(Enumerable.Empty()); - } - - var maxCommitTimestamp = catalogItems.Max(x => x.CommitTimeStamp); - // Create a single batch of all unprocessed catalog items so that we can have complete control of the // parallelism in this class. - return Task.FromResult>(new[] - { - new CatalogCommitItemBatch( - catalogItems, - key: null, - commitTimestamp: maxCommitTimestamp), - }); + return Task.FromResult(_utility.CreateSingleBatch(catalogItems)); } public async Task OnProcessBatchAsync( IEnumerable items) { var itemList = items.ToList(); - - // Ignore all but the latest catalog commit items per package identity. - var latestItems = items - .GroupBy(x => x.PackageIdentity) - .Select(GetLatest) - .ToList(); - - // Group the catalog commit items by package ID. - var workEnumerable = latestItems - .GroupBy(x => x.PackageIdentity.Id, StringComparer.OrdinalIgnoreCase) - .Select(x => new Work(x.Key, x.ToList())); - var allWork = new ConcurrentBag(workEnumerable); + var latestItems = _utility.GetLatestPerIdentity(items); + var allWork = _utility.GroupById(latestItems); using (_telemetryService.TrackCatalog2AzureSearchProcessBatch(itemList.Count, latestItems.Count, allWork.Count)) { @@ -109,10 +80,10 @@ public async Task OnProcessBatchAsync( private async Task>> ProcessWorkAsync( IReadOnlyList latestItems, - ConcurrentBag allWork) + ConcurrentBag>> allWork) { // Fetch the full catalog leaf for each item that is the package details type. - var allEntryToLeaf = await GetEntryToLeafAsync(latestItems); + var allEntryToLeaf = await _utility.GetEntryToDetailsLeafAsync(latestItems); // Process the package ID groups in parallel, collecting all index actions for later. var output = new ConcurrentBag>(); @@ -124,11 +95,11 @@ private async Task>> ProcessWorkAsync( while (allWork.TryTake(out var work)) { var entryToLeaf = work - .Entries - .Where(IsOnlyPackageDetails) + .Value + .Where(CommitCollectorUtility.IsOnlyPackageDetails) .ToDictionary(e => e, e => allEntryToLeaf[e], ReferenceEqualityComparer.Default); - var indexActions = await GetPackageIdIndexActionsAsync(work.Entries, entryToLeaf); - output.Add(new IdAndValue(work.PackageId, indexActions)); + var indexActions = await GetPackageIdIndexActionsAsync(work.Value, entryToLeaf); + output.Add(new IdAndValue(work.Id, indexActions)); } }) .ToList(); @@ -137,97 +108,6 @@ private async Task>> ProcessWorkAsync( return output; } - private CatalogCommitItem GetLatest(IEnumerable entries) - { - CatalogCommitItem max = null; - foreach (var entry in entries) - { - if (max == null) - { - max = entry; - continue; - } - - Guard.Assert( - StringComparer.OrdinalIgnoreCase.Equals(max.PackageIdentity, entry.PackageIdentity), - "The entries compared should have the same package identity."); - - if (entry.CommitTimeStamp > max.CommitTimeStamp) - { - max = entry; - } - else if (entry.CommitTimeStamp == max.CommitTimeStamp) - { - const string message = "There are multiple catalog leaves for a single package at one time."; - _logger.LogError( - message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}", - entry.PackageIdentity.Id, - entry.PackageIdentity.Version.ToFullString(), - entry.CommitTimeStamp); - throw new InvalidOperationException(message); - } - } - - return max; - } - - private async Task> GetEntryToLeafAsync( - IEnumerable entries) - { - var packageDetailsEntries = entries.Where(IsOnlyPackageDetails); - var allWork = new ConcurrentBag(packageDetailsEntries); - var output = new ConcurrentBag>(); - - using (_telemetryService.TrackCatalogLeafDownloadBatch(allWork.Count)) - { - var tasks = Enumerable - .Range(0, _options.Value.MaxConcurrentCatalogLeafDownloads) - .Select(async x => - { - await Task.Yield(); - while (allWork.TryTake(out var work)) - { - try - { - _logger.LogInformation( - "Downloading catalog leaf for {PackageId} {Version}: {Url}", - work.PackageIdentity.Id, - work.PackageIdentity.Version.ToNormalizedString(), - work.Uri.AbsoluteUri); - - var leaf = await _catalogClient.GetPackageDetailsLeafAsync(work.Uri.AbsoluteUri); - output.Add(KeyValuePair.Create(work, leaf)); - } - catch (Exception ex) - { - _logger.LogError( - 0, - ex, - "An exception was thrown when fetching the package details leaf for {Id} {Version}. " + - "The URL is {Url}", - work.PackageIdentity.Id, - work.PackageIdentity.Version, - work.Uri.AbsoluteUri); - throw; - } - } - }) - .ToList(); - - await Task.WhenAll(tasks); - - return output.ToDictionary( - x => x.Key, - x => x.Value, - ReferenceEqualityComparer.Default); - } - } - - private static bool IsOnlyPackageDetails(CatalogCommitItem e) - { - return e.IsPackageDetails && !e.IsPackageDelete; - } - private async Task GetPackageIdIndexActionsAsync( IReadOnlyList entries, IReadOnlyDictionary entryToLeaf) @@ -242,19 +122,5 @@ private async Task GetPackageIdIndexActionsAsync( entries, entryToLeaf); } - - private class Work - { - public Work( - string packageId, - IReadOnlyList entries) - { - PackageId = packageId ?? throw new ArgumentNullException(nameof(packageId)); - Entries = entries ?? throw new ArgumentNullException(nameof(entries)); - } - - public string PackageId { get; } - public IReadOnlyList Entries { get; } - } } } diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs index 35912b976..d2d6c4db3 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs @@ -11,6 +11,7 @@ using Microsoft.WindowsAzure.Storage; using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; namespace NuGet.Services.AzureSearch.Catalog2AzureSearch { diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs index b14b386a3..ca37f2147 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs @@ -3,10 +3,11 @@ using System; using System.Collections.Generic; +using NuGet.Services.V3; namespace NuGet.Services.AzureSearch.Catalog2AzureSearch { - public class Catalog2AzureSearchConfiguration : AzureSearchJobConfiguration + public class Catalog2AzureSearchConfiguration : AzureSearchJobConfiguration, ICommitCollectorConfiguration { public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64; public bool CreateContainersAndIndexes { get; set; } diff --git a/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs b/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs index 9629ca23f..318b1cc49 100644 --- a/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs +++ b/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Net; using System.Net.Http; using Autofac; using Microsoft.Azure.Search; @@ -13,10 +12,8 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; -using NuGet.Jobs.Validation; using NuGet.Protocol; using NuGet.Protocol.Catalog; -using NuGet.Protocol.Registration; using NuGet.Services.AzureSearch.Auxiliary2AzureSearch; using NuGet.Services.AzureSearch.AuxiliaryFiles; using NuGet.Services.AzureSearch.Catalog2AzureSearch; @@ -24,10 +21,9 @@ using NuGet.Services.AzureSearch.Owners2AzureSearch; using NuGet.Services.AzureSearch.SearchService; using NuGet.Services.AzureSearch.Wrappers; -using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; using NuGetGallery; -using NuGetGallery.Diagnostics; namespace NuGet.Services.AzureSearch { @@ -106,7 +102,7 @@ private static void RegisterAzureSearchStorageServices(ContainerBuilder containe var options = c.Resolve>(); return new CloudBlobClientWrapper( options.Value.StorageConnectionString, - GetBlobRequestOptions()); + DefaultBlobRequestOptions.Create()); }) .Keyed(key); @@ -204,7 +200,7 @@ private static void RegisterAuxiliaryDataStorageServices(ContainerBuilder contai var options = c.Resolve>(); return new CloudBlobClientWrapper( options.Value.AuxiliaryDataStorageConnectionString, - GetBlobRequestOptions()); + DefaultBlobRequestOptions.Create()); }) .Keyed(key); @@ -216,31 +212,9 @@ private static void RegisterAuxiliaryDataStorageServices(ContainerBuilder contai c.Resolve>())); } - private static BlobRequestOptions GetBlobRequestOptions() - { - return new BlobRequestOptions - { - ServerTimeout = TimeSpan.FromMinutes(2), - MaximumExecutionTime = TimeSpan.FromMinutes(10), - LocationMode = LocationMode.PrimaryThenSecondary, - RetryPolicy = new ExponentialRetry(), - }; - } - public static IServiceCollection AddAzureSearch(this IServiceCollection services) { - services - .AddTransient(p => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }); - - services - .AddTransient(p => (HttpMessageHandler)new TelemetryHandler( - p.GetRequiredService(), - p.GetRequiredService())); - - services.AddSingleton(p => new HttpClient(p.GetRequiredService())); + services.AddV3(); services .AddTransient(p => @@ -251,11 +225,6 @@ public static IServiceCollection AddAzureSearch(this IServiceCollection services new SearchCredentials(options.Value.SearchServiceApiKey)); }); - services - .AddTransient(p => new CatalogClient( - p.GetRequiredService(), - p.GetRequiredService>())); - services.AddSingleton(); services.AddScoped(p => p.GetRequiredService().Get()); services.AddSingleton(); @@ -269,10 +238,8 @@ public static IServiceCollection AddAzureSearch(this IServiceCollection services services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -281,7 +248,6 @@ public static IServiceCollection AddAzureSearch(this IServiceCollection services services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -289,9 +255,7 @@ public static IServiceCollection AddAzureSearch(this IServiceCollection services services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); - services.AddTransient(); return services; } diff --git a/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs b/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs index 3faf0b8b4..7bddae99e 100644 --- a/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs +++ b/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs @@ -42,7 +42,6 @@ void TrackDownloadCountDecrease( long newDownloads); void TrackWarmQuery(string indexName, TimeSpan elapsed); void TrackLastCommitTimestampQuery(string indexName, DateTimeOffset? lastCommitTimestamp, TimeSpan elapsed); - IDisposable TrackCatalogLeafDownloadBatch(int count); void TrackReadLatestIndexedDownloads(int? packageIdCount, bool notModified, TimeSpan elapsed); IDisposable TrackReplaceLatestIndexedDownloads(int packageIdCount); void TrackAuxiliary2AzureSearchCompleted(JobOutcome outcome, TimeSpan elapsed); diff --git a/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj index e91d44b9f..ab4e339b2 100644 --- a/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj +++ b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj @@ -1,6 +1,7 @@  + Debug AnyCPU @@ -96,7 +97,6 @@ - @@ -105,14 +105,11 @@ - - - @@ -121,7 +118,6 @@ - @@ -138,14 +134,6 @@ - - - - - - - - @@ -198,14 +186,12 @@ - - @@ -245,9 +231,6 @@ 5.0.3 - - 4.3.0-dev-3159955 - 4.8.0 runtime; build; native; contentfiles; analyzers @@ -267,6 +250,10 @@ {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} NuGet.Protocol.Catalog + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + diff --git a/src/NuGet.Services.BasicSearch/Web.config b/src/NuGet.Services.BasicSearch/Web.config index e49002bfd..288e1fe3e 100644 --- a/src/NuGet.Services.BasicSearch/Web.config +++ b/src/NuGet.Services.BasicSearch/Web.config @@ -5,8 +5,8 @@ --> - - + + diff --git a/src/NuGet.Services.SearchService/Web.config b/src/NuGet.Services.SearchService/Web.config index 0c528708c..e68aac17f 100644 --- a/src/NuGet.Services.SearchService/Web.config +++ b/src/NuGet.Services.SearchService/Web.config @@ -2,8 +2,8 @@ - - + + diff --git a/src/NuGet.Services.V3/CommitCollectorConfiguration.cs b/src/NuGet.Services.V3/CommitCollectorConfiguration.cs new file mode 100644 index 000000000..156eaee85 --- /dev/null +++ b/src/NuGet.Services.V3/CommitCollectorConfiguration.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.V3 +{ + public class CommitCollectorConfiguration : ICommitCollectorConfiguration + { + public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64; + public string Source { get; set; } + public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10); + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollector.cs b/src/NuGet.Services.V3/CommitCollectorHost.cs similarity index 82% rename from src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollector.cs rename to src/NuGet.Services.V3/CommitCollectorHost.cs index 0667755d7..db11739a3 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollector.cs +++ b/src/NuGet.Services.V3/CommitCollectorHost.cs @@ -10,21 +10,21 @@ using Newtonsoft.Json.Linq; using NuGet.Services.Metadata.Catalog; -namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +namespace NuGet.Services.V3 { /// - /// This is a minimal integration class between the core of the catalog2azuresearch collector and the complex - /// collector infrastructure that we have today. + /// This is a minimal integration class between the core of the collectors based on NuGet.Jobs infrastructure and + /// the overly complex collector infrastructure that we have today. /// - public class AzureSearchCollector : CommitCollector, ICollector + public class CommitCollectorHost : CommitCollector, ICollector { private readonly ICommitCollectorLogic _logic; - public AzureSearchCollector( + public CommitCollectorHost( ICommitCollectorLogic logic, ITelemetryService telemetryService, Func handlerFunc, - IOptionsSnapshot options) : base( + IOptionsSnapshot options) : base( new Uri(options.Value.Source), telemetryService, handlerFunc, diff --git a/src/NuGet.Services.V3/CommitCollectorUtility.cs b/src/NuGet.Services.V3/CommitCollectorUtility.cs new file mode 100644 index 000000000..eb62f7de4 --- /dev/null +++ b/src/NuGet.Services.V3/CommitCollectorUtility.cs @@ -0,0 +1,169 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.V3 +{ + public class CommitCollectorUtility + { + private readonly ICatalogClient _catalogClient; + private readonly IV3TelemetryService _telemetryService; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public CommitCollectorUtility( + ICatalogClient catalogClient, + IV3TelemetryService telemetryService, + IOptionsSnapshot options, + ILogger logger) + { + _catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentCatalogLeafDownloads <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(ICommitCollectorConfiguration.MaxConcurrentCatalogLeafDownloads)} must be greater than zero."); + } + } + + public IEnumerable CreateSingleBatch(IEnumerable catalogItems) + { + if (!catalogItems.Any()) + { + return Enumerable.Empty(); + } + + var maxCommitTimestamp = catalogItems.Max(x => x.CommitTimeStamp); + + return new[] + { + new CatalogCommitItemBatch( + catalogItems, + key: null, + commitTimestamp: maxCommitTimestamp), + }; + } + + public List GetLatestPerIdentity(IEnumerable items) + { + return items + .GroupBy(x => x.PackageIdentity) + .Select(GetLatestPerSingleIdentity) + .ToList(); + } + + private CatalogCommitItem GetLatestPerSingleIdentity(IEnumerable entries) + { + CatalogCommitItem max = null; + foreach (var entry in entries) + { + if (max == null) + { + max = entry; + continue; + } + + if (!StringComparer.OrdinalIgnoreCase.Equals(max.PackageIdentity, entry.PackageIdentity)) + { + throw new InvalidOperationException("The entries compared should have the same package identity."); + } + + if (entry.CommitTimeStamp > max.CommitTimeStamp) + { + max = entry; + } + else if (entry.CommitTimeStamp == max.CommitTimeStamp) + { + const string message = "There are multiple catalog leaves for a single package at one time."; + _logger.LogError( + message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}", + entry.PackageIdentity.Id, + entry.PackageIdentity.Version.ToFullString(), + entry.CommitTimeStamp); + throw new InvalidOperationException(message); + } + } + + return max; + } + + public ConcurrentBag>> GroupById(List latestItems) + { + var workEnumerable = latestItems + .GroupBy(x => x.PackageIdentity.Id, StringComparer.OrdinalIgnoreCase) + .Select(x => new IdAndValue>(x.Key, x.ToList())); + + var allWork = new ConcurrentBag>>(workEnumerable); + return allWork; + } + + public async Task> GetEntryToDetailsLeafAsync( + IEnumerable entries) + { + var packageDetailsEntries = entries.Where(IsOnlyPackageDetails); + var allWork = new ConcurrentBag(packageDetailsEntries); + var output = new ConcurrentBag>(); + + using (_telemetryService.TrackCatalogLeafDownloadBatch(allWork.Count)) + { + var tasks = Enumerable + .Range(0, _options.Value.MaxConcurrentCatalogLeafDownloads) + .Select(async x => + { + await Task.Yield(); + while (allWork.TryTake(out var work)) + { + try + { + _logger.LogInformation( + "Downloading catalog leaf for {PackageId} {Version}: {Url}", + work.PackageIdentity.Id, + work.PackageIdentity.Version.ToNormalizedString(), + work.Uri.AbsoluteUri); + + var leaf = await _catalogClient.GetPackageDetailsLeafAsync(work.Uri.AbsoluteUri); + output.Add(KeyValuePair.Create(work, leaf)); + } + catch (Exception ex) + { + _logger.LogError( + ex, + "An exception was thrown when fetching the package details leaf for {Id} {Version}. " + + "The URL is {Url}", + work.PackageIdentity.Id, + work.PackageIdentity.Version, + work.Uri.AbsoluteUri); + throw; + } + } + }) + .ToList(); + + await Task.WhenAll(tasks); + + return output.ToDictionary( + x => x.Key, + x => x.Value, + ReferenceEqualityComparer.Default); + } + } + + public static bool IsOnlyPackageDetails(CatalogCommitItem e) + { + return e.IsPackageDetails && !e.IsPackageDelete; + } + } +} diff --git a/src/NuGet.Services.V3/DefaultBlobRequestOptions.cs b/src/NuGet.Services.V3/DefaultBlobRequestOptions.cs new file mode 100644 index 000000000..aabcdbbc7 --- /dev/null +++ b/src/NuGet.Services.V3/DefaultBlobRequestOptions.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.RetryPolicies; + +namespace NuGet.Services.V3 +{ + public static class DefaultBlobRequestOptions + { + public static BlobRequestOptions Create() + { + return new BlobRequestOptions + { + ServerTimeout = TimeSpan.FromMinutes(2), + MaximumExecutionTime = TimeSpan.FromMinutes(10), + LocationMode = LocationMode.PrimaryThenSecondary, + RetryPolicy = new ExponentialRetry(), + }; + } + } +} diff --git a/src/NuGet.Services.V3/DependencyInjectionExtensions.cs b/src/NuGet.Services.V3/DependencyInjectionExtensions.cs new file mode 100644 index 000000000..8ccf9cbba --- /dev/null +++ b/src/NuGet.Services.V3/DependencyInjectionExtensions.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NuGet.Jobs.Validation; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services.Metadata.Catalog; +using NuGetGallery.Diagnostics; + +namespace NuGet.Services.V3 +{ + public static class DependencyInjectionExtensions + { + public static IServiceCollection AddV3(this IServiceCollection services) + { + services + .AddTransient(p => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + + services + .AddTransient(p => (HttpMessageHandler)new TelemetryHandler( + p.GetRequiredService(), + p.GetRequiredService())); + + services.AddSingleton(p => new HttpClient(p.GetRequiredService())); + + services + .AddTransient(p => new CatalogClient( + p.GetRequiredService(), + p.GetRequiredService>())); + + services.AddTransient(); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + return services; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICollector.cs b/src/NuGet.Services.V3/ICollector.cs similarity index 88% rename from src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICollector.cs rename to src/NuGet.Services.V3/ICollector.cs index f95e94a8d..7955dfd2a 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICollector.cs +++ b/src/NuGet.Services.V3/ICollector.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using NuGet.Services.Metadata.Catalog; -namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +namespace NuGet.Services.V3 { public interface ICollector { diff --git a/src/NuGet.Services.V3/ICommitCollectorConfiguration.cs b/src/NuGet.Services.V3/ICommitCollectorConfiguration.cs new file mode 100644 index 000000000..026ec37cf --- /dev/null +++ b/src/NuGet.Services.V3/ICommitCollectorConfiguration.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.V3 +{ + public interface ICommitCollectorConfiguration + { + int MaxConcurrentCatalogLeafDownloads { get; } + string Source { get; } + TimeSpan HttpClientTimeout { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICommitCollectorLogic.cs b/src/NuGet.Services.V3/ICommitCollectorLogic.cs similarity index 90% rename from src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICommitCollectorLogic.cs rename to src/NuGet.Services.V3/ICommitCollectorLogic.cs index e21dcb63d..3476a25da 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICommitCollectorLogic.cs +++ b/src/NuGet.Services.V3/ICommitCollectorLogic.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using NuGet.Services.Metadata.Catalog; -namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +namespace NuGet.Services.V3 { public interface ICommitCollectorLogic { diff --git a/src/NuGet.Services.V3/IV3TelemetryService.cs b/src/NuGet.Services.V3/IV3TelemetryService.cs new file mode 100644 index 000000000..a781b4291 --- /dev/null +++ b/src/NuGet.Services.V3/IV3TelemetryService.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.V3 +{ + public interface IV3TelemetryService + { + IDisposable TrackCatalogLeafDownloadBatch(int count); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.V3/NuGet.Services.V3.csproj b/src/NuGet.Services.V3/NuGet.Services.V3.csproj new file mode 100644 index 000000000..2975611d9 --- /dev/null +++ b/src/NuGet.Services.V3/NuGet.Services.V3.csproj @@ -0,0 +1,96 @@ + + + + + + Debug + AnyCPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B} + Library + Properties + NuGet.Services.V3 + NuGet.Services.V3 + v4.7.2 + 512 + true + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 4.3.0-dev-3159955 + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {DDB34145-870F-42C3-9663-A9390CEE1E35} + NuGet.Indexing + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/NuGet.Services.V3/Properties/AssemblyInfo.cs b/src/NuGet.Services.V3/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..0c48a92ed --- /dev/null +++ b/src/NuGet.Services.V3/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.V3")] +[assembly: ComVisible(false)] +[assembly: Guid("c3f9a738-9759-4b2b-a50d-6507b28a659b")] diff --git a/src/NuGet.Services.AzureSearch/Registration/IRegistrationClient.cs b/src/NuGet.Services.V3/Registration/IRegistrationClient.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/IRegistrationClient.cs rename to src/NuGet.Services.V3/Registration/IRegistrationClient.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/Models/RegistrationCatalogEntry.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationCatalogEntry.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/Models/RegistrationCatalogEntry.cs rename to src/NuGet.Services.V3/Registration/Models/RegistrationCatalogEntry.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/Models/RegistrationIndex.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationIndex.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/Models/RegistrationIndex.cs rename to src/NuGet.Services.V3/Registration/Models/RegistrationIndex.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/Models/RegistrationLeaf.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationLeaf.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/Models/RegistrationLeaf.cs rename to src/NuGet.Services.V3/Registration/Models/RegistrationLeaf.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/Models/RegistrationLeafItem.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationLeafItem.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/Models/RegistrationLeafItem.cs rename to src/NuGet.Services.V3/Registration/Models/RegistrationLeafItem.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/Models/RegistrationPage.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationPage.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/Models/RegistrationPage.cs rename to src/NuGet.Services.V3/Registration/Models/RegistrationPage.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/RegistrationClient.cs b/src/NuGet.Services.V3/Registration/RegistrationClient.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/RegistrationClient.cs rename to src/NuGet.Services.V3/Registration/RegistrationClient.cs diff --git a/src/NuGet.Services.AzureSearch/Registration/RegistrationUrlBuilder.cs b/src/NuGet.Services.V3/Registration/RegistrationUrlBuilder.cs similarity index 100% rename from src/NuGet.Services.AzureSearch/Registration/RegistrationUrlBuilder.cs rename to src/NuGet.Services.V3/Registration/RegistrationUrlBuilder.cs diff --git a/src/NuGet.Services.AzureSearch/VersionList/Guard.cs b/src/NuGet.Services.V3/Support/Guard.cs similarity index 94% rename from src/NuGet.Services.AzureSearch/VersionList/Guard.cs rename to src/NuGet.Services.V3/Support/Guard.cs index d04debb94..83cc15d4c 100644 --- a/src/NuGet.Services.AzureSearch/VersionList/Guard.cs +++ b/src/NuGet.Services.V3/Support/Guard.cs @@ -3,9 +3,9 @@ using System; -namespace NuGet.Services.AzureSearch +namespace NuGet.Services { - internal static class Guard + public static class Guard { /// /// We could use here, but it's preferable in this diff --git a/src/NuGet.Services.AzureSearch/IdAndValue.cs b/src/NuGet.Services.V3/Support/IdAndValue.cs similarity index 86% rename from src/NuGet.Services.AzureSearch/IdAndValue.cs rename to src/NuGet.Services.V3/Support/IdAndValue.cs index 93b4234ab..831ab6125 100644 --- a/src/NuGet.Services.AzureSearch/IdAndValue.cs +++ b/src/NuGet.Services.V3/Support/IdAndValue.cs @@ -3,9 +3,9 @@ using System; -namespace NuGet.Services.AzureSearch +namespace NuGet.Services { - internal class IdAndValue + public class IdAndValue { public IdAndValue(string id, T value) { diff --git a/src/NuGet.Services.AzureSearch/VersionList/KeyValuePair.cs b/src/NuGet.Services.V3/Support/KeyValuePair.cs similarity index 84% rename from src/NuGet.Services.AzureSearch/VersionList/KeyValuePair.cs rename to src/NuGet.Services.V3/Support/KeyValuePair.cs index c0eda4565..177f08df0 100644 --- a/src/NuGet.Services.AzureSearch/VersionList/KeyValuePair.cs +++ b/src/NuGet.Services.V3/Support/KeyValuePair.cs @@ -3,9 +3,9 @@ using System.Collections.Generic; -namespace NuGet.Services.AzureSearch +namespace NuGet.Services { - internal static class KeyValuePair + public static class KeyValuePair { public static KeyValuePair Create(TKey key, TValue value) { diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ReferenceEqualityComparer.cs b/src/NuGet.Services.V3/Support/ReferenceEqualityComparer.cs similarity index 88% rename from src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ReferenceEqualityComparer.cs rename to src/NuGet.Services.V3/Support/ReferenceEqualityComparer.cs index a352d4a1a..366baf7c5 100644 --- a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ReferenceEqualityComparer.cs +++ b/src/NuGet.Services.V3/Support/ReferenceEqualityComparer.cs @@ -2,9 +2,10 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.CompilerServices; -namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +namespace NuGet.Services { /// /// Source: https://stackoverflow.com/a/35520207 @@ -17,11 +18,13 @@ private ReferenceEqualityComparer() { } + [DebuggerStepThrough] public bool Equals(T x, T y) { return ReferenceEquals(x, y); } + [DebuggerStepThrough] public int GetHashCode(T obj) { return RuntimeHelpers.GetHashCode(obj); diff --git a/src/NuGet.Services.V3/V3TelemetryService.cs b/src/NuGet.Services.V3/V3TelemetryService.cs new file mode 100644 index 000000000..d91d092e9 --- /dev/null +++ b/src/NuGet.Services.V3/V3TelemetryService.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Logging; + +namespace NuGet.Services.V3 +{ + public class V3TelemetryService : IV3TelemetryService + { + private const string Prefix = "V3."; + + private readonly ITelemetryClient _telemetryClient; + + public V3TelemetryService(ITelemetryClient telemetryClient) + { + _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + } + + public IDisposable TrackCatalogLeafDownloadBatch(int count) + { + return _telemetryClient.TrackDuration( + Prefix + "CatalogLeafDownloadBatchSeconds", + new Dictionary + { + { "Count", count.ToString() }, + }); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs index 158e92cd9..875e77a90 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Options; using Moq; using NuGet.Services.AzureSearch.AuxiliaryFiles; -using NuGet.Services.AzureSearch.Support; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs index b12f4746f..039f0ad4b 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs @@ -10,7 +10,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Moq; using NuGet.Services.AzureSearch.Db2AzureSearch; -using NuGet.Services.AzureSearch.Support; using NuGetGallery; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs index 3b3faf833..a69369681 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs @@ -13,7 +13,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.AzureSearch.Support; using NuGetGallery; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs index bed782038..67c53e88a 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs @@ -15,7 +15,6 @@ using Moq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.AzureSearch.Support; using NuGetGallery; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs index 43a8ad864..29d36ac3a 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs @@ -13,7 +13,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.AzureSearch.Support; using NuGetGallery; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs index 644148721..ebd8b2057 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs @@ -12,7 +12,6 @@ using Microsoft.Rest; using Microsoft.Rest.Azure; using Moq; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.AzureSearch.Wrappers; using NuGetGallery; using Xunit; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs index 59521adf7..c105b0a57 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs @@ -8,8 +8,8 @@ using Moq; using NuGet.Packaging.Core; using NuGet.Protocol.Catalog; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; using NuGet.Versioning; using Xunit; using Xunit.Abstractions; @@ -238,10 +238,15 @@ public abstract class BaseFacts protected readonly Mock _catalogClient; protected readonly Mock _catalogIndexActionBuilder; protected readonly Mock _batchPusher; - protected readonly Mock> _options; - protected readonly Catalog2AzureSearchConfiguration _config; + protected readonly CommitCollectorUtility _utility; + protected readonly Mock> _utilityOptions; + protected readonly Mock> _collectorOptions; + protected readonly CommitCollectorConfiguration _utilityConfig; + protected readonly Catalog2AzureSearchConfiguration _collectorConfig; protected readonly Mock _telemetryService; + protected readonly Mock _v3TelemetryService; protected readonly RecordingLogger _logger; + protected readonly RecordingLogger _utilityLogger; protected readonly AzureSearchCollectorLogic _target; public BaseFacts(ITestOutputHelper output) @@ -249,20 +254,31 @@ public BaseFacts(ITestOutputHelper output) _catalogClient = new Mock(); _catalogIndexActionBuilder = new Mock(); _batchPusher = new Mock(); - _options = new Mock>(); - _config = new Catalog2AzureSearchConfiguration(); + _utilityOptions = new Mock>(); + _collectorOptions = new Mock>(); + _utilityConfig = new CommitCollectorConfiguration(); + _collectorConfig = new Catalog2AzureSearchConfiguration(); _telemetryService = new Mock(); + _v3TelemetryService = new Mock(); _logger = output.GetLogger(); + _utilityLogger = output.GetLogger(); - _options.Setup(x => x.Value).Returns(() => _config); - _config.MaxConcurrentBatches = 1; - _config.MaxConcurrentCatalogLeafDownloads = 1; + _utilityOptions.Setup(x => x.Value).Returns(() => _utilityConfig); + _utilityConfig.MaxConcurrentCatalogLeafDownloads = 1; + _collectorOptions.Setup(x => x.Value).Returns(() => _collectorConfig); + _collectorConfig.MaxConcurrentBatches = 1; - _target = new AzureSearchCollectorLogic( + _utility = new CommitCollectorUtility( _catalogClient.Object, + _v3TelemetryService.Object, + _utilityOptions.Object, + _utilityLogger); + + _target = new AzureSearchCollectorLogic( _catalogIndexActionBuilder.Object, () => _batchPusher.Object, - _options.Object, + _utility, + _collectorOptions.Object, _telemetryService.Object, _logger); } diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs index 4d158a8f2..1e5e5e76f 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs @@ -10,9 +10,9 @@ using Microsoft.Extensions.Options; using Moq; using Newtonsoft.Json; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.Metadata.Catalog; using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs index 0828a399a..a37c786cf 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs @@ -8,7 +8,6 @@ using Moq; using NuGet.Protocol.Catalog; using NuGet.Protocol.Registration; -using NuGet.Services.AzureSearch.Support; using NuGet.Versioning; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs index d8b7b105b..5d15e840e 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs @@ -18,6 +18,7 @@ using NuGet.Services.Entities; using NuGet.Services.Logging; using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; using NuGet.Versioning; using NuGetGallery; using Xunit; @@ -27,10 +28,13 @@ namespace NuGet.Services.AzureSearch.Catalog2AzureSearch.Integration { public class AzureSearchCollectorLogicIntegrationTests { + private CommitCollectorConfiguration _utilityConfig; + private Mock> _utilityOptions; private Catalog2AzureSearchConfiguration _config; private Mock> _options; private Mock _telemetryClient; private AzureSearchTelemetryService _telemetryService; + private V3TelemetryService _v3TelemetryService; private Mock _entitiesContextFactory; private Mock _entitiesContext; private DatabaseOwnerFetcher _ownerFetcher; @@ -47,15 +51,22 @@ public class AzureSearchCollectorLogicIntegrationTests private InMemoryDocumentsOperations _searchDocuments; private Mock _hijackIndex; private InMemoryDocumentsOperations _hijackDocuments; + private CommitCollectorUtility _commitCollectorUtility; private AzureSearchCollectorLogic _collector; public AzureSearchCollectorLogicIntegrationTests(ITestOutputHelper output) { + _utilityConfig = new CommitCollectorConfiguration + { + MaxConcurrentCatalogLeafDownloads = 1, + }; + _utilityOptions = new Mock>(); + _utilityOptions.Setup(x => x.Value).Returns(() => _utilityConfig); + _config = new Catalog2AzureSearchConfiguration { MaxConcurrentBatches = 1, MaxConcurrentVersionListWriters = 1, - MaxConcurrentCatalogLeafDownloads = 1, StorageContainer = "integration-tests-container", StoragePath = "integration-tests-path", RegistrationsBaseUrl = "https://example/registrations/", @@ -70,6 +81,7 @@ public AzureSearchCollectorLogicIntegrationTests(ITestOutputHelper output) _telemetryClient = new Mock(); _telemetryService = new AzureSearchTelemetryService(_telemetryClient.Object); + _v3TelemetryService = new V3TelemetryService(_telemetryClient.Object); // Mock the database that is used for fetching owner information. The product code only reads // from the database so it is less important to have a realistic, stateful implementation. @@ -114,8 +126,13 @@ public AzureSearchCollectorLogicIntegrationTests(ITestOutputHelper output) _hijackDocuments = new InMemoryDocumentsOperations(); _hijackIndex.Setup(x => x.Documents).Returns(() => _hijackDocuments); - _collector = new AzureSearchCollectorLogic( + _commitCollectorUtility = new CommitCollectorUtility( _catalogClient, + _v3TelemetryService, + _utilityOptions.Object, + output.GetLogger()); + + _collector = new AzureSearchCollectorLogic( _builder, () => new BatchPusher( _searchIndex.Object, @@ -124,6 +141,7 @@ public AzureSearchCollectorLogicIntegrationTests(ITestOutputHelper output) _options.Object, _telemetryService, output.GetLogger()), + _commitCollectorUtility, _options.Object, _telemetryService, output.GetLogger()); diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs index 3a3a2648c..9b8837ff5 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs @@ -12,7 +12,6 @@ using Moq; using NuGet.Protocol.Catalog; using NuGet.Services.AzureSearch.AuxiliaryFiles; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.Entities; using NuGet.Services.Metadata.Catalog.Persistence; using NuGetGallery; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs index 87ca0eb77..8c7408e35 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs @@ -13,7 +13,6 @@ using Microsoft.WindowsAzure.Storage; using Moq; using NuGet.Services.AzureSearch.AuxiliaryFiles; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.Entities; using NuGetGallery; using Xunit; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs index 20c2bae48..37cd523c7 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs @@ -3,7 +3,6 @@ using Microsoft.Azure.Search.Models; using Moq; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.Entities; using NuGet.Versioning; using NuGetGallery; diff --git a/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs index b2b9ce2bd..9c91396ca 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs @@ -14,7 +14,6 @@ using Moq; using NuGet.Services.AzureSearch.ScoringProfiles; using NuGet.Services.AzureSearch.SearchService; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.AzureSearch.Wrappers; using NuGetGallery; using Xunit; diff --git a/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj b/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj index fb55c10d1..6acdf1930 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj +++ b/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj @@ -71,9 +71,7 @@ - - @@ -85,18 +83,8 @@ - - - - - - - - - - @@ -123,9 +111,13 @@ {1a53fe3d-8041-4773-942f-d73aef5b82b2} NuGet.Services.AzureSearch - - {1f3bc053-796c-4a35-88f4-955a0f142197} - NuGet.Protocol.Catalog.Tests + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + + + {ccb4d5ef-ac84-449d-ac6e-0a0ad295483a} + NuGet.Services.V3.Tests diff --git a/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/OwnerSetComparerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/OwnerSetComparerFacts.cs index 0aabad3ab..e21fe09e5 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/OwnerSetComparerFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/OwnerSetComparerFacts.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Linq; using Moq; -using NuGet.Services.AzureSearch.Support; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/Owners2AzureSearchCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/Owners2AzureSearchCommandFacts.cs index 9279e5a8d..26da89ff2 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/Owners2AzureSearchCommandFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Owners2AzureSearch/Owners2AzureSearchCommandFacts.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Options; using Moq; using NuGet.Services.AzureSearch.AuxiliaryFiles; -using NuGet.Services.AzureSearch.Support; using NuGetGallery; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs index b551e34e3..83078960b 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Options; using Moq; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.AzureSearch.Wrappers; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs index b2d4b1bf0..1589be21e 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Options; using Moq; using NuGet.Services.AzureSearch.AuxiliaryFiles; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.AzureSearch.Wrappers; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs index 9bc2afba8..6509a3b86 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Options; using Moq; -using NuGet.Services.AzureSearch.Support; using NuGet.Services.AzureSearch.Wrappers; using NuGet.Services.KeyVault; using Xunit; diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs index 4f9a8e2e4..dbb783142 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs @@ -12,7 +12,6 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Moq; -using NuGet.Services.AzureSearch.Support; using NuGetGallery; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs index 83861f241..e88c0f14c 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs @@ -14,7 +14,6 @@ using Microsoft.Rest.Azure; using Moq; using Moq.Language; -using NuGet.Services.AzureSearch.Support; using Xunit; using Xunit.Abstractions; diff --git a/tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj b/tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj new file mode 100644 index 000000000..13e3b4206 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj @@ -0,0 +1,91 @@ + + + + + Debug + AnyCPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A} + Library + Properties + NuGet.Services.V3 + NuGet.Services.V3.Tests + v4.7.2 + 512 + true + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + 4.10.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + + + {1f3bc053-796c-4a35-88f4-955a0f142197} + NuGet.Protocol.Catalog.Tests + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..110538383 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.V3.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("ccb4d5ef-ac84-449d-ac6e-0a0ad295483a")] diff --git a/tests/NuGet.Services.AzureSearch.Tests/Registration/RegistrationClientFacts.cs b/tests/NuGet.Services.V3.Tests/Registration/RegistrationClientFacts.cs similarity index 100% rename from tests/NuGet.Services.AzureSearch.Tests/Registration/RegistrationClientFacts.cs rename to tests/NuGet.Services.V3.Tests/Registration/RegistrationClientFacts.cs diff --git a/tests/NuGet.Services.AzureSearch.Tests/Registration/RegistrationUrlBuilderFacts.cs b/tests/NuGet.Services.V3.Tests/Registration/RegistrationUrlBuilderFacts.cs similarity index 100% rename from tests/NuGet.Services.AzureSearch.Tests/Registration/RegistrationUrlBuilderFacts.cs rename to tests/NuGet.Services.V3.Tests/Registration/RegistrationUrlBuilderFacts.cs diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/Cursor.cs b/tests/NuGet.Services.V3.Tests/Support/Cursor.cs similarity index 87% rename from tests/NuGet.Services.AzureSearch.Tests/Support/Cursor.cs rename to tests/NuGet.Services.V3.Tests/Support/Cursor.cs index af3eda86c..e6b6202de 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/Cursor.cs +++ b/tests/NuGet.Services.V3.Tests/Support/Cursor.cs @@ -4,7 +4,7 @@ using System; using Newtonsoft.Json; -namespace NuGet.Services.AzureSearch.Support +namespace NuGet.Services { public class Cursor { diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/DbSetMockFactory.cs b/tests/NuGet.Services.V3.Tests/Support/DbSetMockFactory.cs similarity index 97% rename from tests/NuGet.Services.AzureSearch.Tests/Support/DbSetMockFactory.cs rename to tests/NuGet.Services.V3.Tests/Support/DbSetMockFactory.cs index 4b5521565..c81cb446f 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/DbSetMockFactory.cs +++ b/tests/NuGet.Services.V3.Tests/Support/DbSetMockFactory.cs @@ -7,7 +7,7 @@ using System.Linq; using Moq; -namespace NuGet.Services.AzureSearch.Support +namespace NuGet.Services { public static class DbSetMockFactory { diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/RecordingLogger.cs b/tests/NuGet.Services.V3.Tests/Support/RecordingLogger.cs similarity index 96% rename from tests/NuGet.Services.AzureSearch.Tests/Support/RecordingLogger.cs rename to tests/NuGet.Services.V3.Tests/Support/RecordingLogger.cs index 59c65abe4..e65233692 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/RecordingLogger.cs +++ b/tests/NuGet.Services.V3.Tests/Support/RecordingLogger.cs @@ -7,7 +7,7 @@ using System.Linq; using Microsoft.Extensions.Logging; -namespace NuGet.Services.AzureSearch.Support +namespace NuGet.Services { public class RecordingLogger : ILogger { diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/TestCursorStorage.cs b/tests/NuGet.Services.V3.Tests/Support/TestCursorStorage.cs similarity index 98% rename from tests/NuGet.Services.AzureSearch.Tests/Support/TestCursorStorage.cs rename to tests/NuGet.Services.V3.Tests/Support/TestCursorStorage.cs index 564c30ffc..04589234d 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/TestCursorStorage.cs +++ b/tests/NuGet.Services.V3.Tests/Support/TestCursorStorage.cs @@ -9,7 +9,7 @@ using Newtonsoft.Json; using NuGet.Services.Metadata.Catalog.Persistence; -namespace NuGet.Services.AzureSearch.Support +namespace NuGet.Services { public class TestCursorStorage : Metadata.Catalog.Persistence.Storage { diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/TestDbAsyncQueryProvider.cs b/tests/NuGet.Services.V3.Tests/Support/TestDbAsyncQueryProvider.cs similarity index 98% rename from tests/NuGet.Services.AzureSearch.Tests/Support/TestDbAsyncQueryProvider.cs rename to tests/NuGet.Services.V3.Tests/Support/TestDbAsyncQueryProvider.cs index 48531f212..10518b91e 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/TestDbAsyncQueryProvider.cs +++ b/tests/NuGet.Services.V3.Tests/Support/TestDbAsyncQueryProvider.cs @@ -8,7 +8,7 @@ using System.Threading; using System.Threading.Tasks; -namespace NuGet.Services.AzureSearch.Support +namespace NuGet.Services { // Copied from https://msdn.microsoft.com/en-us/library/dn314429.aspx public class TestDbAsyncQueryProvider : IDbAsyncQueryProvider diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/TestHttpMessageHandler.cs b/tests/NuGet.Services.V3.Tests/Support/TestHttpMessageHandler.cs similarity index 94% rename from tests/NuGet.Services.AzureSearch.Tests/Support/TestHttpMessageHandler.cs rename to tests/NuGet.Services.V3.Tests/Support/TestHttpMessageHandler.cs index da431b627..65abe41f4 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/TestHttpMessageHandler.cs +++ b/tests/NuGet.Services.V3.Tests/Support/TestHttpMessageHandler.cs @@ -6,7 +6,7 @@ using System.Threading; using System.Threading.Tasks; -namespace NuGet.Services.AzureSearch.Support +namespace NuGet.Services { public class TestHttpMessageHandler : HttpMessageHandler { diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/TestOutputHelperExtensions.cs b/tests/NuGet.Services.V3.Tests/Support/TestOutputHelperExtensions.cs similarity index 93% rename from tests/NuGet.Services.AzureSearch.Tests/Support/TestOutputHelperExtensions.cs rename to tests/NuGet.Services.V3.Tests/Support/TestOutputHelperExtensions.cs index de8d79df9..ec5bb11dd 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/Support/TestOutputHelperExtensions.cs +++ b/tests/NuGet.Services.V3.Tests/Support/TestOutputHelperExtensions.cs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Microsoft.Extensions.Logging; -using NuGet.Services.AzureSearch.Support; +using NuGet.Services; namespace Xunit.Abstractions { diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/XunitLogger.cs b/tests/NuGet.Services.V3.Tests/Support/XunitLogger.cs similarity index 100% rename from tests/NuGet.Services.AzureSearch.Tests/Support/XunitLogger.cs rename to tests/NuGet.Services.V3.Tests/Support/XunitLogger.cs diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/XunitLoggerFactoryExtensions.cs b/tests/NuGet.Services.V3.Tests/Support/XunitLoggerFactoryExtensions.cs similarity index 100% rename from tests/NuGet.Services.AzureSearch.Tests/Support/XunitLoggerFactoryExtensions.cs rename to tests/NuGet.Services.V3.Tests/Support/XunitLoggerFactoryExtensions.cs diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/XunitLoggerProvider.cs b/tests/NuGet.Services.V3.Tests/Support/XunitLoggerProvider.cs similarity index 100% rename from tests/NuGet.Services.AzureSearch.Tests/Support/XunitLoggerProvider.cs rename to tests/NuGet.Services.V3.Tests/Support/XunitLoggerProvider.cs