From ae0d22f66487c9359441a48959ca86c7722f43ea Mon Sep 17 00:00:00 2001 From: Damon Tivel Date: Mon, 19 Nov 2018 19:16:16 -0800 Subject: [PATCH] Catalog2Dnx: parallelize commit processing (#412) Progress on https://github.com/NuGet/Engineering/issues/1843. --- src/Catalog/Dnx/DnxCatalogCollector.cs | 92 ++++- .../Persistence/AzureCloudBlockBlob.cs | 1 + src/Catalog/Persistence/AzureStorage.cs | 71 ++-- src/Catalog/Persistence/ICloudBlockBlob.cs | 1 + src/Catalog/Telemetry/TelemetryConstants.cs | 1 + src/Ng/Jobs/Catalog2DnxJob.cs | 2 + src/Ng/Jobs/NgJob.cs | 2 +- .../Dnx/DnxCatalogCollectorTests.cs | 359 ++++++++++++++++-- .../PackageCatalogItemCreatorTests.cs | 1 - tests/NgTests/Data/Catalogs.cs | 19 - .../Data/TestCatalogEntries.Designer.cs | 74 ---- tests/NgTests/Data/TestCatalogEntries.resx | 163 -------- tests/NgTests/Feed2CatalogTests.cs | 2 +- .../Infrastructure/MockTelemetryService.cs | 25 +- tests/NgTests/Infrastructure/TestPackage.cs | 22 +- 15 files changed, 486 insertions(+), 349 deletions(-) diff --git a/src/Catalog/Dnx/DnxCatalogCollector.cs b/src/Catalog/Dnx/DnxCatalogCollector.cs index d7feb131a..c3448287d 100644 --- a/src/Catalog/Dnx/DnxCatalogCollector.cs +++ b/src/Catalog/Dnx/DnxCatalogCollector.cs @@ -7,6 +7,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -24,7 +25,8 @@ public class DnxCatalogCollector : CommitCollector private readonly IAzureStorage _sourceStorage; private readonly DnxMaker _dnxMaker; private readonly ILogger _logger; - private readonly int _maxDegreeOfParallelism; + private readonly int _maxConcurrentBatches; + private readonly int _maxConcurrentCommitItemsWithinBatch; private readonly Uri _contentBaseAddress; public DnxCatalogCollector( @@ -52,7 +54,43 @@ public DnxCatalogCollector( string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue)); } - _maxDegreeOfParallelism = maxDegreeOfParallelism; + // Find two factors which are close or equal to each other. + var squareRoot = Math.Sqrt(maxDegreeOfParallelism); + + // If the max degree of parallelism is a perfect square, great. + // Otherwise, prefer a greater degree of parallelism in batches than commit items within a batch. + _maxConcurrentBatches = Convert.ToInt32(Math.Ceiling(squareRoot)); + _maxConcurrentCommitItemsWithinBatch = Convert.ToInt32(maxDegreeOfParallelism / _maxConcurrentBatches); + + ServicePointManager.DefaultConnectionLimit = _maxConcurrentBatches * _maxConcurrentCommitItemsWithinBatch; + } + + protected override Task> CreateBatchesAsync( + IEnumerable catalogItems) + { + var batches = CatalogCommitUtilities.CreateCommitItemBatches( + catalogItems, + CatalogCommitUtilities.GetPackageIdKey); + + return Task.FromResult(batches); + } + + protected override Task FetchAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + CancellationToken cancellationToken) + { + return CatalogCommitUtilities.ProcessCatalogCommitsAsync( + client, + front, + back, + FetchCatalogCommitsAsync, + CreateBatchesAsync, + ProcessBatchAsync, + _maxConcurrentBatches, + _logger, + cancellationToken); } protected override async Task OnProcessBatchAsync( @@ -66,7 +104,7 @@ protected override async Task OnProcessBatchAsync( var catalogEntries = items.Select(item => CatalogEntry.Create(item)) .ToList(); - // Sanity check: a single catalog batch should not contain multiple entries for the same package ID and version. + // Sanity check: a single catalog batch should not contain multiple entries for the same package identity. AssertNoMultipleEntriesForSamePackageIdentity(commitTimeStamp, catalogEntries); // Process .nupkg/.nuspec adds and deletes. @@ -85,16 +123,16 @@ private async Task> ProcessCatalogEntriesAsync( { var processedCatalogEntries = new ConcurrentBag(); - await catalogEntries.ForEachAsync(_maxDegreeOfParallelism, async catalogEntry => + await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntry => { var packageId = catalogEntry.PackageId; var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion; if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri) { - var properties = GetTelemetryProperties(catalogEntry); + var telemetryProperties = GetTelemetryProperties(catalogEntry); - using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, properties)) + using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, telemetryProperties)) { var packageFileName = PackageUtility.GetPackageFileName( packageId, @@ -127,6 +165,7 @@ await catalogEntries.ForEachAsync(_maxDegreeOfParallelism, async catalogEntry => packageId, normalizedPackageVersion, sourceUri, + telemetryProperties, cancellationToken)) { processedCatalogEntries.Add(catalogEntry); @@ -170,7 +209,7 @@ private async Task UpdatePackageVersionIndexAsync( { var catalogEntryGroups = catalogEntries.GroupBy(catalogEntry => catalogEntry.PackageId); - await catalogEntryGroups.ForEachAsync(_maxDegreeOfParallelism, async catalogEntryGroup => + await catalogEntryGroups.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntryGroup => { var packageId = catalogEntryGroup.Key; var properties = new Dictionary() @@ -209,11 +248,13 @@ private async Task ProcessPackageDetailsAsync( string packageId, string normalizedPackageVersion, Uri sourceUri, + Dictionary telemetryProperties, CancellationToken cancellationToken) { if (await ProcessPackageDetailsViaStorageAsync( packageId, normalizedPackageVersion, + telemetryProperties, cancellationToken)) { return true; @@ -229,12 +270,14 @@ private async Task ProcessPackageDetailsAsync( packageId, normalizedPackageVersion, sourceUri, + telemetryProperties, cancellationToken); } private async Task ProcessPackageDetailsViaStorageAsync( string packageId, string normalizedPackageVersion, + Dictionary telemetryProperties, CancellationToken cancellationToken) { if (_sourceStorage == null) @@ -255,6 +298,8 @@ private async Task ProcessPackageDetailsViaStorageAsync( // If the ETag's differ, we'll fall back to using a single HTTP GET request. var token1 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + telemetryProperties[TelemetryConstants.SizeInBytes] = sourceBlob.Length.ToString(); + var nuspec = await GetNuspecAsync(sourceBlob, packageId, cancellationToken); if (string.IsNullOrEmpty(nuspec)) @@ -306,6 +351,7 @@ private async Task ProcessPackageDetailsViaHttpAsync( string id, string version, Uri sourceUri, + Dictionary telemetryProperties, CancellationToken cancellationToken) { var packageDownloader = new PackageDownloader(client, _logger); @@ -320,6 +366,8 @@ private async Task ProcessPackageDetailsViaHttpAsync( return false; } + telemetryProperties[TelemetryConstants.SizeInBytes] = stream.Length.ToString(); + var nuspec = GetNuspec(stream, id); if (nuspec == null) @@ -380,7 +428,7 @@ private static void AssertNoMultipleEntriesForSamePackageIdentity( } } - private static async Task GetNuspecAsync( + private async Task GetNuspecAsync( ICloudBlockBlob sourceBlob, string packageId, CancellationToken cancellationToken) @@ -438,6 +486,34 @@ private static Dictionary GetTelemetryProperties(string packageI }; } + private async Task ProcessBatchAsync( + CollectorHttpClient client, + JToken context, + string packageId, + CatalogCommitItemBatch batch, + CatalogCommitItemBatch lastBatch, + CancellationToken cancellationToken) + { + await Task.Yield(); + + using (_telemetryService.TrackDuration( + TelemetryConstants.ProcessBatchSeconds, + new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() } + })) + { + await OnProcessBatchAsync( + client, + batch.Items, + context, + batch.CommitTimeStamp, + isLastBatch: false, + cancellationToken: cancellationToken); + } + } + private sealed class CatalogEntry { internal DateTime CommitTimeStamp { get; } diff --git a/src/Catalog/Persistence/AzureCloudBlockBlob.cs b/src/Catalog/Persistence/AzureCloudBlockBlob.cs index 0071e29ad..4e0a431df 100644 --- a/src/Catalog/Persistence/AzureCloudBlockBlob.cs +++ b/src/Catalog/Persistence/AzureCloudBlockBlob.cs @@ -26,6 +26,7 @@ public string ContentMD5 } public string ETag => _blob.Properties.ETag; + public long Length => _blob.Properties.Length; public Uri Uri => _blob.Uri; public AzureCloudBlockBlob(CloudBlockBlob blob) diff --git a/src/Catalog/Persistence/AzureStorage.cs b/src/Catalog/Persistence/AzureStorage.cs index 714b8705d..444143df4 100644 --- a/src/Catalog/Persistence/AzureStorage.cs +++ b/src/Catalog/Persistence/AzureStorage.cs @@ -20,7 +20,6 @@ public class AzureStorage : Storage, IAzureStorage { private readonly bool _compressContent; private readonly CloudBlobDirectory _directory; - private readonly BlobRequestOptions _blobRequestOptions; private readonly bool _useServerSideCopy; public static readonly TimeSpan DefaultServerTimeout = TimeSpan.FromSeconds(30); @@ -72,22 +71,24 @@ private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, TimeSpan max { _directory = directory; + // Unless overridden at the level of a single API call, these options will apply to all service calls that + // use BlobRequestOptions. + _directory.ServiceClient.DefaultRequestOptions = new BlobRequestOptions() + { + ServerTimeout = serverTimeout, + MaximumExecutionTime = maxExecutionTime, + RetryPolicy = new ExponentialRetry() + }; + if (_directory.Container.CreateIfNotExists()) { _directory.Container.SetPermissions(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); if (Verbose) { - Trace.WriteLine(String.Format("Created '{0}' publish container", _directory.Container.Name)); + Trace.WriteLine(string.Format("Created '{0}' publish container", _directory.Container.Name)); } } - - _blobRequestOptions = new BlobRequestOptions() - { - ServerTimeout = serverTimeout, - MaximumExecutionTime = maxExecutionTime, - RetryPolicy = new ExponentialRetry() - }; } public override async Task GetOptimisticConcurrencyControlTokenAsync( @@ -134,7 +135,7 @@ public override bool Exists(string fileName) } if (Verbose) { - Trace.WriteLine(String.Format("The blob {0} does not exist.", packageRegistrationUri)); + Trace.WriteLine(string.Format("The blob {0} does not exist.", packageRegistrationUri)); } return false; } @@ -230,11 +231,7 @@ protected override async Task OnSaveAsync(Uri resourceUri, StorageContent conten destinationStream.Seek(0, SeekOrigin.Begin); - await blob.UploadFromStreamAsync(destinationStream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); + await blob.UploadFromStreamAsync(destinationStream, cancellationToken); Trace.WriteLine(string.Format("Saved compressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); } @@ -243,11 +240,7 @@ await blob.UploadFromStreamAsync(destinationStream, { using (Stream stream = content.GetContentStream()) { - await blob.UploadFromStreamAsync(stream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); + await blob.UploadFromStreamAsync(stream, cancellationToken); } Trace.WriteLine(string.Format("Saved uncompressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); @@ -306,32 +299,30 @@ protected override async Task OnLoadAsync(Uri resourceUri, Cance if (blob.Exists()) { - MemoryStream originalStream = new MemoryStream(); - await blob.DownloadToStreamAsync(originalStream, - accessCondition: null, - options: _blobRequestOptions, - operationContext: null, - cancellationToken: cancellationToken); - - originalStream.Seek(0, SeekOrigin.Begin); - string content; - if (blob.Properties.ContentEncoding == "gzip") + using (var originalStream = new MemoryStream()) { - using (var uncompressedStream = new GZipStream(originalStream, CompressionMode.Decompress)) + await blob.DownloadToStreamAsync(originalStream, cancellationToken); + + originalStream.Seek(0, SeekOrigin.Begin); + + if (blob.Properties.ContentEncoding == "gzip") { - using (var reader = new StreamReader(uncompressedStream)) + using (var uncompressedStream = new GZipStream(originalStream, CompressionMode.Decompress)) { - content = await reader.ReadToEndAsync(); + using (var reader = new StreamReader(uncompressedStream)) + { + content = await reader.ReadToEndAsync(); + } } } - } - else - { - using (var reader = new StreamReader(originalStream)) + else { - content = await reader.ReadToEndAsync(); + using (var reader = new StreamReader(originalStream)) + { + content = await reader.ReadToEndAsync(); + } } } @@ -340,7 +331,7 @@ await blob.DownloadToStreamAsync(originalStream, if (Verbose) { - Trace.WriteLine(String.Format("Can't load '{0}'. Blob doesn't exist", resourceUri)); + Trace.WriteLine(string.Format("Can't load '{0}'. Blob doesn't exist", resourceUri)); } return null; @@ -353,7 +344,7 @@ protected override async Task OnDeleteAsync(Uri resourceUri, CancellationToken c CloudBlockBlob blob = _directory.GetBlockBlobReference(name); await blob.DeleteAsync(deleteSnapshotsOption: DeleteSnapshotsOption.IncludeSnapshots, accessCondition: null, - options: _blobRequestOptions, + options: null, operationContext: null, cancellationToken: cancellationToken); } diff --git a/src/Catalog/Persistence/ICloudBlockBlob.cs b/src/Catalog/Persistence/ICloudBlockBlob.cs index 1ec994b14..1e17a7b0d 100644 --- a/src/Catalog/Persistence/ICloudBlockBlob.cs +++ b/src/Catalog/Persistence/ICloudBlockBlob.cs @@ -15,6 +15,7 @@ public interface ICloudBlockBlob { string ContentMD5 { get; set; } string ETag { get; } + long Length { get; } Uri Uri { get; } Task ExistsAsync(CancellationToken cancellationToken); diff --git a/src/Catalog/Telemetry/TelemetryConstants.cs b/src/Catalog/Telemetry/TelemetryConstants.cs index 3ba1be6b0..ed0a64de7 100644 --- a/src/Catalog/Telemetry/TelemetryConstants.cs +++ b/src/Catalog/Telemetry/TelemetryConstants.cs @@ -39,6 +39,7 @@ public static class TelemetryConstants public const string ProcessPackageDetailsSeconds = "ProcessPackageDetailsSeconds"; public const string ProcessPackageVersionIndexSeconds = "ProcessPackageVersionIndexSeconds"; public const string RegistrationDeltaCount = "RegistrationDeltaCount"; + public const string SizeInBytes = "SizeInBytes"; public const string StatusCode = "StatusCode"; public const string Success = "Success"; public const string Uri = "Uri"; diff --git a/src/Ng/Jobs/Catalog2DnxJob.cs b/src/Ng/Jobs/Catalog2DnxJob.cs index 996951b63..6f9e7ac90 100644 --- a/src/Ng/Jobs/Catalog2DnxJob.cs +++ b/src/Ng/Jobs/Catalog2DnxJob.cs @@ -79,6 +79,8 @@ protected override void Init(IDictionary arguments, Cancellation preferredPackageSourceStorageFactory); Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout); + MaxDegreeOfParallelism = 256; + _collector = new DnxCatalogCollector( new Uri(source), storageFactory, diff --git a/src/Ng/Jobs/NgJob.cs b/src/Ng/Jobs/NgJob.cs index 973c2069b..9d65c0a5d 100644 --- a/src/Ng/Jobs/NgJob.cs +++ b/src/Ng/Jobs/NgJob.cs @@ -17,7 +17,7 @@ public abstract class NgJob protected ILoggerFactory LoggerFactory; protected ILogger Logger; - protected int MaxDegreeOfParallelism { get; } + protected int MaxDegreeOfParallelism { get; set; } protected NgJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) { diff --git a/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs b/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs index e3f27b92a..03a97e0fc 100644 --- a/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs +++ b/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs @@ -12,8 +12,12 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using CatalogTests.Helpers; using Microsoft.Extensions.Logging; using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; using NgTests.Data; using NgTests.Infrastructure; using NuGet.Services.Metadata.Catalog; @@ -25,22 +29,51 @@ namespace CatalogTests.Dnx { public class DnxCatalogCollectorTests { + private static readonly Uri _baseUri = new Uri("https://nuget.test"); private const string _nuspecData = "nuspec data"; private const int _maxDegreeOfParallelism = 20; private static readonly HttpContent _noContent = new ByteArrayContent(new byte[0]); private const IAzureStorage _nullPreferredPackageSourceStorage = null; private static readonly Uri _contentBaseAddress = new Uri("http://tempuri.org/packages/"); + private readonly JObject _contextKeyword = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.NuGet, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Items, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Item), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Parent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.CommitTimeStamp, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastCreated, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastEdited, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastDeleted, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); + private readonly JsonSerializerSettings _jsonSettings = new JsonSerializerSettings() + { + DateParseHandling = DateParseHandling.None, + NullValueHandling = NullValueHandling.Ignore + }; private MemoryStorage _catalogToDnxStorage; private TestStorageFactory _catalogToDnxStorageFactory; private MockServerHttpClientHandler _mockServer; private ILogger _logger; private DnxCatalogCollector _target; + private Random _random; private Uri _cursorJsonUri; public DnxCatalogCollectorTests() { - _catalogToDnxStorage = new MemoryStorage(); + _catalogToDnxStorage = new MemoryStorage(_baseUri); _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); _mockServer = new MockServerHttpClientHandler(); _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); @@ -59,6 +92,7 @@ public DnxCatalogCollectorTests() () => _mockServer); _cursorJsonUri = _catalogToDnxStorage.ResolveUri("cursor.json"); + _random = new Random(); } [Fact] @@ -630,11 +664,12 @@ public async Task RunAsync_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCod var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); - var exception = await Assert.ThrowsAsync( + var exception = await Assert.ThrowsAsync( () => _target.RunAsync(front, back, CancellationToken.None)); + Assert.IsType(exception.InnerException); Assert.Equal( $"Expected status code OK for package download, actual: {statusCode}", - exception.Message); + exception.InnerException.Message); Assert.Equal(0, _catalogToDnxStorage.Content.Count); } @@ -717,7 +752,9 @@ public async Task RunAsync_WhenExceptionOccurs_DoesNotSkipPackage(string catalog ReadCursor back = MemoryCursor.CreateMax(); // Act - await Assert.ThrowsAsync(() => _target.RunAsync(front, back, CancellationToken.None)); + var exception = await Assert.ThrowsAsync( + () => _target.RunAsync(front, back, CancellationToken.None)); + Assert.IsType(exception.InnerException); var cursorBeforeRetry = front.Value; await _target.RunAsync(front, back, CancellationToken.None); var cursorAfterRetry = front.Value; @@ -738,33 +775,296 @@ public async Task RunAsync_WhenExceptionOccurs_DoesNotSkipPackage(string catalog .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/anotherpackage/1.0.0/anotherpackage.1.0.0.nupkg")); Assert.NotNull(anotherPackage100.Key); - Assert.Equal(DateTime.Parse(expectedCursorBeforeRetry).ToUniversalTime(), cursorBeforeRetry); + Assert.Equal(MemoryCursor.MinValue, cursorBeforeRetry); Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry); } [Fact] - public async Task RunAsync_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Throws() + public async Task RunAsync_WithIdenticalCommitItems_ProcessesPackage() { - var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData); - var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); - var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); - var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); - var expectedNupkg = File.ReadAllBytes(@"Packages\ListedPackage.1.0.0.zip"); - var catalogStorage = Catalogs.CreateTestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatch(); - - await _mockServer.AddStorageAsync(catalogStorage); - - _mockServer.SetAction( - "/packages/listedpackage.1.0.0.nupkg", - request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(expectedNupkg) })); + using (var package = TestPackage.Create(_random)) + { + var catalogIndexUri = new Uri(_baseUri, "index.json"); + var catalogPageUri = new Uri(_baseUri, "page0.json"); + + var commitId = Guid.NewGuid().ToString(); + var commitTimeStamp = DateTimeOffset.UtcNow; + var independentPackageDetails0 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId, + commitTimeStamp); + var independentPackageDetails1 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId, + commitTimeStamp); + var packageDetails = new[] + { + CatalogPackageDetails.Create(independentPackageDetails0), + CatalogPackageDetails.Create(independentPackageDetails1) + }; + + var independentPage = new CatalogIndependentPage( + catalogPageUri.AbsoluteUri, + CatalogConstants.CatalogPage, + commitId, + commitTimeStamp.ToString(CatalogConstants.CommitTimeStampFormat), + packageDetails.Length, + catalogIndexUri.AbsoluteUri, + packageDetails, + _contextKeyword); + + var index = CatalogIndex.Create(independentPage, _contextKeyword); + var catalogStorage = new MemoryStorage(_baseUri); + + catalogStorage.Content.TryAdd(catalogIndexUri, CreateStringStorageContent(index)); + catalogStorage.Content.TryAdd(catalogPageUri, CreateStringStorageContent(independentPage)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails0.IdKeyword), + CreateStringStorageContent(independentPackageDetails0)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails1.IdKeyword), + CreateStringStorageContent(independentPackageDetails1)); + + byte[] expectedNupkgBytes = ReadPackageBytes(package); + + await _mockServer.AddStorageAsync(catalogStorage); + + var packageId = package.Id.ToLowerInvariant(); + var packageVersion = package.Version.ToNormalizedString().ToLowerInvariant(); + var nupkgPathAndQuery = $"/packages/{packageId}.{packageVersion}.nupkg"; + + _mockServer.SetAction( + nupkgPathAndQuery, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(expectedNupkgBytes) + })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.Equal(3, _mockServer.Requests.Count); + + Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri); + Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri); + Assert.Contains(nupkgPathAndQuery, _mockServer.Requests[2].RequestUri.AbsoluteUri); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri($"{packageId}/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.nuspec"); + + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(_cursorJsonUri, out var cursorBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var actualNupkgBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspecBytes)); + + var actualCursorJson = Encoding.UTF8.GetString(cursorBytes); + var actualIndexJson = Encoding.UTF8.GetString(indexBytes); + var actualNuspec = Encoding.UTF8.GetString(nuspecBytes); + + Assert.Equal(GetExpectedCursorJsonContent(front.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")), actualCursorJson); + Assert.Equal(GetExpectedIndexJsonContent(packageVersion), actualIndexJson); + Assert.Equal(expectedNupkgBytes, actualNupkgBytes); + Assert.Equal(package.Nuspec, actualNuspec); + } + } - var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); - ReadCursor back = MemoryCursor.CreateMax(); + [Fact] + public async Task RunAsync_WhenMultipleCommitItemsWithSamePackageIdentityExistAcrossMultipleCommits_OnlyLastCommitIsProcessed() + { + using (var package = TestPackage.Create(_random)) + { + var catalogIndexUri = new Uri(_baseUri, "index.json"); + var catalogPageUri = new Uri(_baseUri, "page0.json"); + + var commitTimeStamp1 = DateTimeOffset.UtcNow; + var commitTimeStamp0 = commitTimeStamp1.AddMinutes(-1); + var independentPackageDetails0 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + Guid.NewGuid().ToString(), + commitTimeStamp0); + var independentPackageDetails1 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + Guid.NewGuid().ToString(), + commitTimeStamp1); + var packageDetails = new[] + { + CatalogPackageDetails.Create(independentPackageDetails0), + CatalogPackageDetails.Create(independentPackageDetails1) + }; + + var independentPage = new CatalogIndependentPage( + catalogPageUri.AbsoluteUri, + CatalogConstants.CatalogPage, + independentPackageDetails1.CommitId, + independentPackageDetails1.CommitTimeStamp, + packageDetails.Length, + catalogIndexUri.AbsoluteUri, + packageDetails, + _contextKeyword); + + var index = CatalogIndex.Create(independentPage, _contextKeyword); + var catalogStorage = new MemoryStorage(_baseUri); + + catalogStorage.Content.TryAdd(catalogIndexUri, CreateStringStorageContent(index)); + catalogStorage.Content.TryAdd(catalogPageUri, CreateStringStorageContent(independentPage)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails0.IdKeyword), + CreateStringStorageContent(independentPackageDetails0)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails1.IdKeyword), + CreateStringStorageContent(independentPackageDetails1)); + + byte[] expectedNupkgBytes = ReadPackageBytes(package); + + await _mockServer.AddStorageAsync(catalogStorage); + + var packageId = package.Id.ToLowerInvariant(); + var packageVersion = package.Version.ToNormalizedString().ToLowerInvariant(); + var nupkgPathAndQuery = $"/packages/{packageId}.{packageVersion}.nupkg"; + + _mockServer.SetAction( + nupkgPathAndQuery, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(expectedNupkgBytes) + })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.Equal(3, _mockServer.Requests.Count); + + Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri); + Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri); + Assert.Contains(nupkgPathAndQuery, _mockServer.Requests[2].RequestUri.AbsoluteUri); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri($"{packageId}/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.nuspec"); + + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(_cursorJsonUri, out var cursorBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var actualNupkgBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspecBytes)); + + var actualCursorJson = Encoding.UTF8.GetString(cursorBytes); + var actualIndexJson = Encoding.UTF8.GetString(indexBytes); + var actualNuspec = Encoding.UTF8.GetString(nuspecBytes); + + Assert.Equal(GetExpectedCursorJsonContent(front.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")), actualCursorJson); + Assert.Equal(GetExpectedIndexJsonContent(packageVersion), actualIndexJson); + Assert.Equal(expectedNupkgBytes, actualNupkgBytes); + Assert.Equal(package.Nuspec, actualNuspec); + } + } - var exception = await Assert.ThrowsAsync( - () => _target.RunAsync(front, back, CancellationToken.None)); + [Fact] + public async Task RunAsync_WhenMultipleCommitItemsHaveSameCommitTimeStampButDifferentCommitId_Throws() + { + using (var package0 = TestPackage.Create(_random)) + using (var package1 = TestPackage.Create(_random)) + { + var catalogIndexUri = new Uri(_baseUri, "index.json"); + var catalogPageUri = new Uri(_baseUri, "page0.json"); + + var commitId0 = Guid.NewGuid().ToString(); + var commitId1 = Guid.NewGuid().ToString(); + var commitTimeStamp = DateTime.UtcNow; + var independentPackageDetails0 = new CatalogIndependentPackageDetails( + package0.Id, + package0.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId0, + commitTimeStamp); + var independentPackageDetails1 = new CatalogIndependentPackageDetails( + package1.Id, + package1.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId1, + commitTimeStamp); + var packageDetails = new[] + { + CatalogPackageDetails.Create(independentPackageDetails0), + CatalogPackageDetails.Create(independentPackageDetails1) + }; + + var independentPage = new CatalogIndependentPage( + catalogPageUri.AbsoluteUri, + CatalogConstants.CatalogPage, + independentPackageDetails1.CommitId, + commitTimeStamp.ToString(CatalogConstants.CommitTimeStampFormat), + packageDetails.Length, + catalogIndexUri.AbsoluteUri, + packageDetails, + _contextKeyword); + + var index = CatalogIndex.Create(independentPage, _contextKeyword); + var catalogStorage = new MemoryStorage(_baseUri); + + catalogStorage.Content.TryAdd(catalogIndexUri, CreateStringStorageContent(index)); + catalogStorage.Content.TryAdd(catalogPageUri, CreateStringStorageContent(independentPage)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails0.IdKeyword), + CreateStringStorageContent(independentPackageDetails0)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails1.IdKeyword), + CreateStringStorageContent(independentPackageDetails1)); + + byte[] expectedNupkgBytes = ReadPackageBytes(package0); + + await _mockServer.AddStorageAsync(catalogStorage); + + var packageId = package0.Id.ToLowerInvariant(); + var packageVersion = package0.Version.ToNormalizedString().ToLowerInvariant(); + var nupkgPathAndQuery = $"/packages/{packageId}.{packageVersion}.nupkg"; + + _mockServer.SetAction( + nupkgPathAndQuery, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(expectedNupkgBytes) + })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + var exception = await Assert.ThrowsAsync( + () => _target.RunAsync(front, back, CancellationToken.None)); + + var expectedMessage = "Multiple commits exist with the same commit timestamp but different commit ID's: " + + $"{{ CommitId = {commitId0}, CommitTimeStamp = {commitTimeStamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")} }}, " + + $"{{ CommitId = {commitId1}, CommitTimeStamp = {commitTimeStamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")} }}"; + + Assert.StartsWith(expectedMessage, exception.Message); + } + } - Assert.Equal("The catalog batch 10/13/2015 6:40:07 AM contains multiple entries for the same package identity. Package(s): listedpackage 1.0.0", exception.Message); + private static string GetExpectedCursorJsonContent(string cursor) + { + return $"{{\r\n \"value\": \"{cursor}\"\r\n}}"; } private static string GetExpectedIndexJsonContent(string version) @@ -789,6 +1089,11 @@ private void FailFirstRequest(string relativeUri) _mockServer.SetAction(relativeUri, failFirst); } + private StringStorageContent CreateStringStorageContent(T value) + { + return new StringStorageContent(JsonConvert.SerializeObject(value, _jsonSettings)); + } + private static MemoryStream CreateZipStreamWithEntry(string name, string content) { var zipWithNoNuspec = new MemoryStream(); @@ -809,6 +1114,14 @@ private static MemoryStream CreateZipStreamWithEntry(string name, string content return zipWithNoNuspec; } + private static byte[] ReadPackageBytes(TestPackage package) + { + using (var reader = new BinaryReader(package.Stream)) + { + return reader.ReadBytes((int)package.Stream.Length); + } + } + private class SynchronizedMemoryStorage : MemoryStorage { protected HashSet SynchronizedUris { get; private set; } diff --git a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs index 55471bf0d..5c6711aac 100644 --- a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs +++ b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net; using System.Net.Http; diff --git a/tests/NgTests/Data/Catalogs.cs b/tests/NgTests/Data/Catalogs.cs index 56041e544..695b1d977 100644 --- a/tests/NgTests/Data/Catalogs.cs +++ b/tests/NgTests/Data/Catalogs.cs @@ -186,25 +186,6 @@ public static MemoryStorage CreateTestCatalogWithNonNormalizedDelete() return catalogStorage; } - public static MemoryStorage CreateTestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatch() - { - var catalogStorage = new MemoryStorage(new Uri("http://nuget.test")); - - catalogStorage.Content.TryAdd( - new Uri(catalogStorage.BaseAddress, "index.json"), - new StringStorageContent(TestCatalogEntries.TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchIndex)); - - catalogStorage.Content.TryAdd( - new Uri(catalogStorage.BaseAddress, "page0.json"), - new StringStorageContent(TestCatalogEntries.TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchPage)); - - catalogStorage.Content.TryAdd( - new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/listedpackage.1.0.0.json"), - new StringStorageContent(TestCatalogEntries.TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchListedPackage100)); - - return catalogStorage; - } - public static MemoryStorage CreateTestCatalogWithOnePackage() { var catalogStorage = new MemoryStorage(); diff --git a/tests/NgTests/Data/TestCatalogEntries.Designer.cs b/tests/NgTests/Data/TestCatalogEntries.Designer.cs index 2971298ba..55ffa4abc 100644 --- a/tests/NgTests/Data/TestCatalogEntries.Designer.cs +++ b/tests/NgTests/Data/TestCatalogEntries.Designer.cs @@ -657,80 +657,6 @@ public static string TestCatalogWithCommitThenTwoPackageCommitPage { } } - /// - /// Looks up a localized string similar to { - /// "@id": "http://nuget.test/index.json", - /// "@type": [ - /// "CatalogRoot", - /// "AppendOnlyCatalog", - /// "Permalink" - /// ], - /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - /// "count": 1, - /// "items": [ - /// { - /// "@id": "http://nuget.test/page0.json", - /// "@type": "CatalogPage", - /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - /// "count": 2 - /// } - /// ], - /// "nuget:lastC [rest of string was truncated]";. - /// - public static string TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchIndex { - get { - return ResourceManager.GetString("TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchIndex", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to { - /// "@id": "http://nuget.test/data/2015.10.13.06.40.07/listedpackage.1.0.0.json", - /// "@type": [ - /// "PackageDetails", - /// "catalog:Permalink" - /// ], - /// "authors": "NuGet", - /// "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - /// "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - /// "created": "2015-01-01T00:00:00Z", - /// "description": "Package description.", - /// "id": "ListedPackage", - /// "isPrerelease": false, - /// "lastEdited": "2015-01-01T00:00:00Z", - /// "licenseNames": "", - /// "licenseReport [rest of string was truncated]";. - /// - public static string TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchListedPackage100 { - get { - return ResourceManager.GetString("TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchListedPackage100", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to { - /// "@id": "http://nuget.test/page0.json", - /// "@type": "CatalogPage", - /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - /// "count": 2, - /// "items": [ - /// { - /// "@id": "http://nuget.test/data/2015.10.13.06.40.07/listedpackage.1.0.0.json", - /// "@type": "nuget:PackageDetails", - /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - /// "nuget:id": "ListedPackage", - /// "nuget:vers [rest of string was truncated]";. - /// - public static string TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchPage { - get { - return ResourceManager.GetString("TestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatchPage", resourceCulture); - } - } - /// /// Looks up a localized string similar to { /// "@id": "http://tempuri.org/index.json", diff --git a/tests/NgTests/Data/TestCatalogEntries.resx b/tests/NgTests/Data/TestCatalogEntries.resx index 9351db962..b3935355d 100644 --- a/tests/NgTests/Data/TestCatalogEntries.resx +++ b/tests/NgTests/Data/TestCatalogEntries.resx @@ -1541,169 +1541,6 @@ "@type": "http://www.w3.org/2001/XMLSchema#dateTime" } } -} - - - { - "@id": "http://nuget.test/index.json", - "@type": [ - "CatalogRoot", - "AppendOnlyCatalog", - "Permalink" - ], - "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - "count": 1, - "items": [ - { - "@id": "http://nuget.test/page0.json", - "@type": "CatalogPage", - "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - "count": 2 - } - ], - "nuget:lastCreated": "2015-01-01T00:00:00Z", - "nuget:lastDeleted": "2015-01-01T01:01:01.0748028Z", - "nuget:lastEdited": "2015-01-01T01:01:01.0748028Z", - "@context": { - "@vocab": "http://schema.nuget.org/catalog#", - "nuget": "http://schema.nuget.org/schema#", - "items": { - "@id": "item", - "@container": "@set" - }, - "parent": { - "@type": "@id" - }, - "commitTimeStamp": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - }, - "nuget:lastCreated": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - }, - "nuget:lastEdited": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - }, - "nuget:lastDeleted": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - } - } -} - - - { - "@id": "http://nuget.test/data/2015.10.13.06.40.07/listedpackage.1.0.0.json", - "@type": [ - "PackageDetails", - "catalog:Permalink" - ], - "authors": "NuGet", - "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - "created": "2015-01-01T00:00:00Z", - "description": "Package description.", - "id": "ListedPackage", - "isPrerelease": false, - "lastEdited": "2015-01-01T00:00:00Z", - "licenseNames": "", - "licenseReportUrl": "", - "listed": true, - "packageHash": "EpTkeONwnhX59JBzl5QfuFNNgZADaAbwYxNGn0KEkJ4ylukUQFcS15vISqFUnrWy/+yylox6L6QT3MD/+Us+vg==", - "packageHashAlgorithm": "SHA512", - "packageSize": 2529, - "published": "2015-01-01T00:00:00Z", - "requireLicenseAcceptance": false, - "verbatimVersion": "1.0.0", - "version": "1.0.0", - "@context": { - "@vocab": "http://schema.nuget.org/schema#", - "catalog": "http://schema.nuget.org/catalog#", - "xsd": "http://www.w3.org/2001/XMLSchema#", - "dependencies": { - "@id": "dependency", - "@container": "@set" - }, - "dependencyGroups": { - "@id": "dependencyGroup", - "@container": "@set" - }, - "packageEntries": { - "@id": "packageEntry", - "@container": "@set" - }, - "supportedFrameworks": { - "@id": "supportedFramework", - "@container": "@set" - }, - "tags": { - "@id": "tag", - "@container": "@set" - }, - "published": { - "@type": "xsd:dateTime" - }, - "created": { - "@type": "xsd:dateTime" - }, - "lastEdited": { - "@type": "xsd:dateTime" - }, - "catalog:commitTimeStamp": { - "@type": "xsd:dateTime" - } - } -} - - - { - "@id": "http://nuget.test/page0.json", - "@type": "CatalogPage", - "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - "count": 2, - "items": [ - { - "@id": "http://nuget.test/data/2015.10.13.06.40.07/listedpackage.1.0.0.json", - "@type": "nuget:PackageDetails", - "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - "nuget:id": "ListedPackage", - "nuget:version": "1.0.0" - }, - { - "@id": "http://nuget.test/data/2015.10.13.06.40.07/listedpackage.1.0.0.json", - "@type": "nuget:PackageDetails", - "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", - "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", - "nuget:id": "ListedPackage", - "nuget:version": "1.0.0" - } - ], - "parent": "http://nuget.test/index.json", - "@context": { - "@vocab": "http://schema.nuget.org/catalog#", - "nuget": "http://schema.nuget.org/schema#", - "items": { - "@id": "item", - "@container": "@set" - }, - "parent": { - "@type": "@id" - }, - "commitTimeStamp": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - }, - "nuget:lastCreated": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - }, - "nuget:lastEdited": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - }, - "nuget:lastDeleted": { - "@type": "http://www.w3.org/2001/XMLSchema#dateTime" - } - } } diff --git a/tests/NgTests/Feed2CatalogTests.cs b/tests/NgTests/Feed2CatalogTests.cs index d1faffeee..55d60cb0c 100644 --- a/tests/NgTests/Feed2CatalogTests.cs +++ b/tests/NgTests/Feed2CatalogTests.cs @@ -1552,7 +1552,7 @@ private TestPackage AddPackageEntry(TestPackage package) } } - return new TestPackage(package.Id, package.Version, package.Author, package.Description, stream); + return new TestPackage(package.Id, package.Version, package.Author, package.Description, package.Nuspec, stream); } private static JObject ReadJsonWithoutDateTimeHandling(JObject jObject) diff --git a/tests/NgTests/Infrastructure/MockTelemetryService.cs b/tests/NgTests/Infrastructure/MockTelemetryService.cs index f01ca6093..982b2b13d 100644 --- a/tests/NgTests/Infrastructure/MockTelemetryService.cs +++ b/tests/NgTests/Infrastructure/MockTelemetryService.cs @@ -12,16 +12,15 @@ namespace NgTests.Infrastructure { public sealed class MockTelemetryService : ITelemetryService { - public IDictionary GlobalDimensions => throw new NotImplementedException(); + private readonly List _trackDurationCalls = new List(); + private readonly List _trackMetricCalls = new List(); + private readonly object _durationCallsSyncObject = new object(); + private readonly object _metricCallsSyncObject = new object(); - public List TrackDurationCalls { get; } - public List TrackMetricCalls { get; } + public IDictionary GlobalDimensions => throw new NotImplementedException(); - public MockTelemetryService() - { - TrackDurationCalls = new List(); - TrackMetricCalls = new List(); - } + public IReadOnlyList TrackDurationCalls => _trackDurationCalls; + public IReadOnlyList TrackMetricCalls => _trackMetricCalls; public void TrackCatalogIndexReadDuration(TimeSpan duration, Uri uri) { @@ -62,14 +61,20 @@ public void TrackPackageHashFixed(string packageId, NuGetVersion packageVersion) public DurationMetric TrackDuration(string name, IDictionary properties = null) { - TrackDurationCalls.Add(new TelemetryCall(name, properties)); + lock (_durationCallsSyncObject) + { + _trackDurationCalls.Add(new TelemetryCall(name, properties)); + } return new DurationMetric(Mock.Of(), name, properties); } public void TrackMetric(string name, ulong metric, IDictionary properties = null) { - TrackMetricCalls.Add(new TrackMetricCall(name, metric, properties)); + lock (_metricCallsSyncObject) + { + _trackMetricCalls.Add(new TrackMetricCall(name, metric, properties)); + } } } } \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestPackage.cs b/tests/NgTests/Infrastructure/TestPackage.cs index e4f647761..70e8270ab 100644 --- a/tests/NgTests/Infrastructure/TestPackage.cs +++ b/tests/NgTests/Infrastructure/TestPackage.cs @@ -9,22 +9,24 @@ namespace NgTests.Infrastructure { - internal sealed class TestPackage : IDisposable + public sealed class TestPackage : IDisposable { private bool _isDisposed; - internal string Id { get; } - internal NuGetVersion Version { get; } - internal string Author { get; } - internal string Description { get; } - internal Stream Stream { get; private set; } + public string Id { get; } + public NuGetVersion Version { get; } + public string Author { get; } + public string Description { get; } + public string Nuspec { get; } + public Stream Stream { get; } - internal TestPackage(string id, NuGetVersion version, string author, string description, Stream stream) + public TestPackage(string id, NuGetVersion version, string author, string description, string nuspec, Stream stream) { Id = id; Version = version; Author = author; Description = description; + Nuspec = nuspec; Stream = stream; } @@ -40,7 +42,7 @@ public void Dispose() } } - internal static TestPackage Create(Random random) + public static TestPackage Create(Random random) { var id = TestUtility.CreateRandomAlphanumericString(random); var version = CreateRandomVersion(random); @@ -52,7 +54,7 @@ internal static TestPackage Create(Random random) { var stream = CreatePackageStream(id, nuspec, rng, random); - return new TestPackage(id, version, author, description, stream); + return new TestPackage(id, version, author, description, nuspec, stream); } } @@ -103,6 +105,8 @@ private static MemoryStream CreatePackageStream(string id, string nuspec, Random } } + stream.Position = 0; + return stream; }