diff --git a/build.ps1 b/build.ps1 index 356eceded..194246715 100644 --- a/build.ps1 +++ b/build.ps1 @@ -118,6 +118,7 @@ Invoke-BuildStep 'Creating artifacts' { "src\Ng\Catalog2Monitoring.nuspec", ` "src\Ng\Catalog2Registration.nuspec", ` "src\Ng\Db2Catalog.nuspec", ` + "src\Ng\Db2Monitoring.nuspec", ` "src\Ng\Monitoring2Monitoring.nuspec", ` "src\Ng\MonitoringProcessor.nuspec", ` "src\Ng\Ng.Operations.nuspec", ` diff --git a/src/Catalog/Helpers/Db2CatalogProjection.cs b/src/Catalog/Helpers/Db2CatalogProjection.cs index b678dbf40..4ff78ce77 100644 --- a/src/Catalog/Helpers/Db2CatalogProjection.cs +++ b/src/Catalog/Helpers/Db2CatalogProjection.cs @@ -34,6 +34,7 @@ public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader data var packageId = dataReader[Db2CatalogProjectionColumnNames.PackageId].ToString(); var normalizedPackageVersion = dataReader[Db2CatalogProjectionColumnNames.NormalizedVersion].ToString(); + var fullPackageVersion = dataReader[Db2CatalogProjectionColumnNames.FullVersion].ToString(); var listed = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.Listed)); var hideLicenseReport = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.HideLicenseReport)); @@ -47,6 +48,7 @@ public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader data listed ? dataReader.ReadDateTime(Db2CatalogProjectionColumnNames.Published).ForceUtc() : Constants.UnpublishedDate, packageId, normalizedPackageVersion, + fullPackageVersion, hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseNames]?.ToString(), hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseReportUrl]?.ToString(), deprecationInfo, diff --git a/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs b/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs index 272b7f4cf..309c8e627 100644 --- a/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs +++ b/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs @@ -10,6 +10,7 @@ public static class Db2CatalogProjectionColumnNames { public const string PackageId = "Id"; public const string NormalizedVersion = "NormalizedVersion"; + public const string FullVersion = "Version"; public const string Listed = "Listed"; public const string HideLicenseReport = "HideLicenseReport"; public const string Created = "Created"; diff --git a/src/Catalog/Helpers/FeedPackageDetails.cs b/src/Catalog/Helpers/FeedPackageDetails.cs index 0d2d94d5f..8c26862ee 100644 --- a/src/Catalog/Helpers/FeedPackageDetails.cs +++ b/src/Catalog/Helpers/FeedPackageDetails.cs @@ -12,7 +12,8 @@ public sealed class FeedPackageDetails public DateTime LastEditedDate { get; } public DateTime PublishedDate { get; } public string PackageId { get; } - public string PackageVersion { get; } + public string PackageNormalizedVersion { get; } + public string PackageFullVersion { get; } public string LicenseNames { get; } public string LicenseReportUrl { get; } public bool RequiresLicenseAcceptance { get; } @@ -26,14 +27,16 @@ public FeedPackageDetails( DateTime lastEditedDate, DateTime publishedDate, string packageId, - string packageVersion) + string packageNormalizedVersion, + string packageFullVersion) : this( contentUri, createdDate, lastEditedDate, publishedDate, packageId, - packageVersion, + packageNormalizedVersion, + packageFullVersion, licenseNames: null, licenseReportUrl: null, deprecationInfo: null, @@ -47,7 +50,8 @@ public FeedPackageDetails( DateTime lastEditedDate, DateTime publishedDate, string packageId, - string packageVersion, + string packageNormalizedVersion, + string packageFullVersion, string licenseNames, string licenseReportUrl, PackageDeprecationItem deprecationInfo, @@ -58,7 +62,8 @@ public FeedPackageDetails( LastEditedDate = lastEditedDate; PublishedDate = publishedDate; PackageId = packageId; - PackageVersion = packageVersion; + PackageNormalizedVersion = packageNormalizedVersion; + PackageFullVersion = packageFullVersion; LicenseNames = licenseNames; LicenseReportUrl = licenseReportUrl; DeprecationInfo = deprecationInfo; diff --git a/src/Catalog/Helpers/GalleryDatabaseQueryService.cs b/src/Catalog/Helpers/GalleryDatabaseQueryService.cs index 3518def5c..d9ce2eb0b 100644 --- a/src/Catalog/Helpers/GalleryDatabaseQueryService.cs +++ b/src/Catalog/Helpers/GalleryDatabaseQueryService.cs @@ -22,6 +22,7 @@ public class GalleryDatabaseQueryService : IGalleryDatabaseQueryService private static readonly string Db2CatalogSqlSubQuery = $@" PR.[Id], P.[NormalizedVersion], + P.[Version], P.[Created], P.[LastEdited], P.[Published], diff --git a/src/Catalog/PackageCatalogItemCreator.cs b/src/Catalog/PackageCatalogItemCreator.cs index b697080da..d3228dda2 100644 --- a/src/Catalog/PackageCatalogItemCreator.cs +++ b/src/Catalog/PackageCatalogItemCreator.cs @@ -72,7 +72,7 @@ public async Task CreateAsync( _logger.LogInformation( "Creating package catalog item for {Id} {Version}", packageItem.PackageId, - packageItem.PackageVersion); + packageItem.PackageNormalizedVersion); if (_storage != null) { @@ -87,7 +87,7 @@ public async Task CreateAsync( _logger.LogInformation( "Finished creating package catalog item for {Id} {Version}", packageItem.PackageId, - packageItem.PackageVersion); + packageItem.PackageNormalizedVersion); return item; } @@ -98,8 +98,8 @@ private async Task GetPackageViaStorageAsync( { PackageCatalogItem item = null; var packageId = packageItem.PackageId.ToLowerInvariant(); - var packageVersion = packageItem.PackageVersion.ToLowerInvariant(); - var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion); + var packageNormalizedVersion = packageItem.PackageNormalizedVersion.ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageNormalizedVersion); var blobUri = _storage.ResolveUri(packageFileName); var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri); @@ -108,14 +108,14 @@ private async Task GetPackageViaStorageAsync( _telemetryService.TrackMetric( TelemetryConstants.NonExistentBlob, metric: 1, - properties: GetProperties(packageId, packageVersion, blob)); + properties: GetProperties(packageId, packageNormalizedVersion, blob)); return item; } using (_telemetryService.TrackDuration( TelemetryConstants.PackageBlobReadSeconds, - GetProperties(packageId, packageVersion, blob: null))) + GetProperties(packageId, packageNormalizedVersion, blob: null))) { await blob.FetchAttributesAsync(cancellationToken); @@ -162,7 +162,7 @@ private async Task GetPackageViaStorageAsync( _telemetryService.TrackMetric( TelemetryConstants.BlobModified, metric: 1, - properties: GetProperties(packageId, packageVersion, blob)); + properties: GetProperties(packageId, packageNormalizedVersion, blob)); } } } @@ -171,7 +171,7 @@ private async Task GetPackageViaStorageAsync( _telemetryService.TrackMetric( TelemetryConstants.NonExistentPackageHash, metric: 1, - properties: GetProperties(packageId, packageVersion, blob)); + properties: GetProperties(packageId, packageNormalizedVersion, blob)); } } @@ -243,16 +243,16 @@ private static Dictionary GetProperties(FeedPackageDetails packa { return GetProperties( packageItem.PackageId.ToLowerInvariant(), - packageItem.PackageVersion.ToLowerInvariant(), + packageItem.PackageNormalizedVersion.ToLowerInvariant(), blob); } - private static Dictionary GetProperties(string packageId, string packageVersion, ICloudBlockBlob blob) + private static Dictionary GetProperties(string packageId, string packageNormalizedVersion, ICloudBlockBlob blob) { var properties = new Dictionary() { { TelemetryConstants.Id, packageId }, - { TelemetryConstants.Version, packageVersion } + { TelemetryConstants.Version, packageNormalizedVersion } }; if (blob != null) diff --git a/src/Ng/Db2Monitoring.nuspec b/src/Ng/Db2Monitoring.nuspec new file mode 100644 index 000000000..e7dbabb60 --- /dev/null +++ b/src/Ng/Db2Monitoring.nuspec @@ -0,0 +1,16 @@ + + + + Db2Monitoring + $version$ + .NET Foundation + .NET Foundation + The Db2Monitoring job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2MonitoringJob.cs b/src/Ng/Jobs/Catalog2MonitoringJob.cs index 5f98ba80e..2d079441f 100644 --- a/src/Ng/Jobs/Catalog2MonitoringJob.cs +++ b/src/Ng/Jobs/Catalog2MonitoringJob.cs @@ -26,25 +26,20 @@ public Catalog2MonitoringJob(ITelemetryService telemetryService, ILoggerFactory protected override void Init(IDictionary arguments, CancellationToken cancellationToken) { - var index = arguments.GetOrThrow(Arguments.Index); var source = arguments.GetOrThrow(Arguments.Source); var verbose = arguments.GetOrDefault(Arguments.Verbose, false); CommandHelpers.AssertAzureStorage(arguments); var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); - var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments); - var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); - var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); - var queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); Logger.LogInformation( - "CONFIG index: {Index} storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}", - index, monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri); + "CONFIG storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}", + monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri); _enqueuer = ValidationFactory.CreatePackageValidatorContextEnqueuer( queue, diff --git a/src/Ng/Jobs/Db2MonitoringJob.cs b/src/Ng/Jobs/Db2MonitoringJob.cs new file mode 100644 index 000000000..38b4a1164 --- /dev/null +++ b/src/Ng/Jobs/Db2MonitoringJob.cs @@ -0,0 +1,211 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Ng.Helpers; +using NuGet.Protocol; +using NuGet.Services.Configuration; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Monitoring.Monitoring; +using NuGet.Services.Sql; +using NuGet.Services.Storage; + +using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; +using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; +using Constants = NuGet.Services.Metadata.Catalog.Constants; + +namespace Ng.Jobs +{ + public class Db2MonitoringJob : LoopingNgJob + { + /// + /// This job will continuously check packages within a range of minus 2 * and minus 1 * . + /// + private readonly static TimeSpan ReprocessRange = TimeSpan.FromHours(1); + + /// + /// Any values greater than will be ignored by . + /// + private const int Top = Constants.MaxPageSize; + private const string GalleryCursorFileName = "gallerycursor.json"; + private const string DeletedCursorFileName = "deletedcursor.json"; + + private const int DefaultMaxQueueSize = 100; + private int _maxRequeueQueueSize; + + private IGalleryDatabaseQueryService _galleryDatabaseQueryService; + private IStorageQueue _packageValidatorContextQueue; + private IPackageMonitoringStatusService _statusService; + private ReadCursor _monitoringCursor; + private ReadWriteCursor _galleryCursor; + + private CatalogStorage _auditingStorage; + private ReadWriteCursor _deletedCursor; + + private CollectorHttpClient _client; + + public Db2MonitoringJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) + : base(telemetryService, loggerFactory) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + _maxRequeueQueueSize = arguments.GetOrDefault(Arguments.MaxRequeueQueueSize, DefaultMaxQueueSize); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + _packageValidatorContextQueue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + + Logger.LogInformation( + "CONFIG storage: {Storage}", + monitoringStorageFactory); + + _monitoringCursor = ValidationFactory.GetFront(monitoringStorageFactory); + _galleryCursor = CreateCursor(monitoringStorageFactory, GalleryCursorFileName); + _deletedCursor = CreateCursor(monitoringStorageFactory, DeletedCursorFileName); + + var connectionString = arguments.GetOrThrow(Arguments.ConnectionString); + var galleryDbConnection = new AzureSqlConnectionFactory( + connectionString, + new EmptySecretInjector(), + LoggerFactory.CreateLogger()); + + var packageContentUriBuilder = new PackageContentUriBuilder( + arguments.GetOrThrow(Arguments.PackageContentUrlFormat)); + + var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); + _galleryDatabaseQueryService = new GalleryDatabaseQueryService( + galleryDbConnection, + packageContentUriBuilder, + TelemetryService, + timeoutInSeconds); + + var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( + "Auditing", + arguments, + verbose, + new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); + + _auditingStorage = auditingStorageFactory.Create(); + + var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); + _client = new CollectorHttpClient(messageHandlerFactory()); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + var databaseSource = new DatabasePackageStatusOutdatedCheckSource( + _galleryCursor, _galleryDatabaseQueryService); + + var auditingSource = new AuditingStoragePackageStatusOutdatedCheckSource( + _deletedCursor, _auditingStorage, LoggerFactory.CreateLogger()); + + var sources = new IPackageStatusOutdatedCheckSource[] { databaseSource, auditingSource }; + + var hasPackagesToProcess = true; + while (hasPackagesToProcess) + { + var currentMessageCount = await _packageValidatorContextQueue.GetMessageCount(cancellationToken); + if (currentMessageCount > _maxRequeueQueueSize) + { + Logger.LogInformation( + "Can't continue processing packages because the queue has too many messages ({CurrentMessageCount} > {MaxRequeueQueueSize})!", + currentMessageCount, _maxRequeueQueueSize); + return; + } + + hasPackagesToProcess = await CheckPackages( + sources, + cancellationToken); + } + + Logger.LogInformation("All packages have had their status checked."); + await _monitoringCursor.LoadAsync(cancellationToken); + var newCursorValue = _monitoringCursor.Value - ReprocessRange - ReprocessRange; + Logger.LogInformation("Restarting source cursors to {NewCursorValue}.", newCursorValue); + foreach (var source in sources) + { + await source.MoveBackAsync(newCursorValue, cancellationToken); + } + } + + private async Task CheckPackages( + IReadOnlyCollection sources, + CancellationToken cancellationToken) + { + Logger.LogInformation("Fetching packages to check status of."); + var packagesToCheck = new List(); + await _monitoringCursor.LoadAsync(cancellationToken); + foreach (var source in sources) + { + packagesToCheck.AddRange(await source.GetPackagesToCheckAsync( + _monitoringCursor.Value - ReprocessRange, Top, cancellationToken)); + } + + var packagesToCheckBag = new ConcurrentBag(packagesToCheck); + + Logger.LogInformation("Found {PackagesToCheckCount} packages to check status of.", packagesToCheck.Count()); + await ParallelAsync.Repeat(() => ProcessPackagesAsync(packagesToCheckBag, cancellationToken)); + Logger.LogInformation("Finished checking status of packages."); + + foreach (var source in sources) + { + await source.MarkPackagesCheckedAsync(cancellationToken); + } + + return packagesToCheck.Any(); + } + + private async Task ProcessPackagesAsync( + ConcurrentBag checkBag, CancellationToken cancellationToken) + { + while (checkBag.TryTake(out var check)) + { + if (await IsStatusOutdatedAsync(check, cancellationToken)) + { + Logger.LogWarning("Status for {Id} {Version} is outdated!", check.Identity.Id, check.Identity.Version); + var context = new PackageValidatorContext(check.Identity, null); + await _packageValidatorContextQueue.AddAsync(context, cancellationToken); + } + else + { + Logger.LogInformation("Status for {Id} {Version} is up to date.", check.Identity.Id, check.Identity.Version); + } + } + } + + private async Task IsStatusOutdatedAsync( + PackageStatusOutdatedCheck check, CancellationToken cancellationToken) + { + var status = await _statusService.GetAsync(check.Identity, cancellationToken); + + var catalogEntries = status?.ValidationResult?.CatalogEntries; + if (catalogEntries == null || !catalogEntries.Any()) + { + return true; + } + + var latestCatalogEntryTimestampMetadata = await PackageTimestampMetadata.FromCatalogEntries(_client, catalogEntries); + return check.Timestamp > latestCatalogEntryTimestampMetadata.Last; + } + + private DurableCursor CreateCursor(CatalogStorageFactory storageFactory, string filename) + { + var storage = storageFactory.Create(); + return new DurableCursor(storage.ResolveUri(filename), storage, MemoryCursor.MinValue); + } + } +} diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs index 2069d354e..3c62c4d41 100644 --- a/src/Ng/Jobs/MonitoringProcessorJob.cs +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -10,7 +10,6 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Queue.Protocol; using Ng.Helpers; -using NuGet.Packaging.Core; using NuGet.Protocol; using NuGet.Protocol.Core.Types; using NuGet.Services.Configuration; @@ -32,7 +31,6 @@ public class MonitoringProcessorJob : LoopingNgJob private IStorageQueue _queue; private IPackageMonitoringStatusService _statusService; private IMonitoringNotificationService _notificationService; - private RegistrationResourceV3 _regResource; private CollectorHttpClient _client; public MonitoringProcessorJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) @@ -97,8 +95,6 @@ protected override void Init(IDictionary arguments, Cancellation _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger()); - _regResource = Repository.Factory.GetCoreV3(index).GetResource(cancellationToken); - _client = new CollectorHttpClient(messageHandlerFactory()); SetUserAgentString(); @@ -249,7 +245,11 @@ private async Task SaveFailedPackageMonitoringStatusAsync( Exception exception, CancellationToken token) { - var feedPackage = new FeedPackageIdentity(queuedContext.Package.Id, queuedContext.Package.Version); + var queuedVersion = queuedContext.Package.Version; + var version = NuGetVersion.TryParse(queuedVersion, out var parsedVersion) + ? parsedVersion.ToFullString() : queuedVersion; + + var feedPackage = new FeedPackageIdentity(queuedContext.Package.Id, version); await _notificationService.OnPackageValidationFailedAsync(feedPackage.Id, feedPackage.Version, exception, token); diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index 8ae94cb67..c783fbb5b 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -63,6 +63,7 @@ + diff --git a/src/Ng/NgJobFactory.cs b/src/Ng/NgJobFactory.cs index 85e868054..40b841a1d 100644 --- a/src/Ng/NgJobFactory.cs +++ b/src/Ng/NgJobFactory.cs @@ -14,6 +14,7 @@ public static class NgJobFactory public static IDictionary JobMap = new Dictionary() { { "db2catalog", typeof(Db2CatalogJob) }, + { "db2monitoring", typeof(Db2MonitoringJob) }, { "catalog2registration", typeof(Catalog2RegistrationJob) }, { "catalog2lucene", typeof(Catalog2LuceneJob) }, { "catalog2dnx", typeof(Catalog2DnxJob) }, diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs index 295e417af..8707cd36f 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs @@ -40,7 +40,7 @@ public PackageRegistrationIndexMetadata(FeedPackageDetails package) : base(package) { Id = package.PackageId; - Version = NuGetVersion.Parse(package.PackageVersion); + Version = NuGetVersion.Parse(package.PackageNormalizedVersion); RequireLicenseAcceptance = package.RequiresLicenseAcceptance; if (package.HasDeprecationInfo) { diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs index 549772917..5ef73fae0 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs @@ -53,11 +53,33 @@ public static PackageTimestampMetadata CreateForMissingPackage(DateTime? deleted }; } + /// + /// In the past, the catalog entries referred to the CDN's host instead our DNS. + /// If we encounter one of these outdated hosts, we should hit the proper host instead. + /// + private static readonly IDictionary _catalogEntryUriHostMap = + new Dictionary + { + { "az635243.vo.msecnd.net", "apidev.nugettest.org" }, + { "az636225.vo.msecnd.net", "apiint.nugettest.org" } + }; + public static async Task FromCatalogEntry( CollectorHttpClient client, CatalogIndexEntry catalogEntry) { - var catalogLeaf = await client.GetJObjectAsync(catalogEntry.Uri); + var uri = catalogEntry.Uri; + if (_catalogEntryUriHostMap.TryGetValue(catalogEntry.Uri.Host, out var replacementHost)) + { + var builder = new UriBuilder(uri) + { + Host = replacementHost + }; + + uri = builder.Uri; + } + + var catalogLeaf = await client.GetJObjectAsync(uri); try { @@ -88,8 +110,11 @@ public static async Task FromCatalogEntries( { var packageTimestampMetadatas = await Task.WhenAll(catalogEntries.Select(entry => FromCatalogEntry(client, entry))); - var maxTimestamp = packageTimestampMetadatas.Where(p => p != null).Max(p => p.Last); - return packageTimestampMetadatas.FirstOrDefault(p => p.Last == maxTimestamp); + + return packageTimestampMetadatas + .Where(p => p != null) + .OrderByDescending(p => p.Last) + .First(); } } } \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/AuditingStoragePackageStatusOutdatedCheckSource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/AuditingStoragePackageStatusOutdatedCheckSource.cs new file mode 100644 index 000000000..a72c72d9b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/AuditingStoragePackageStatusOutdatedCheckSource.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; + +using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + /// + /// Fetches s from . + /// The s fetched represent packages that were deleted. + /// + /// + /// Some packages that were deleted may have been reuploaded. + /// This source does not prevent those packages from being returned. + /// + public class AuditingStoragePackageStatusOutdatedCheckSource : PackageStatusOutdatedCheckSource + { + private readonly CatalogStorage _auditingStorage; + private readonly ILogger _logger; + + private IReadOnlyCollection _cachedAuditEntries; + + public AuditingStoragePackageStatusOutdatedCheckSource( + ReadWriteCursor cursor, + CatalogStorage auditingStorage, + ILogger logger) + : base(cursor) + { + _auditingStorage = auditingStorage ?? throw new ArgumentNullException(nameof(auditingStorage)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override DateTime GetCursorValue(DeletionAuditEntry package) + { + return package.TimestampUtc.Value; + } + + protected override PackageStatusOutdatedCheck GetPackageStatusOutdatedCheck(DeletionAuditEntry package) + { + return new PackageStatusOutdatedCheck(package); + } + + protected override async Task> GetPackagesToCheckAsync(DateTime since, DateTime max, int top, CancellationToken cancellationToken) + { + // Fetching audit entries is expensive, so cache them. + // When we run out of cached entries, fetch more. + if (_cachedAuditEntries == null || !_cachedAuditEntries.Any()) + { + _logger.LogInformation("Fetching audit entries from storage."); + var auditEntries = await DeletionAuditEntry + .GetAsync(_auditingStorage, CancellationToken.None, minTime: since, maxTime: max, logger: _logger); + + // A package may have multiple deleted audit entries. + // Choose only the latest for each package. + _cachedAuditEntries = auditEntries + .GroupBy(e => new FeedPackageIdentity(e.PackageId, e.PackageVersion)) + .Select(g => g.OrderByDescending(e => e.TimestampUtc).First()) + .OrderBy(e => e.TimestampUtc) + .ToList(); + + _logger.LogInformation("Cached {AuditEntryCount} audit entries.", _cachedAuditEntries.Count()); + } + else + { + _logger.LogInformation("Using cached audit entries."); + } + + var nextAuditEntries = _cachedAuditEntries.Take(top).ToList(); + _cachedAuditEntries = _cachedAuditEntries + .Skip(top) + .ToList(); + + return nextAuditEntries; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/DatabasePackageStatusOutdatedCheckSource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/DatabasePackageStatusOutdatedCheckSource.cs new file mode 100644 index 000000000..5750e67c8 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/DatabasePackageStatusOutdatedCheckSource.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + /// + /// Fetches from . + /// The s fetched represent the latest state for existing packages. + /// + public class DatabasePackageStatusOutdatedCheckSource : PackageStatusOutdatedCheckSource + { + private readonly IGalleryDatabaseQueryService _galleryDatabaseQueryService; + + public DatabasePackageStatusOutdatedCheckSource( + ReadWriteCursor cursor, + IGalleryDatabaseQueryService galleryDatabase) + : base(cursor) + { + _galleryDatabaseQueryService = galleryDatabase ?? throw new ArgumentNullException(nameof(galleryDatabase)); + } + + protected override DateTime GetCursorValue(FeedPackageDetails package) + { + return package.LastEditedDate; + } + + protected override PackageStatusOutdatedCheck GetPackageStatusOutdatedCheck(FeedPackageDetails package) + { + return new PackageStatusOutdatedCheck(package); + } + + /// + /// Any values greater than will be ignored by . + /// + protected override async Task> GetPackagesToCheckAsync(DateTime since, DateTime max, int top, CancellationToken cancellationToken) + { + return (await _galleryDatabaseQueryService.GetPackagesEditedSince(since, top)) + .SelectMany(p => p.Value) + .Where(p => p.LastEditedDate < max) + .ToList(); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheck.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheck.cs new file mode 100644 index 000000000..b08e8e4ae --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheck.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + public class PackageStatusOutdatedCheck + { + public PackageStatusOutdatedCheck( + FeedPackageIdentity identity, + DateTime commitTimestamp) + { + Identity = identity; + Timestamp = commitTimestamp; + } + + public PackageStatusOutdatedCheck( + FeedPackageDetails package) + : this( + new FeedPackageIdentity( + package.PackageId, + ParseVersionString(package.PackageFullVersion)), + package.LastEditedDate) + { + } + + public PackageStatusOutdatedCheck( + DeletionAuditEntry auditEntry) + : this( + new FeedPackageIdentity(auditEntry.PackageId, ParseVersionString(auditEntry.PackageVersion)), + auditEntry.TimestampUtc.Value) + { + } + + public FeedPackageIdentity Identity { get; } + public DateTime Timestamp { get; } + + private static string ParseVersionString(string version) + { + return NuGetVersion.TryParse(version, out var parsedVersion) + ? parsedVersion.ToFullString() : version; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheckSource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheckSource.cs new file mode 100644 index 000000000..7fac82f55 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheckSource.cs @@ -0,0 +1,90 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + /// + /// Fetches s from a source and maintains a on it. + /// + public interface IPackageStatusOutdatedCheckSource + { + /// + /// Fetches the next batch of s. + /// + /// Up to this many s are returned. + Task> GetPackagesToCheckAsync(DateTime max, int top, CancellationToken cancellationToken); + + /// + /// Updates the based on the last batch of packages returned by . + /// If was never called, or the last batch was already processed, this method does nothing. + /// + Task MarkPackagesCheckedAsync(CancellationToken cancellationToken); + + /// + /// Moves back the to a specific time. + /// The next call to will return the batches of packages after that time. + /// If the cursor would be moving ahead, the change is not done. + /// + Task MoveBackAsync(DateTime value, CancellationToken cancellationToken); + } + + public abstract class PackageStatusOutdatedCheckSource : IPackageStatusOutdatedCheckSource + { + private readonly ReadWriteCursor _cursor; + private DateTime? _pendingCursorValue; + + public PackageStatusOutdatedCheckSource(ReadWriteCursor cursor) + { + _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); + } + + public async Task> GetPackagesToCheckAsync(DateTime max, int top, CancellationToken cancellationToken) + { + await _cursor.LoadAsync(cancellationToken); + var packages = await GetPackagesToCheckAsync(_cursor.Value, max, top, cancellationToken); + _pendingCursorValue = packages.Any() + ? packages.Max(GetCursorValue) + : (DateTime?)null; + + return packages + .Select(GetPackageStatusOutdatedCheck) + .ToList(); + } + + public async Task MarkPackagesCheckedAsync(CancellationToken cancellationToken) + { + if (!_pendingCursorValue.HasValue) + { + return; + } + + await SetAsync(_pendingCursorValue.Value, cancellationToken); + _pendingCursorValue = null; + } + + public async Task MoveBackAsync(DateTime value, CancellationToken cancellationToken) + { + await _cursor.LoadAsync(cancellationToken); + await SetAsync(new[] { _cursor.Value, value }.Min(), cancellationToken); + } + + private Task SetAsync(DateTime value, CancellationToken cancellationToken) + { + _cursor.Value = value; + return _cursor.SaveAsync(cancellationToken); + } + + protected abstract DateTime GetCursorValue(T package); + + protected abstract PackageStatusOutdatedCheck GetPackageStatusOutdatedCheck(T package); + + protected abstract Task> GetPackagesToCheckAsync( + DateTime since, DateTime max, int top, CancellationToken cancellationToken); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj index 40179c9d3..73ca9fdfe 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -59,6 +59,10 @@ + + + + diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs index 90d91e57f..dda649799 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs @@ -4,7 +4,6 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; -using System.Linq; namespace NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Exceptions { diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs index 4a62526c3..a27bfa8a5 100644 --- a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs @@ -130,7 +130,7 @@ public static PackageValidatorContextEnqueuer CreatePackageValidatorContextEnque return container.Resolve(); } - private static DurableCursor GetFront(Persistence.IStorageFactory storageFactory) + public static DurableCursor GetFront(Persistence.IStorageFactory storageFactory) { var storage = storageFactory.Create(); return new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); diff --git a/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs b/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs index 1b35e068b..430bc7377 100644 --- a/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs +++ b/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs @@ -405,7 +405,8 @@ internal WritePackageDetailsToCatalogAsyncTest() UtcNow.AddMinutes(-30), UtcNow.AddMinutes(-15), packageId: "a", - packageVersion: "1.0.0"); + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0.0"); } public void Dispose() diff --git a/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs b/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs index 9a85610dd..e4a00ae9d 100644 --- a/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs +++ b/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs @@ -59,6 +59,7 @@ public void PerformsCorrectProjections(bool listed, bool hideLicenseReport) // Arrange const string packageId = "Package.Id"; const string normalizedPackageVersion = "1.0.0"; + const string fullPackageVersion = "1.0.0.0"; const string licenseNames = "MIT"; const string licenseReportUrl = "https://unittest.org/licenses/MIT"; const bool requiresLicenseAcceptance = true; @@ -76,6 +77,7 @@ public void PerformsCorrectProjections(bool listed, bool hideLicenseReport) var dataRecordMock = MockDataReader( packageId, normalizedPackageVersion, + fullPackageVersion, createdDate, lastEditedDate, publishedDate, @@ -90,7 +92,8 @@ public void PerformsCorrectProjections(bool listed, bool hideLicenseReport) // Assert Assert.Equal(packageId, projection.PackageId); - Assert.Equal(normalizedPackageVersion, projection.PackageVersion); + Assert.Equal(normalizedPackageVersion, projection.PackageNormalizedVersion); + Assert.Equal(fullPackageVersion, projection.PackageFullVersion); Assert.Equal(createdDate, projection.CreatedDate); Assert.Equal(lastEditedDate, projection.LastEditedDate); Assert.Equal(expectedPublishedDate, projection.PublishedDate); @@ -104,6 +107,7 @@ public void PerformsCorrectProjections(bool listed, bool hideLicenseReport) private static Mock MockDataReader( string packageId, string normalizedPackageVersion, + string fullPackageVersion, DateTime createdDate, DateTime lastEditedDate, DateTime publishedDate, @@ -113,17 +117,18 @@ private static Mock MockDataReader( string licenseReportUrl, bool requiresLicenseAcceptance) { - const int ordinalCreated = 2; - const int ordinalLastEdited = 3; - const int ordinalPublished = 4; - const int ordinalListed = 5; - const int ordinalHideLicenseReport = 6; - const int ordinalRequiresLicenseAcceptance = 9; + const int ordinalCreated = 3; + const int ordinalLastEdited = 4; + const int ordinalPublished = 5; + const int ordinalListed = 6; + const int ordinalHideLicenseReport = 7; + const int ordinalRequiresLicenseAcceptance = 10; var dataReaderMock = new Mock(MockBehavior.Strict); dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.PackageId]).Returns(packageId); dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.NormalizedVersion]).Returns(normalizedPackageVersion); + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.FullVersion]).Returns(fullPackageVersion); dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.LicenseNames]).Returns(licenseNames); dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.LicenseReportUrl]).Returns(licenseReportUrl); diff --git a/tests/CatalogTests/Helpers/FeedHelpersTests.cs b/tests/CatalogTests/Helpers/FeedHelpersTests.cs index 852f015c4..8867e34d0 100644 --- a/tests/CatalogTests/Helpers/FeedHelpersTests.cs +++ b/tests/CatalogTests/Helpers/FeedHelpersTests.cs @@ -78,7 +78,7 @@ private bool ArePackagesEqual(FeedPackageDetails feedPackage, ODataPackage oData { return feedPackage.PackageId == oDataPackage.Id && - feedPackage.PackageVersion == oDataPackage.Version && + feedPackage.PackageNormalizedVersion == oDataPackage.Version && feedPackage.ContentUri.ToString() == $"{_baseUri}/package/{oDataPackage.Id}/{NuGetVersion.Parse(oDataPackage.Version).ToNormalizedString()}" && feedPackage.CreatedDate.Ticks == oDataPackage.Created.Ticks && feedPackage.LastEditedDate.Ticks == oDataPackage.LastEdited.Value.Ticks && diff --git a/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs b/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs index 85a1a7b7e..f88c4af9b 100644 --- a/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs +++ b/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs @@ -107,14 +107,16 @@ public void OrdersBySelectedPropertyDescending() lastEditedDate: utcNow, publishedDate: utcNow.AddDays(-2), packageId: "Package.Id", - packageVersion: "1.0.1"); + packageNormalizedVersion: "1.0.1", + packageFullVersion: "1.0.1"); var firstEditedPackage = new FeedPackageDetails( new Uri("https://unittest.org/packages/Package.Id/1.0.0"), createdDate: utcNow.AddDays(-1), lastEditedDate: utcNow.AddHours(-8), publishedDate: utcNow.AddDays(-1), packageId: "Package.Id", - packageVersion: "1.0.0"); + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0"); var packages = new List { @@ -153,7 +155,8 @@ public void WillSkipLastTimestampIfItWouldResultInPageOverflow() lastEditedDate: utcNow.AddDays(-i), publishedDate: utcNow.AddDays(-i), packageId: $"Package.Id{i}", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Ensure the last timestamp has enough packages to have it exceed the max page size. @@ -167,7 +170,8 @@ public void WillSkipLastTimestampIfItWouldResultInPageOverflow() lastEditedDate: lastTimestamp, publishedDate: utcNow.AddHours(-9), packageId: "BatchUpdatedPackage.Id", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Act @@ -196,7 +200,8 @@ public void WillSkipAnyTimestampThatWouldResultInPageOverflow() lastEditedDate: utcNow.AddDays(-i), publishedDate: utcNow.AddDays(-i), packageId: $"Package.Id{i}", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Ensure the next timestamp has enough packages to have it reach the max page size. @@ -210,7 +215,8 @@ public void WillSkipAnyTimestampThatWouldResultInPageOverflow() lastEditedDate: timestampForBulkChanges, publishedDate: utcNow.AddHours(-9), packageId: "BatchUpdatedPackage.Id", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Add some more timestamps with a package each to the end of the top 20 resultset. @@ -222,7 +228,8 @@ public void WillSkipAnyTimestampThatWouldResultInPageOverflow() lastEditedDate: utcNow.AddMinutes(-i), publishedDate: utcNow.AddDays(-i), packageId: $"Another.Package.Id{i}", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Act @@ -251,7 +258,8 @@ public void WillNotSkipSingleTimestampIfThatWouldResultInPageOverflow() lastEditedDate: timestampForBulkChanges, publishedDate: utcNow.AddDays(-i), packageId: $"BatchUpdatedPackage.Id{i}", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Add 19 more timestamps to simulate top 20. @@ -263,7 +271,8 @@ public void WillNotSkipSingleTimestampIfThatWouldResultInPageOverflow() lastEditedDate: utcNow.AddMinutes(-i), publishedDate: utcNow.AddHours(-9), packageId: "Package.Id", - packageVersion: $"1.0.{i}")); + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); } // Act diff --git a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs index fc1250472..29e3b5768 100644 --- a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs +++ b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs @@ -112,7 +112,8 @@ public async Task CreateAsync_WhenCancellationTokenIsCancelled_Throws() DateTime.UtcNow, DateTime.UtcNow, packageId: "a", - packageVersion: "1.0.0"); + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0"); await Assert.ThrowsAsync( () => creator.CreateAsync(packageItem, DateTime.UtcNow, new CancellationToken(canceled: true))); @@ -383,6 +384,7 @@ internal Test(bool useStorage = true) Timestamp.AddHours(-2), Timestamp.AddHours(-1), PackageId, + PackageVersion, PackageVersion); var stream = TestHelper.GetStream(PackageFileName); diff --git a/tests/NgTests/Db2CatalogTests.cs b/tests/NgTests/Db2CatalogTests.cs index fbf9b959a..5bfd99fd8 100644 --- a/tests/NgTests/Db2CatalogTests.cs +++ b/tests/NgTests/Db2CatalogTests.cs @@ -1072,14 +1072,16 @@ private SortedList> GetCreatedPackages() lastEditedDate: DateTime.MinValue, publishedDate: new DateTime(2015, 1, 1).ForceUtc(), packageId: "ListedPackage", - packageVersion: "1.0.0"), + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0.0"), new FeedPackageDetails( contentUri: _packageContentUriBuilder.Build("UnlistedPackage", "1.0.0"), createdDate: new DateTime(2015, 1, 1).ForceUtc(), lastEditedDate: DateTime.MinValue, publishedDate: Constants.UnpublishedDate, packageId: "UnlistedPackage", - packageVersion: "1.0.0"), + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0+metadata"), // The real SemVer2 version is embedded in the nupkg. // The below FeedPackageDetails entity expects normalized versions. @@ -1089,7 +1091,8 @@ private SortedList> GetCreatedPackages() lastEditedDate: DateTime.MinValue, publishedDate: new DateTime(2015, 1, 1).ForceUtc(), packageId: "TestPackage.SemVer2", - packageVersion: "1.0.0-alpha.1") + packageNormalizedVersion: "1.0.0-alpha.1", + packageFullVersion: "1.0.0-alpha.1") }; return GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.CreatedDate); @@ -1105,7 +1108,8 @@ private SortedList> GetEditedPackages() lastEditedDate: new DateTime(2015, 1, 1).ForceUtc(), publishedDate: new DateTime(2014, 1, 1).ForceUtc(), packageId: "ListedPackage", - packageVersion: "1.0.1") + packageNormalizedVersion: "1.0.1", + packageFullVersion: "1.0.1") }; return GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.LastEditedDate); @@ -1126,7 +1130,8 @@ private SortedList> GetCreatedPackagesSecond lastEditedDate: new DateTime(2015, 1, 1, 1, 1, 3).ForceUtc(), publishedDate: new DateTime(2015, 1, 1, 1, 1, 3).ForceUtc(), packageId: "OtherPackage", - packageVersion: "1.0.0") + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0") }; return GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.CreatedDate); @@ -1166,13 +1171,15 @@ private PackageCreationOrEdit CreatePackageCreationOrEdit(DateTime? createdDate } var normalizedVersion = package.Version.ToNormalizedString(); + var fullVersion = package.Version.ToFullString(); var feedPackageDetails = new FeedPackageDetails( contentUri: _packageContentUriBuilder.Build(package.Id, normalizedVersion), createdDate: created.UtcDateTime, lastEditedDate: DateTime.MinValue, publishedDate: isListed ? created.UtcDateTime : Constants.UnpublishedDate, packageId: package.Id, - packageVersion: normalizedVersion, + packageNormalizedVersion: normalizedVersion, + packageFullVersion: fullVersion, licenseNames: null, licenseReportUrl: null, deprecationInfo: null, @@ -1207,7 +1214,8 @@ private PackageCreationOrEdit AddEditedPackageToFeed(PackageCreationOrEdit entry lastEditedDate: edited, publishedDate: entry.FeedPackageDetails.PublishedDate, packageId: entry.FeedPackageDetails.PackageId, - packageVersion: entry.FeedPackageDetails.PackageVersion, + packageNormalizedVersion: entry.FeedPackageDetails.PackageNormalizedVersion, + packageFullVersion: entry.FeedPackageDetails.PackageFullVersion, licenseNames: entry.FeedPackageDetails.LicenseNames, licenseReportUrl: entry.FeedPackageDetails.LicenseReportUrl, deprecationInfo: entry.FeedPackageDetails.DeprecationInfo, diff --git a/tests/NgTests/PackageMonitoringStatusServiceTests.cs b/tests/NgTests/PackageMonitoringStatusServiceTests.cs index 3d7daade1..4344f964c 100644 --- a/tests/NgTests/PackageMonitoringStatusServiceTests.cs +++ b/tests/NgTests/PackageMonitoringStatusServiceTests.cs @@ -307,7 +307,7 @@ public async Task GetByPackageDeletesOutdatedStatuses(PackageState latest, Packa // Assert PackageMonitoringStatusTestUtility.AssertStatus(latestStatus, status); - Assert.False(DoesPackageExists(storageFactory, outdatedStatus.State, package)); + Assert.Equal(latest == outdated, DoesPackageExists(storageFactory, outdatedStatus.State, package)); Assert.True(DoesPackageExists(storageFactory, latestStatus.State, package)); } @@ -422,7 +422,7 @@ private Task SaveToStorage(MemoryStorageFactory storageFactory, PackageMonitorin var json = JsonConvert.SerializeObject(status, JsonSerializerUtility.SerializerSettings); var content = new StringStorageContentWithAccessCondition( json, - AccessCondition.GenerateIfNotExistsCondition(), + AccessCondition.GenerateEmptyCondition(), "application/json"); return SaveToStorage(storageFactory, status.State, status.Package, content);