Merge pull request #532 from NuGet/dev

[ReleasePrep][2019.05.28]RI of dev into master

build.ps1

@@ -117,7 +117,7 @@ Invoke-BuildStep 'Creating artifacts' {
             "src\Ng\Catalog2Lucene.nuspec", `
             "src\Ng\Catalog2Monitoring.nuspec", `
             "src\Ng\Catalog2Registration.nuspec", `
-            "src\Ng\Feed2Catalog.nuspec", `
+            "src\Ng\Db2Catalog.nuspec", `
             "src\Ng\Monitoring2Monitoring.nuspec", `
             "src\Ng\MonitoringProcessor.nuspec", `
             "src\Ng\Ng.Operations.nuspec", `

src/Catalog/Constants.cs

@@ -7,7 +7,7 @@ namespace NuGet.Services.Metadata.Catalog
 {
     public static class Constants
     {
-        public static readonly DateTime DateTimeMinValueUtc = new DateTime(0L, DateTimeKind.Utc);
+        public static readonly DateTime DateTimeMinValueUtc = DateTimeOffset.MinValue.UtcDateTime;
         public const int MaxPageSize = 550;
         public const string Sha512 = "SHA512";
         public static readonly DateTime UnpublishedDate = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc);

src/Catalog/Extensions/DateTimeExtensions.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace System
+{
+    public static class DateTimeExtensions
+    {
+        public static DateTime ForceUtc(this DateTime date)
+        {
+            if (date.Kind != DateTimeKind.Utc)
+            {
+                date = new DateTime(date.Ticks, DateTimeKind.Utc);
+            }
+
+            return date;
+        }
+    }
+}

src/Catalog/Extensions/IDataRecordExtensions.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace System.Data
+{
+    /// <summary>
+    /// Extension methods that make working with <see cref="IDataRecord"/> more convenient.
+    /// </summary>
+    public static class IDataRecordExtensions
+    {
+        public static DateTime ReadNullableUtcDateTime(this IDataRecord dataRecord, string columnName)
+        {
+            if (dataRecord == null)
+            {
+                throw new ArgumentNullException(nameof(dataRecord));
+            }
+
+            if (columnName == null)
+            {
+                throw new ArgumentNullException(nameof(columnName));
+            }
+
+            return (dataRecord[columnName] == DBNull.Value ? DateTime.MinValue : ReadDateTime(dataRecord, columnName)).ForceUtc();
+        }
+
+        public static DateTime ReadDateTime(this IDataRecord dataRecord, string columnName)
+        {
+            if (dataRecord == null)
+            {
+                throw new ArgumentNullException(nameof(dataRecord));
+            }
+
+            if (columnName == null)
+            {
+                throw new ArgumentNullException(nameof(columnName));
+            }
+
+            return dataRecord.GetDateTime(dataRecord.GetOrdinal(columnName));
+        }
+    }
+}

src/Catalog/Helpers/Db2CatalogCursor.cs

@@ -0,0 +1,34 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Data.SqlTypes;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+    public sealed class Db2CatalogCursor
+    {
+        public const string ColumnNameCreated = "Created";
+        public const string ColumnNameLastEdited = "LastEdited";
+
+        private Db2CatalogCursor(string columnName, DateTime cursorValue, int top)
+        {
+            ColumnName = columnName ?? throw new ArgumentNullException(nameof(columnName));
+            CursorValue = cursorValue < SqlDateTime.MinValue.Value ? SqlDateTime.MinValue.Value : cursorValue;
+
+            if (top <= 0)
+            {
+                throw new ArgumentOutOfRangeException("Argument value must be a positive non-zero integer.", nameof(top));
+            }
+
+            Top = top;
+        }
+
+        public string ColumnName { get; }
+        public DateTime CursorValue { get; }
+        public int Top { get; }
+
+        public static Db2CatalogCursor ByCreated(DateTime since, int top) => new Db2CatalogCursor(ColumnNameCreated, since, top);
+        public static Db2CatalogCursor ByLastEdited(DateTime since, int top) => new Db2CatalogCursor(ColumnNameLastEdited, since, top);
+    }
+}

src/Catalog/Helpers/Db2CatalogProjection.cs

@@ -0,0 +1,47 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Data;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+    /// <summary>
+    /// Utility class to project <see cref="IDataRecord"/>s retrieved by db2catalog
+    /// into a format that is consumable by the catalog writer.
+    /// </summary>
+    public class Db2CatalogProjection
+    {
+        private readonly PackageContentUriBuilder _packageContentUriBuilder;
+
+        public Db2CatalogProjection(PackageContentUriBuilder packageContentUriBuilder)
+        {
+            _packageContentUriBuilder = packageContentUriBuilder ?? throw new ArgumentNullException(nameof(packageContentUriBuilder));
+        }
+
+        public FeedPackageDetails FromDataRecord(IDataRecord dataRecord)
+        {
+            if (dataRecord == null)
+            {
+                throw new ArgumentNullException(nameof(dataRecord));
+            }
+
+            var packageId = dataRecord["Id"].ToString();
+            var normalizedPackageVersion = dataRecord["NormalizedVersion"].ToString();
+            var listed = dataRecord.GetBoolean(dataRecord.GetOrdinal("Listed"));
+            var hideLicenseReport = dataRecord.GetBoolean(dataRecord.GetOrdinal("HideLicenseReport"));
+
+            var packageContentUri = _packageContentUriBuilder.Build(packageId, normalizedPackageVersion);
+
+            return new FeedPackageDetails(
+                packageContentUri,
+                dataRecord.ReadDateTime("Created").ForceUtc(),
+                dataRecord.ReadNullableUtcDateTime("LastEdited"),
+                listed ? dataRecord.ReadDateTime("Published").ForceUtc() : Constants.UnpublishedDate,
+                packageId,
+                normalizedPackageVersion,
+                hideLicenseReport ? null : dataRecord["LicenseNames"]?.ToString(),
+                hideLicenseReport ? null : dataRecord["LicenseReportUrl"]?.ToString());
+        }
+    }
+}

src/Catalog/Helpers/FeedHelpers.cs

@@ -140,9 +140,9 @@ public static async Task<IList<FeedPackageDetails>> GetPackages(HttpClient clien
 
                 // NOTE that DateTime returned by the v2 feed does not have Z at the end even though it is in UTC. So, the DateTime kind is unspecified
                 // So, forcibly convert it to UTC here
-                createdDate = ForceUtc(createdDate);
-                lastEditedDate = ForceUtc(lastEditedDate);
-                publishedDate = ForceUtc(publishedDate);
+                createdDate = createdDate.ForceUtc();
+                lastEditedDate = lastEditedDate.ForceUtc();
+                publishedDate = publishedDate.ForceUtc();
 
                 packages.Add(new FeedPackageDetails(
                     content,
@@ -158,15 +158,6 @@ public static async Task<IList<FeedPackageDetails>> GetPackages(HttpClient clien
             return packages;
         }
 
-        private static DateTime ForceUtc(DateTime date)
-        {
-            if (date.Kind == DateTimeKind.Unspecified)
-            {
-                date = new DateTime(date.Ticks, DateTimeKind.Utc);
-            }
-            return date;
-        }
-
         /// <summary>
         /// Asynchronously writes package metadata to the catalog.
         /// </summary>

src/Catalog/Helpers/GalleryDatabaseQueryService.cs

@@ -0,0 +1,173 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.Threading.Tasks;
+using NuGet.Services.Entities;
+using NuGet.Services.Sql;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+    /// <summary>
+    /// Utility class for all SQL queries invoked by Db2Catalog.
+    /// </summary>
+    public class GalleryDatabaseQueryService : IGalleryDatabaseQueryService
+    {
+        private const string CursorParameterName = "Cursor";
+
+        private readonly ISqlConnectionFactory _connectionFactory;
+        private readonly Db2CatalogProjection _db2catalogProjection;
+        private readonly ITelemetryService _telemetryService;
+        private readonly int _commandTimeout;
+
+        public GalleryDatabaseQueryService(
+            ISqlConnectionFactory connectionFactory,
+            PackageContentUriBuilder packageContentUriBuilder,
+            ITelemetryService telemetryService,
+            int commandTimeout)
+        {
+            _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
+            _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+            _db2catalogProjection = new Db2CatalogProjection(packageContentUriBuilder);
+            _commandTimeout = commandTimeout;
+        }
+
+        public Task<SortedList<DateTime, IList<FeedPackageDetails>>> GetPackagesCreatedSince(DateTime since, int top)
+        {
+            return GetPackagesInOrder(
+                package => package.CreatedDate,
+                Db2CatalogCursor.ByCreated(since, top));
+        }
+
+        public Task<SortedList<DateTime, IList<FeedPackageDetails>>> GetPackagesEditedSince(DateTime since, int top)
+        {
+            return GetPackagesInOrder(
+                package => package.LastEditedDate,
+                Db2CatalogCursor.ByLastEdited(since, top));
+        }
+
+        /// <summary>
+        /// Returns a <see cref="SortedList{DateTime, IList{FeedPackageDetails}}"/> from the gallery database.
+        /// </summary>
+        /// <param name="keyDateFunc">The <see cref="DateTime"/> field to sort the <see cref="FeedPackageDetails"/> on.</param>
+        private async Task<SortedList<DateTime, IList<FeedPackageDetails>>> GetPackagesInOrder(
+            Func<FeedPackageDetails, DateTime> keyDateFunc,
+            Db2CatalogCursor cursor)
+        {
+            var allPackages = await GetPackages(cursor);
+
+            return OrderPackagesByKeyDate(allPackages, keyDateFunc);
+        }
+
+        /// <summary>
+        /// Returns a <see cref="SortedList{DateTime, IList{FeedPackageDetails}}"/> of packages.
+        /// </summary>
+        /// <param name="keyDateFunc">The <see cref="DateTime"/> field to sort the <see cref="FeedPackageDetails"/> on.</param>
+        internal static SortedList<DateTime, IList<FeedPackageDetails>> OrderPackagesByKeyDate(
+            IReadOnlyCollection<FeedPackageDetails> packages,
+            Func<FeedPackageDetails, DateTime> keyDateFunc)
+        {
+            var result = new SortedList<DateTime, IList<FeedPackageDetails>>();
+
+            foreach (var package in packages)
+            {
+                var packageKeyDate = keyDateFunc(package);
+                if (!result.TryGetValue(packageKeyDate, out IList<FeedPackageDetails> packagesWithSameKeyDate))
+                {
+                    packagesWithSameKeyDate = new List<FeedPackageDetails>();
+                    result.Add(packageKeyDate, packagesWithSameKeyDate);
+                }
+
+                packagesWithSameKeyDate.Add(package);
+            }
+
+            var packagesCount = 0;
+            var filteredResult = new SortedList<DateTime, IList<FeedPackageDetails>>();
+            foreach (var keyDate in result.Keys)
+            {
+                if (result.TryGetValue(keyDate, out IList<FeedPackageDetails> packagesForKeyDate))
+                {
+                    if (packagesCount > 0 && packagesCount + packagesForKeyDate.Count > Constants.MaxPageSize)
+                    {
+                        break;
+                    }
+
+                    packagesCount += packagesForKeyDate.Count;
+                    filteredResult.Add(keyDate, packagesForKeyDate);
+                }
+            }
+
+            return filteredResult;
+        }
+
+        /// <summary>
+        /// Builds the SQL query string for db2catalog.
+        /// </summary>
+        /// <param name="cursor">The <see cref="Db2CatalogCursor"/> to be used.</param>
+        /// <returns>The SQL query string for the db2catalog job, build from the the provided <see cref="Db2CatalogCursor"/>.</returns>
+        internal static string BuildDb2CatalogSqlQuery(Db2CatalogCursor cursor)
+        {
+            return $@"SELECT TOP {cursor.Top} WITH TIES
+                        PR.[Id],
+                        P.[NormalizedVersion],
+                        P.[Created],
+                        P.[LastEdited],
+                        P.[Published],
+                        P.[Listed],
+                        P.[HideLicenseReport],
+                        P.[LicenseNames],
+                        P.[LicenseReportUrl]
+                    FROM [dbo].[Packages] AS P
+                    INNER JOIN [dbo].[PackageRegistrations] AS PR ON P.[PackageRegistrationKey] = PR.[Key]
+                    WHERE P.[PackageStatusKey] = {(int)PackageStatus.Available} 
+                        AND P.[{cursor.ColumnName}] > @{CursorParameterName}
+                    ORDER BY P.[{cursor.ColumnName}]";
+        }
+
+        /// <summary>
+        /// Asynchronously gets a <see cref="IReadOnlyCollection{FeedPackageDetails}"/> from the gallery database.
+        /// </summary>
+        /// <param name="cursor">Defines the cursor to be used.</param>
+        /// <returns>A task that represents the asynchronous operation.
+        /// The task result (<see cref="Task{TResult}.Result" />) returns an
+        /// <see cref="IReadOnlyCollection{FeedPackageDetails}" />.</returns>
+        private async Task<IReadOnlyCollection<FeedPackageDetails>> GetPackages(Db2CatalogCursor cursor)
+        {
+            using (var sqlConnection = await _connectionFactory.OpenAsync())
+            {
+                return await GetPackageDetailsAsync(sqlConnection, cursor);
+            }
+        }
+
+        private async Task<IReadOnlyCollection<FeedPackageDetails>> GetPackageDetailsAsync(
+            SqlConnection sqlConnection,
+            Db2CatalogCursor cursor)
+        {
+            var packages = new List<FeedPackageDetails>();
+            var packageQuery = BuildDb2CatalogSqlQuery(cursor);
+
+            using (var packagesCommand = new SqlCommand(packageQuery, sqlConnection)
+            {
+                CommandTimeout = _commandTimeout
+            })
+            {
+                packagesCommand.Parameters.AddWithValue(CursorParameterName, cursor.CursorValue);
+
+                using (_telemetryService.TrackGetPackageDetailsQueryDuration(cursor))
+                {
+                    using (var packagesReader = await packagesCommand.ExecuteReaderAsync())
+                    {
+                        while (await packagesReader.ReadAsync())
+                        {
+                            packages.Add(_db2catalogProjection.FromDataRecord(packagesReader));
+                        }
+                    }
+                }
+            }
+
+            return packages;
+        }
+    }
+}

src/Catalog/Helpers/IGalleryDatabaseQueryService.cs

@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+    public interface IGalleryDatabaseQueryService
+    {
+        Task<SortedList<DateTime, IList<FeedPackageDetails>>> GetPackagesCreatedSince(DateTime since, int top);
+        Task<SortedList<DateTime, IList<FeedPackageDetails>>> GetPackagesEditedSince(DateTime since, int top);
+    }
+}