Add RegistrationDeprecationValidator to verify deprecation changes ar…

…e properly consumed (#554)

src/Catalog/Helpers/Db2CatalogProjection.cs

@@ -49,7 +49,8 @@ public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader data
                 normalizedPackageVersion,
                 hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseNames]?.ToString(),
                 hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseReportUrl]?.ToString(),
-                deprecationInfo);
+                deprecationInfo,
+                dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.RequiresLicenseAcceptance)));
         }
 
         public PackageDeprecationItem ReadDeprecationInfoFromDataReader(DbDataReader dataReader)

src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs

@@ -21,5 +21,6 @@ public static class Db2CatalogProjectionColumnNames
         public const string AlternatePackageVersion = "AlternatePackageVersion";
         public const string DeprecationStatus = "DeprecationStatus";
         public const string DeprecationMessage = "DeprecationMessage";
+        public const string RequiresLicenseAcceptance = "RequiresLicenseAcceptance";
     }
 }

src/Catalog/Helpers/FeedHelpers.cs

@@ -2,11 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Collections.Generic;
 using System.Net.Http;
-using System.Threading.Tasks;
-using System.Web;
-using System.Xml.Linq;
 
 namespace NuGet.Services.Metadata.Catalog.Helpers
 {
@@ -23,101 +19,5 @@ public static HttpClient CreateHttpClient(Func<HttpMessageHandler> handlerFunc)
             var handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true };
             return new HttpClient(handler);
         }
-
-        /// <summary>
-        /// Builds a <see cref="Uri"/> for accessing the metadata of a specific package on the feed.
-        /// </summary>
-        public static Uri MakeUriForPackage(string source, string id, string version)
-        {
-            var uri = new Uri($"{source.Trim('/')}/Packages(Id='{HttpUtility.UrlEncode(id)}',Version='{HttpUtility.UrlEncode(version)}')");
-            return UriUtils.GetNonhijackableUri(uri);
-        }
-
-        /// <summary>
-        /// Asynchronously gets a <see cref="IList{FeedPackageDetails}"/> from the feed.
-        /// </summary>
-        /// <param name="client">An HTTP client.</param>
-        /// <param name="uri">The feed URI.</param>
-        /// <returns>A task that represents the asynchronous operation.
-        /// The task result (<see cref="Task{TResult}.Result" />) returns an
-        /// <see cref="IList{FeedPackageDetails}" />.</returns>
-        public static async Task<IList<FeedPackageDetails>> GetPackages(HttpClient client, Uri uri)
-        {
-            const string createdDateProperty = "Created";
-            const string lastEditedDateProperty = "LastEdited";
-            const string publishedDateProperty = "Published";
-            const string idProperty = "Id";
-            const string normalizedVersionProperty = "NormalizedVersion";
-            const string licenseNamesProperty = "LicenseNames";
-            const string licenseReportUrlProperty = "LicenseReportUrl";
-
-            var packages = new List<FeedPackageDetails>();
-
-            XElement feed;
-            try
-            {
-                using (var stream = await client.GetStreamAsync(uri))
-                {
-                    feed = XElement.Load(stream);
-                }
-            }
-            catch (TaskCanceledException tce)
-            {
-                // If the HTTP request timed out, a TaskCanceledException will be thrown.
-                throw new HttpClientTimeoutException($"HttpClient request timed out in {nameof(FeedHelpers.GetPackages)}.", tce);
-            }
-
-            XNamespace atom = "http://www.w3.org/2005/Atom";
-            XNamespace dataservices = "http://schemas.microsoft.com/ado/2007/08/dataservices";
-            XNamespace metadata = "http://schemas.microsoft.com/ado/2007/08/dataservices/metadata";
-
-            foreach (var entry in feed.Elements(atom + "entry"))
-            {
-                var content = new Uri(entry.Element(atom + "content").Attribute("src").Value);
-
-                var propertiesElement = entry.Element(metadata + "properties");
-
-                var packageIdElement = propertiesElement.Element(dataservices + idProperty);
-                var packageId = packageIdElement?.Value;
-                var packageNormalizedVersionElement = propertiesElement.Element(dataservices + normalizedVersionProperty);
-                var packageNormalizedVersion = packageNormalizedVersionElement?.Value;
-
-                var createdElement = propertiesElement.Element(dataservices + createdDateProperty);
-                var createdValue = createdElement?.Value;
-                var createdDate = string.IsNullOrEmpty(createdValue) ? DateTime.MinValue : DateTime.Parse(createdValue);
-
-                var lastEditedValue = propertiesElement.Element(dataservices + lastEditedDateProperty).Value;
-                var lastEditedDate = string.IsNullOrEmpty(lastEditedValue) ? DateTime.MinValue : DateTime.Parse(lastEditedValue);
-
-                var publishedValue = propertiesElement.Element(dataservices + publishedDateProperty).Value;
-                var publishedDate = string.IsNullOrEmpty(publishedValue) ? createdDate : DateTime.Parse(publishedValue);
-
-                // License details
-                var licenseNamesElement = propertiesElement.Element(dataservices + licenseNamesProperty);
-                var licenseNames = licenseNamesElement?.Value;
-
-                var licenseReportUrlElement = propertiesElement.Element(dataservices + licenseReportUrlProperty);
-                var licenseReportUrl = licenseReportUrlElement?.Value;
-
-                // NOTE that DateTime returned by the v2 feed does not have Z at the end even though it is in UTC. So, the DateTime kind is unspecified
-                // So, forcibly convert it to UTC here
-                createdDate = createdDate.ForceUtc();
-                lastEditedDate = lastEditedDate.ForceUtc();
-                publishedDate = publishedDate.ForceUtc();
-
-                packages.Add(new FeedPackageDetails(
-                    content,
-                    createdDate,
-                    lastEditedDate,
-                    publishedDate,
-                    packageId,
-                    packageNormalizedVersion,
-                    licenseNames,
-                    licenseReportUrl,
-                    deprecationInfo: null));
-            }
-
-            return packages;
-        }
     }
 }

src/Catalog/Helpers/FeedPackageDetails.cs

@@ -15,6 +15,7 @@ public sealed class FeedPackageDetails
         public string PackageVersion { get; }
         public string LicenseNames { get; }
         public string LicenseReportUrl { get; }
+        public bool RequiresLicenseAcceptance { get; }
         public PackageDeprecationItem DeprecationInfo { get; }
 
         public bool HasDeprecationInfo => DeprecationInfo != null;
@@ -35,7 +36,8 @@ public FeedPackageDetails(
                 packageVersion,
                 licenseNames: null,
                 licenseReportUrl: null,
-                deprecationInfo: null)
+                deprecationInfo: null,
+                requiresLicenseAcceptance: false)
         {
         }
 
@@ -48,7 +50,8 @@ public FeedPackageDetails(
             string packageVersion,
             string licenseNames,
             string licenseReportUrl,
-            PackageDeprecationItem deprecationInfo)
+            PackageDeprecationItem deprecationInfo,
+            bool requiresLicenseAcceptance)
         {
             ContentUri = contentUri;
             CreatedDate = createdDate;
@@ -59,6 +62,7 @@ public FeedPackageDetails(
             LicenseNames = licenseNames;
             LicenseReportUrl = licenseReportUrl;
             DeprecationInfo = deprecationInfo;
+            RequiresLicenseAcceptance = requiresLicenseAcceptance;
         }
     }
 }

src/Catalog/Helpers/GalleryDatabaseQueryService.cs

@@ -29,6 +29,7 @@ public class GalleryDatabaseQueryService : IGalleryDatabaseQueryService
                         P.[HideLicenseReport],
                         P.[LicenseNames],
                         P.[LicenseReportUrl],
+                        P.[RequiresLicenseAcceptance],
                         PD.[Status] AS '{Db2CatalogProjectionColumnNames.DeprecationStatus}',
                         APR.[Id] AS '{Db2CatalogProjectionColumnNames.AlternatePackageId}',
                         AP.[NormalizedVersion] AS '{Db2CatalogProjectionColumnNames.AlternatePackageVersion}',

src/Catalog/Helpers/JsonSort.cs

@@ -4,8 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace NuGet.Services.Metadata.Catalog.Helpers
 {

src/Catalog/Helpers/UriUtils.cs

@@ -4,9 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net.Http;
 using System.Text.RegularExpressions;
-using System.Threading.Tasks;
 using NuGet.Versioning;
 
 namespace NuGet.Services.Metadata.Catalog.Helpers

src/Catalog/Helpers/XsltHelper.cs

@@ -1,8 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System.Collections.Generic;
-using System.Linq;
 using System.Xml;
 using System.Xml.XPath;
 using NuGet.Services.Metadata.Catalog.Helpers;

src/Catalog/PackageCatalogItem.cs

@@ -152,13 +152,14 @@ public override IGraph CreateContentGraph(CatalogContext context)
             return graph;
         }
 
-        private bool GetListed(DateTime published)
+        public static bool GetListed(DateTime published)
         {
             //If the published date is 1900/01/01, then the package is unlisted
-            if (published.ToUniversalTime() == Convert.ToDateTime("1900-01-01T00:00:00Z").ToUniversalTime())
+            if (published.ToUniversalTime() == Constants.UnpublishedDate)
             {
                 return false;
             }
+
             return true;
         }
 

src/Catalog/SingleGraphPersistence.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Newtonsoft.Json.Linq;
 using NuGet.Services.Metadata.Catalog.Persistence;
 using VDS.RDF;
 

src/Catalog/VerboseFileSystemEmulatorHandler.cs

@@ -1,6 +1,5 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-using System;
 using System.Diagnostics;
 using System.Net.Http;
 using System.Threading;

src/Ng/Jobs/Catalog2PackageFixupJob.cs

@@ -7,7 +7,6 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Net.Http;
-using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;

src/Ng/Jobs/ClearLuceneJob.cs

@@ -1,7 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;

src/Ng/Jobs/MonitoringProcessorJob.cs

@@ -122,7 +122,7 @@ protected override async Task RunInternalAsync(CancellationToken cancellationTok
 
         private async Task ProcessPackagesAsync(CancellationToken token)
         {
-            StorageQueueMessage<PackageValidatorContext> queueMessage = null;
+            StorageQueueMessage<PackageValidatorContext> queueMessage;
             do
             {
                 Logger.LogInformation("Fetching next queue message.");

src/NuGet.ApplicationInsights.Owin/ExceptionTrackingMiddleware.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.ComponentModel;
 using System.Threading.Tasks;
 using Microsoft.ApplicationInsights;
 using Microsoft.ApplicationInsights.Extensibility;

src/NuGet.Indexing/ChainedFilter.cs

@@ -1,8 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Text;
 
 using Lucene.Net.Search;

src/NuGet.Indexing/DescriptionAnalyzer.cs

@@ -1,7 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 using Lucene.Net.Analysis;
-using System.Collections.Generic;
 using System.IO;
 
 namespace NuGet.Indexing

src/NuGet.Indexing/DownloadsByVersion.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 
 namespace NuGet.Indexing
 {

src/NuGet.Indexing/FileLoader.cs

@@ -4,7 +4,6 @@
 using Newtonsoft.Json;
 using System.IO;
 using System;
-using System.Threading.Tasks;
 
 namespace NuGet.Indexing
 {

src/NuGet.Indexing/IndexConsistencyReport.cs

@@ -1,9 +1,5 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-    using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 using Newtonsoft.Json.Linq;
 
 namespace NuGet.Indexing

src/NuGet.Indexing/IndexDirectoryProvider/IIndexDirectoryProvider.cs

@@ -1,5 +1,4 @@
-using System.Threading.Tasks;
-using Lucene.Net.Store;
+using Lucene.Net.Store;
 
 namespace NuGet.Indexing.IndexDirectoryProvider
 {

src/NuGet.Indexing/IndexDirectoryProvider/LocalIndexDirectoryProvider.cs

@@ -1,5 +1,4 @@
-using System.Threading.Tasks;
-using Lucene.Net.Store;
+using Lucene.Net.Store;
 
 namespace NuGet.Indexing.IndexDirectoryProvider
 {

src/NuGet.Indexing/KeyCollector.cs

@@ -1,7 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 using Lucene.Net.Search;
-using Newtonsoft.Json.Linq;
 using System.Collections.Generic;
 
 namespace NuGet.Indexing

src/NuGet.Indexing/OwnerAnalyzer.cs

@@ -1,7 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 using Lucene.Net.Analysis;
-using System.Collections.Generic;
 using System.IO;
 
 namespace NuGet.Indexing

src/NuGet.Indexing/PerfEventTracker.cs

@@ -3,10 +3,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace NuGet.Indexing
 {

src/NuGet.Indexing/SearcherManagerT.cs

@@ -6,7 +6,6 @@
 using Lucene.Net.Index;
 using Lucene.Net.Search;
 using Lucene.Net.Store;
-using System.Threading.Tasks;
 
 namespace NuGet.Indexing
 {

src/NuGet.Indexing/SegmentInfoEventArgs.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 
 namespace NuGet.Indexing
 {

src/NuGet.Indexing/TenantFilter.cs

@@ -3,7 +3,6 @@
 using Lucene.Net.Index;
 using Lucene.Net.Search;
 using Lucene.Net.Util;
-using System;
 
 namespace NuGet.Indexing
 {

src/NuGet.Indexing/TypeFilter.cs

@@ -2,8 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 using Lucene.Net.Index;
 using Lucene.Net.Search;
-using Lucene.Net.Util;
-using System;
 
 namespace NuGet.Indexing
 {

src/NuGet.Services.BasicSearch/Configuration/SecretReaderFactory.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Security.Cryptography.X509Certificates;
 using System.Threading.Tasks;
 using NuGet.Services.Configuration;
 using NuGet.Services.KeyVault;