Deprecation metadata should only be present in SemVer 2.0.0 hive (#546)

src/Catalog/Registration/RegistrationCollector.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
@@ -24,7 +25,8 @@ public class RegistrationCollector : SortingGraphCollector
 
         private readonly StorageFactory _legacyStorageFactory;
         private readonly StorageFactory _semVer2StorageFactory;
-        private readonly ShouldIncludeRegistrationPackage _shouldIncludeSemVer2;
+        private readonly ShouldIncludeRegistrationPackage _shouldIncludeSemVer2ForLegacyStorageFactory;
+        private readonly RegistrationMakerCatalogItem.PostProcessGraph _postProcessGraphForLegacyStorageFactory;
         private readonly int _maxConcurrentBatches;
         private readonly ILogger _logger;
 
@@ -49,7 +51,8 @@ public RegistrationCollector(
             _legacyStorageFactory = legacyStorageFactory ?? throw new ArgumentNullException(nameof(legacyStorageFactory));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _semVer2StorageFactory = semVer2StorageFactory;
-            _shouldIncludeSemVer2 = GetShouldIncludeRegistrationPackage(_semVer2StorageFactory);
+            _shouldIncludeSemVer2ForLegacyStorageFactory = GetShouldIncludeRegistrationPackageForLegacyStorageFactory(_semVer2StorageFactory);
+            _postProcessGraphForLegacyStorageFactory = GetPostProcessGraphForLegacyStorageFactory(_semVer2StorageFactory);
             ContentBaseAddress = contentBaseAddress;
             GalleryBaseAddress = galleryBaseAddress;
 
@@ -145,8 +148,9 @@ protected override async Task ProcessGraphsAsync(
                 var legacyTask = RegistrationMaker.ProcessAsync(
                     registrationKey: new RegistrationKey(sortedGraphs.Key),
                     newItems: sortedGraphs.Value,
-                    shouldInclude: _shouldIncludeSemVer2,
+                    shouldInclude: _shouldIncludeSemVer2ForLegacyStorageFactory,
                     storageFactory: _legacyStorageFactory,
+                    postProcessGraph: _postProcessGraphForLegacyStorageFactory,
                     contentBaseAddress: ContentBaseAddress,
                     galleryBaseAddress: GalleryBaseAddress,
                     partitionSize: PartitionSize,
@@ -174,7 +178,7 @@ protected override async Task ProcessGraphsAsync(
             }
         }
 
-        public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPackage(StorageFactory semVer2StorageFactory)
+        public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPackageForLegacyStorageFactory(StorageFactory semVer2StorageFactory)
         {
             // If SemVer 2.0.0 storage is disabled, put SemVer 2.0.0 registration in the legacy storage factory. In no
             // case should a package be completely ignored. That is, if a package is SemVer 2.0.0 but SemVer 2.0.0
@@ -187,6 +191,30 @@ public static ShouldIncludeRegistrationPackage GetShouldIncludeRegistrationPacka
             return (k, u, g) => !NuGetVersionUtility.IsGraphSemVer2(k.Version, u, g);
         }
 
+        public static RegistrationMakerCatalogItem.PostProcessGraph GetPostProcessGraphForLegacyStorageFactory(StorageFactory semVer2StorageFactory)
+        {
+            // If SemVer 2.0.0 storage is disabled, put deprecation metadata in the legacy storage.
+            // A package's deprecation metadata should never be completely ignored.
+            // If a package contains deprecation metadata but SemVer 2.0.0 storage is not enabled,
+            // our only choice is to put deprecation metadata in the legacy storage.
+            if (semVer2StorageFactory == null)
+            {
+                return g => g;
+            }
+
+            return FilterOutDeprecationInformation;
+        }
+
+        public static RegistrationMakerCatalogItem.PostProcessGraph FilterOutDeprecationInformation = g =>
+        {
+            var deprecationTriples = g
+                .GetTriplesWithPredicate(g.CreateUriNode(Schema.Predicates.Deprecation))
+                .ToList();
+
+            g.Retract(deprecationTriples);
+            return g;
+        };
+
         private async Task ProcessBatchAsync(
             CollectorHttpClient client,
             JToken context,

src/Catalog/Registration/RegistrationMaker.cs

@@ -29,6 +29,7 @@ await ProcessAsync(
                 newItems,
                 (k, u, g) => true,
                 storageFactory,
+                g => g,
                 contentBaseAddress,
                 galleryBaseAddress,
                 partitionSize,
@@ -42,6 +43,7 @@ public static async Task ProcessAsync(
             IReadOnlyDictionary<string, IGraph> newItems,
             ShouldIncludeRegistrationPackage shouldInclude,
             StorageFactory storageFactory,
+            RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph,
             Uri contentBaseAddress,
             Uri galleryBaseAddress,
             int partitionSize,
@@ -51,7 +53,7 @@ public static async Task ProcessAsync(
         {
             Trace.TraceInformation("RegistrationMaker.Process: registrationKey = {0} newItems: {1}", registrationKey, newItems.Count);
 
-            IRegistrationPersistence registration = new RegistrationPersistence(storageFactory, registrationKey, partitionSize, packageCountThreshold, contentBaseAddress, galleryBaseAddress);
+            IRegistrationPersistence registration = new RegistrationPersistence(storageFactory, postProcessGraph, registrationKey, partitionSize, packageCountThreshold, contentBaseAddress, galleryBaseAddress);
 
             IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> existing = await registration.Load(cancellationToken);
 

src/Catalog/Registration/RegistrationMakerCatalogItem.cs

@@ -15,6 +15,8 @@ namespace NuGet.Services.Metadata.Catalog.Registration
 {
     public class RegistrationMakerCatalogItem : CatalogItem
     {
+        public delegate IGraph PostProcessGraph(IGraph graph);
+
         private readonly Uri _catalogUri;
         private readonly IGraph _catalogItem;
         private Uri _itemAddress;
@@ -25,17 +27,26 @@ public class RegistrationMakerCatalogItem : CatalogItem
         private Uri _registrationAddress;
         private DateTime _publishedDate;
         private bool _listed;
+        private readonly PostProcessGraph _postProcessGraph;
 
         // This should be set before class is instantiated
         public static IPackagePathProvider PackagePathProvider = null;
 
-        public RegistrationMakerCatalogItem(Uri catalogUri, IGraph catalogItem, Uri registrationBaseAddress, bool isExistingItem, Uri packageContentBaseAddress = null, Uri galleryBaseAddress = null)
+        public RegistrationMakerCatalogItem(
+            Uri catalogUri, 
+            IGraph catalogItem, 
+            Uri registrationBaseAddress, 
+            bool isExistingItem, 
+            PostProcessGraph postProcessGraph, 
+            Uri packageContentBaseAddress = null, 
+            Uri galleryBaseAddress = null)
         {
             _catalogUri = catalogUri;
             _catalogItem = catalogItem;
             _packageContentBaseAddress = packageContentBaseAddress;
             _galleryBaseAddress = galleryBaseAddress;
             _registrationBaseAddress = registrationBaseAddress;
+            _postProcessGraph = postProcessGraph;
 
             IsExistingItem = isExistingItem;
         }
@@ -108,9 +119,7 @@ private DateTime GetPublishedDate()
 
                 if (pubTriple != null)
                 {
-                    ILiteralNode node = pubTriple.Object as ILiteralNode;
-
-                    if (node != null)
+                    if (pubTriple.Object is ILiteralNode node)
                     {
                         _publishedDate = DateTime.Parse(node.Value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
                     }
@@ -194,8 +203,10 @@ public override IGraph CreatePageContent(CatalogContext context)
                 {
                     store.Add(_catalogItem, true);
 
-                    SparqlParameterizedString sparql = new SparqlParameterizedString();
-                    sparql.CommandText = Utils.GetResource("sparql.ConstructRegistrationPageContentGraph.rq");
+                    SparqlParameterizedString sparql = new SparqlParameterizedString
+                    {
+                        CommandText = Utils.GetResource("sparql.ConstructRegistrationPageContentGraph.rq")
+                    };
 
                     sparql.SetUri("package", GetItemAddress());
                     sparql.SetUri("catalogEntry", _catalogUri);
@@ -207,7 +218,7 @@ public override IGraph CreatePageContent(CatalogContext context)
                     content = SparqlHelpers.Construct(store, sparql.ToString());
                 }
 
-                return content;
+                return _postProcessGraph(content);
             }
             catch (Exception e)
             {

src/Catalog/Registration/RegistrationPersistence.cs

@@ -21,13 +21,22 @@ public class RegistrationPersistence : IRegistrationPersistence
         private readonly int _packageCountThreshold;
         private readonly int _partitionSize;
         private readonly RecordingStorage _storage;
+        private readonly RegistrationMakerCatalogItem.PostProcessGraph _postProcessGraph;
         private readonly Uri _registrationBaseAddress;
         private readonly Uri _contentBaseAddress;
         private readonly Uri _galleryBaseAddress;
 
-        public RegistrationPersistence(StorageFactory storageFactory, RegistrationKey registrationKey, int partitionSize, int packageCountThreshold, Uri contentBaseAddress, Uri galleryBaseAddress)
+        public RegistrationPersistence(
+            StorageFactory storageFactory, 
+            RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph, 
+            RegistrationKey registrationKey, 
+            int partitionSize, 
+            int packageCountThreshold, 
+            Uri contentBaseAddress, 
+            Uri galleryBaseAddress)
         {
             _storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString()));
+            _postProcessGraph = postProcessGraph;
             _registrationUri = _storage.ResolveUri("index.json");
             _packageCountThreshold = packageCountThreshold;
             _partitionSize = partitionSize;
@@ -43,7 +52,7 @@ public Task<IDictionary<RegistrationEntryKey, RegistrationCatalogEntry>> Load(Ca
 
         public async Task Save(IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> registration, CancellationToken cancellationToken)
         {
-            await Save(_storage, _registrationBaseAddress, registration, _partitionSize, _packageCountThreshold, _contentBaseAddress, _galleryBaseAddress, cancellationToken);
+            await Save(_storage, _postProcessGraph, _registrationBaseAddress, registration, _partitionSize, _packageCountThreshold, _contentBaseAddress, _galleryBaseAddress, cancellationToken);
 
             await Cleanup(_storage, cancellationToken);
         }
@@ -100,8 +109,11 @@ private static void AddExistingItem(IDictionary<RegistrationEntryKey, Registrati
         {
             Trace.TraceInformation("RegistrationPersistence.AddExistingItem: catalogEntry = {0}", catalogEntry);
 
-            SparqlParameterizedString sparql = new SparqlParameterizedString();
-            sparql.CommandText = Utils.GetResource("sparql.ConstructCatalogEntryGraph.rq");
+            SparqlParameterizedString sparql = new SparqlParameterizedString
+            {
+                CommandText = Utils.GetResource("sparql.ConstructCatalogEntryGraph.rq")
+            };
+
             sparql.SetUri("catalogEntry", catalogEntry);
 
             IGraph graph = SparqlHelpers.Construct(store, sparql.ToString());
@@ -157,7 +169,16 @@ private static async Task<IGraph> LoadCatalogPage(IStorage storage, Uri pageUri,
         }
 
         //  Save implementation
-        private static async Task Save(IStorage storage, Uri registrationBaseAddress, IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> registration, int partitionSize, int packageCountThreshold, Uri contentBaseAddress, Uri galleryBaseAddress, CancellationToken cancellationToken)
+        private static async Task Save(
+            IStorage storage, 
+            RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph, 
+            Uri registrationBaseAddress, 
+            IDictionary<RegistrationEntryKey, RegistrationCatalogEntry> registration, 
+            int partitionSize, 
+            int packageCountThreshold, 
+            Uri contentBaseAddress, 
+            Uri galleryBaseAddress, 
+            CancellationToken cancellationToken)
         {
             Trace.TraceInformation("RegistrationPersistence.Save");
 
@@ -170,23 +191,30 @@ private static async Task Save(IStorage storage, Uri registrationBaseAddress, ID
 
             if (items.Count < packageCountThreshold)
             {
-                await SaveSmallRegistration(storage, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
+                await SaveSmallRegistration(storage, preprocessGraph, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
             }
             else
             {
-                await SaveLargeRegistration(storage, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
+                await SaveLargeRegistration(storage, preprocessGraph, registrationBaseAddress, items, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
             }
         }
 
-        private static async Task SaveSmallRegistration(IStorage storage, Uri registrationBaseAddress, IList<RegistrationCatalogEntry> items, int partitionSize, Uri contentBaseAddress, Uri galleryBaseAddress, CancellationToken cancellationToken)
+        private static async Task SaveSmallRegistration(
+            IStorage storage, 
+            RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph, 
+            Uri registrationBaseAddress, 
+            IList<RegistrationCatalogEntry> items, 
+            int partitionSize, Uri contentBaseAddress, 
+            Uri galleryBaseAddress, 
+            CancellationToken cancellationToken)
         {
             Trace.TraceInformation("RegistrationPersistence.SaveSmallRegistration");
 
             SingleGraphPersistence graphPersistence = new SingleGraphPersistence(storage);
 
             //await graphPersistence.Initialize();
 
-            await SaveRegistration(storage, registrationBaseAddress, items, null, graphPersistence, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
+            await SaveRegistration(storage, preprocessGraph, registrationBaseAddress, items, null, graphPersistence, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
 
             // now the commit has happened the graphPersistence.Graph should contain all the data
 
@@ -195,24 +223,42 @@ private static async Task SaveSmallRegistration(IStorage storage, Uri registrati
             await storage.SaveAsync(graphPersistence.ResourceUri, content, cancellationToken);
         }
 
-        private static async Task SaveLargeRegistration(IStorage storage, Uri registrationBaseAddress, IList<RegistrationCatalogEntry> items, int partitionSize, Uri contentBaseAddress, Uri galleryBaseAddress,  CancellationToken cancellationToken)
+        private static async Task SaveLargeRegistration(
+            IStorage storage, 
+            RegistrationMakerCatalogItem.PostProcessGraph preprocessGraph, 
+            Uri registrationBaseAddress, 
+            IList<RegistrationCatalogEntry> items, 
+            int partitionSize, 
+            Uri contentBaseAddress, 
+            Uri galleryBaseAddress,  
+            CancellationToken cancellationToken)
         {
             Trace.TraceInformation("RegistrationPersistence.SaveLargeRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count);
 
             IList<Uri> cleanUpList = new List<Uri>();
 
-            await SaveRegistration(storage, registrationBaseAddress, items, cleanUpList, null, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
+            await SaveRegistration(storage, preprocessGraph, registrationBaseAddress, items, cleanUpList, null, partitionSize, contentBaseAddress, galleryBaseAddress, cancellationToken);
         }
 
-        private static async Task SaveRegistration(IStorage storage, Uri registrationBaseAddress, IList<RegistrationCatalogEntry> items, IList<Uri> cleanUpList, SingleGraphPersistence graphPersistence, int partitionSize, Uri contentBaseAddress, Uri galleryBaseAddress, CancellationToken cancellationToken)
+        private static async Task SaveRegistration(
+            IStorage storage, 
+            RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph, 
+            Uri registrationBaseAddress, 
+            IList<RegistrationCatalogEntry> items, 
+            IList<Uri> cleanUpList, 
+            SingleGraphPersistence graphPersistence, 
+            int partitionSize, 
+            Uri contentBaseAddress, 
+            Uri galleryBaseAddress, 
+            CancellationToken cancellationToken)
         {
             Trace.TraceInformation("RegistrationPersistence.SaveRegistration: registrationBaseAddress = {0} items: {1}", registrationBaseAddress, items.Count);
 
             using (RegistrationMakerCatalogWriter writer = new RegistrationMakerCatalogWriter(storage, partitionSize, cleanUpList, graphPersistence))
             {
                 foreach (var item in items)
                 {
-                    writer.Add(new RegistrationMakerCatalogItem(new Uri(item.ResourceUri), item.Graph, registrationBaseAddress, item.IsExistingItem, contentBaseAddress, galleryBaseAddress));
+                    writer.Add(new RegistrationMakerCatalogItem(new Uri(item.ResourceUri), item.Graph, registrationBaseAddress, item.IsExistingItem, postProcessGraph, contentBaseAddress, galleryBaseAddress));
                 }
                 await writer.Commit(DateTime.UtcNow, null, cancellationToken);
             }