[ReleasePrep][2018.11.26]RI of dev into master

NuGet.Services.Metadata.sln

@@ -40,6 +40,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CatalogMetadataTests", "tes
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "V3PerPackage", "src\V3PerPackage\V3PerPackage.csproj", "{E76E73FA-4462-4F07-94C0-8B9CC413F696}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Protocol.Catalog", "src\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj", "{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Protocol.Catalog.Tests", "tests\NuGet.Protocol.Catalog.Tests\NuGet.Protocol.Catalog.Tests.csproj", "{1F3BC053-796C-4A35-88F4-955A0F142197}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -190,6 +194,30 @@ Global
 		{E76E73FA-4462-4F07-94C0-8B9CC413F696}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{E76E73FA-4462-4F07-94C0-8B9CC413F696}.Release|x64.ActiveCfg = Release|Any CPU
 		{E76E73FA-4462-4F07-94C0-8B9CC413F696}.Release|x64.Build.0 = Release|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|x64.Build.0 = Debug|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|x64.ActiveCfg = Release|Any CPU
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|x64.Build.0 = Release|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|x64.Build.0 = Debug|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Release|x64.ActiveCfg = Release|Any CPU
+		{1F3BC053-796C-4A35-88F4-955A0F142197}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -207,6 +235,8 @@ Global
 		{1745A383-D0BE-484B-81EB-27B20F6AC6C5} = {5DE01C58-D5F7-482F-8256-A8333064384C}
 		{34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4} = {F1C83FD9-A498-483E-ADFA-B55D82A14965}
 		{E76E73FA-4462-4F07-94C0-8B9CC413F696} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}
+		{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} = {5DE01C58-D5F7-482F-8256-A8333064384C}
+		{1F3BC053-796C-4A35-88F4-955A0F142197} = {F1C83FD9-A498-483E-ADFA-B55D82A14965}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D3AB83E9-02B4-4FFA-A2D0-637F0B97E626}

build.ps1

@@ -9,7 +9,7 @@ param (
     [string]$SemanticVersion = '1.0.0-zlocal',
     [string]$Branch,
     [string]$CommitSHA,
-    [string]$BuildBranch = '5fd8377a9abf3ff411918dbb973948a6677432db'
+    [string]$BuildBranch = 'b5f9d1c89da96c462935e2195ceb00e69287b93e'
 )
 
 # For TeamCity - If any issue occurs, this script fails the build. - By default, TeamCity returns an exit code of 0 for all powershell scripts, even if they fail
@@ -79,7 +79,8 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' {
             "src\Catalog\Properties\AssemblyInfo.g.cs", `
             "src\NuGet.ApplicationInsights.Owin\Properties\AssemblyInfo.g.cs", `
             "src\Ng\Properties\AssemblyInfo.g.cs", `
-            "src\NuGet.Services.Metadata.Catalog.Monitoring\Properties\AssemblyInfo.g.cs"
+            "src\NuGet.Services.Metadata.Catalog.Monitoring\Properties\AssemblyInfo.g.cs", `
+            "src\NuGet.Protocol.Catalog\Properties\AssemblyInfo.g.cs"
 
         Foreach ($assemblyInfo in $assemblyInfos) {
             Set-VersionInfo -Path (Join-Path $PSScriptRoot $assemblyInfo) -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA

src/Catalog/BatchProcessingException.cs

@@ -8,7 +8,7 @@ namespace NuGet.Services.Metadata.Catalog
     public sealed class BatchProcessingException : Exception
     {
         public BatchProcessingException(Exception inner)
-            : base(Strings.BatchProcessingFailure)
+            : base(Strings.BatchProcessingFailure, inner ?? throw new ArgumentNullException(nameof(inner)))
         {
         }
     }

src/Catalog/CatalogCommit.cs

@@ -0,0 +1,51 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Globalization;
+using Newtonsoft.Json.Linq;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// Represents a single catalog commit.
+    /// </summary>
+    public sealed class CatalogCommit : IComparable
+    {
+        private CatalogCommit(DateTime commitTimeStamp, Uri uri)
+        {
+            CommitTimeStamp = commitTimeStamp;
+            Uri = uri;
+        }
+
+        public DateTime CommitTimeStamp { get; }
+        public Uri Uri { get; }
+
+        public int CompareTo(object obj)
+        {
+            var other = obj as CatalogCommit;
+
+            if (ReferenceEquals(other, null))
+            {
+                throw new ArgumentException(
+                    string.Format(CultureInfo.InvariantCulture, Strings.ArgumentMustBeInstanceOfType, nameof(CatalogCommit)),
+                    nameof(obj));
+            }
+
+            return CommitTimeStamp.CompareTo(other.CommitTimeStamp);
+        }
+
+        public static CatalogCommit Create(JObject commit)
+        {
+            if (commit == null)
+            {
+                throw new ArgumentNullException(nameof(commit));
+            }
+
+            var commitTimeStamp = Utils.Deserialize<DateTime>(commit, "commitTimeStamp");
+            var uri = Utils.Deserialize<Uri>(commit, "@id");
+
+            return new CatalogCommit(commitTimeStamp, uri);
+        }
+    }
+}

src/Catalog/CatalogCommitItem.cs

@@ -0,0 +1,108 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using Newtonsoft.Json.Linq;
+using NuGet.Packaging.Core;
+using NuGet.Versioning;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// Represents a single item in a catalog commit.
+    /// </summary>
+    public sealed class CatalogCommitItem : IComparable
+    {
+        private const string _typeKeyword = "@type";
+
+        private CatalogCommitItem(
+            Uri uri,
+            string commitId,
+            DateTime commitTimeStamp,
+            IReadOnlyList<string> types,
+            IReadOnlyList<Uri> typeUris,
+            PackageIdentity packageIdentity)
+        {
+            Uri = uri;
+            CommitId = commitId;
+            CommitTimeStamp = commitTimeStamp;
+            PackageIdentity = packageIdentity;
+            Types = types;
+            TypeUris = typeUris;
+        }
+
+        public Uri Uri { get; }
+        public DateTime CommitTimeStamp { get; }
+        public string CommitId { get; }
+        public PackageIdentity PackageIdentity { get; }
+        public IReadOnlyList<string> Types { get; }
+        public IReadOnlyList<Uri> TypeUris { get; }
+
+        public int CompareTo(object obj)
+        {
+            var other = obj as CatalogCommitItem;
+
+            if (ReferenceEquals(other, null))
+            {
+                throw new ArgumentException(
+                    string.Format(CultureInfo.InvariantCulture, Strings.ArgumentMustBeInstanceOfType, nameof(CatalogCommitItem)),
+                    nameof(obj));
+            }
+
+            return CommitTimeStamp.CompareTo(other.CommitTimeStamp);
+        }
+
+        public static CatalogCommitItem Create(JObject context, JObject commitItem)
+        {
+            if (context == null)
+            {
+                throw new ArgumentNullException(nameof(context));
+            }
+
+            if (commitItem == null)
+            {
+                throw new ArgumentNullException(nameof(commitItem));
+            }
+
+            var commitTimeStamp = Utils.Deserialize<DateTime>(commitItem, "commitTimeStamp");
+            var commitId = Utils.Deserialize<string>(commitItem, "commitId");
+            var idUri = Utils.Deserialize<Uri>(commitItem, "@id");
+            var packageId = Utils.Deserialize<string>(commitItem, "nuget:id");
+            var packageVersion = Utils.Deserialize<string>(commitItem, "nuget:version");
+            var packageIdentity = new PackageIdentity(packageId, new NuGetVersion(packageVersion));
+            var types = GetTypes(commitItem).ToArray();
+
+            if (!types.Any())
+            {
+                throw new ArgumentException(
+                    string.Format(CultureInfo.InvariantCulture, Strings.NonEmptyPropertyValueRequired, _typeKeyword),
+                    nameof(commitItem));
+            }
+
+            var typeUris = types.Select(type => Utils.Expand(context, type)).ToArray();
+
+            return new CatalogCommitItem(idUri, commitId, commitTimeStamp, types, typeUris, packageIdentity);
+        }
+
+        private static IEnumerable<string> GetTypes(JObject commitItem)
+        {
+            if (commitItem.TryGetValue(_typeKeyword, out var value))
+            {
+                if (value is JArray)
+                {
+                    foreach (JToken typeToken in ((JArray)value).Values())
+                    {
+                        yield return typeToken.ToString();
+                    }
+                }
+                else
+                {
+                    yield return value.ToString();
+                }
+            }
+        }
+    }
+}

src/Catalog/CatalogCommitItemBatch.cs

@@ -0,0 +1,44 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// Represents a group of <see cref="CatalogCommitItem" />.
+    /// Items may span multiple commits but are grouped on common criteria (e.g.:  lower-cased package ID).
+    /// </summary>
+    public sealed class CatalogCommitItemBatch
+    {
+        /// <summary>
+        /// Initializes a <see cref="CatalogCommitItemBatch" /> instance.
+        /// </summary>
+        /// <param name="items">An enumerable of <see cref="CatalogCommitItem" />.  Items may span multiple commits.</param>
+        /// <param name="key">A unique key for all items in a batch.  This is used for parallelization and may be
+        /// <c>null</c> if parallelization is not used.</param>
+        /// <exception cref="ArgumentException">Thrown if <paramref name="items" /> is either <c>null</c> or empty.</exception>
+        public CatalogCommitItemBatch(IEnumerable<CatalogCommitItem> items, string key = null)
+        {
+            if (items == null || !items.Any())
+            {
+                throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(items));
+            }
+
+            var list = items.ToList();
+
+            CommitTimeStamp = list.Min(item => item.CommitTimeStamp);
+            Key = key;
+
+            list.Sort();
+
+            Items = list;
+        }
+
+        public DateTime CommitTimeStamp { get; }
+        public IReadOnlyList<CatalogCommitItem> Items { get; }
+        public string Key { get; }
+    }
+}

src/Catalog/CatalogCommitItemBatchTask.cs

@@ -0,0 +1,68 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// Represents an asynchrononous task associated with catalog changes for a specific
+    /// <see cref="CatalogCommitItemBatch" /> and potentially spanning multiple commits.
+    /// </summary>
+    public sealed class CatalogCommitItemBatchTask : IEquatable<CatalogCommitItemBatchTask>
+    {
+        /// <summary>
+        /// Initializes a <see cref="CatalogCommitItemBatchTask" /> instance.
+        /// </summary>
+        /// <param name="batch">A <see cref="CatalogCommitItemBatch" />.</param>
+        /// <param name="task">A <see cref="System.Threading.Tasks.Task" /> tracking completion of
+        /// <paramref name="batch" /> processing.</param>
+        /// <exception cref="ArgumentNullException">Thrown if <paramref name="batch" /> is <c>null</c>.</exception>
+        /// <exception cref="ArgumentException">Thrown if <see cref="CatalogCommitItemBatch.Key" /> is <c>null</c>.</exception>
+        /// <exception cref="ArgumentNullException">Thrown if <paramref name="task" /> is <c>null</c>.</exception>
+        public CatalogCommitItemBatchTask(CatalogCommitItemBatch batch, Task task)
+        {
+            if (batch == null)
+            {
+                throw new ArgumentNullException(nameof(batch));
+            }
+
+            if (batch.Key == null)
+            {
+                throw new ArgumentException(Strings.ArgumentMustNotBeNull, $"{nameof(batch)}.{nameof(batch.Key)}");
+            }
+
+            if (task == null)
+            {
+                throw new ArgumentNullException(nameof(task));
+            }
+
+            Batch = batch;
+            Task = task;
+        }
+
+        public CatalogCommitItemBatch Batch { get; }
+        public Task Task { get; }
+
+        public override int GetHashCode()
+        {
+            return Batch.Key.GetHashCode();
+        }
+
+        public override bool Equals(object obj)
+        {
+            return Equals(obj as CatalogCommitItemBatchTask);
+        }
+
+        public bool Equals(CatalogCommitItemBatchTask other)
+        {
+            if (ReferenceEquals(other, null))
+            {
+                return false;
+            }
+
+            return string.Equals(Batch.Key, other.Batch.Key);
+        }
+    }
+}