Merge pull request #392 from NuGet/dev

[ReleasePrep][2018.11.06]RI of dev into master

build.ps1

@@ -9,7 +9,7 @@ param (
     [string]$SemanticVersion = '1.0.0-zlocal',
     [string]$Branch,
     [string]$CommitSHA,
-    [string]$BuildBranch = '795fed66b8bae2d248237ee5ec82e688e7174a42'
+    [string]$BuildBranch = '5fd8377a9abf3ff411918dbb973948a6677432db'
 )
 
 # For TeamCity - If any issue occurs, this script fails the build. - By default, TeamCity returns an exit code of 0 for all powershell scripts, even if they fail
@@ -105,7 +105,8 @@ Invoke-BuildStep 'Creating artifacts' {
         }
 
         $nuspecPackages = `
-            "src\Ng\Ng.nuspec"
+            "src\Ng\Ng.nuspec", `
+            "src\Ng\Ng.Operations.nuspec"
 
         $nuspecPackages | ForEach-Object {
             New-Package (Join-Path $PSScriptRoot $_) -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch

sign.thirdparty.targets

@@ -4,7 +4,9 @@
     </ItemGroup>
     <Target Name="EnumerateThirdPartyBinariesToSign" AfterTargets="AfterBuild" Condition="'$(SignType)' != 'none'">
         <ItemGroup>
+            <ThirdPartyBinaries Include="AnglicanGeek.MarkdownMailer.dll" />
             <ThirdPartyBinaries Include="dotNetRDF.dll" />
+            <ThirdPartyBinaries Include="Elmah.dll" />
             <ThirdPartyBinaries Include="HtmlAgilityPack.dll" />
             <ThirdPartyBinaries Include="ICSharpCode.SharpZipLib.dll" />
             <ThirdPartyBinaries Include="json-ld.net.StrongName.dll" />
@@ -20,6 +22,7 @@
             <ThirdPartyBinaries Include="Lucene.Net.Contrib.SpellChecker.dll" />
             <ThirdPartyBinaries Include="Lucene.Net.dll" />
             <ThirdPartyBinaries Include="Lucene.Net.Store.Azure.dll" />
+            <ThirdPartyBinaries Include="MarkdownSharp.dll" />
             <ThirdPartyBinaries Include="Newtonsoft.Json.dll" />
             <ThirdPartyBinaries Include="Serilog.dll" />
             <ThirdPartyBinaries Include="Serilog.Enrichers.Environment.dll" />

src/Catalog/CatalogIndexEntry.cs

@@ -11,134 +11,111 @@
 
 namespace NuGet.Services.Metadata.Catalog
 {
-    public class CatalogIndexEntry : IComparable<CatalogIndexEntry>
+    public sealed class CatalogIndexEntry : IComparable<CatalogIndexEntry>
     {
-        private string _type;
-
-        public CatalogIndexEntry(JToken item)
-            : this(
-                new Uri(item["@id"].ToString()),
-                item["@type"].ToString(),
-                item["commitId"].ToString(),
-                DateTime.Parse(item["commitTimeStamp"].ToString()),
-                item["nuget:id"].ToString(),
-                NuGetVersion.Parse(item["nuget:version"].ToString()))
+        private static readonly CatalogIndexEntryDateComparer _commitTimeStampComparer = new CatalogIndexEntryDateComparer();
+
+        [JsonConstructor]
+        private CatalogIndexEntry()
         {
+            Types = Enumerable.Empty<string>();
         }
 
         public CatalogIndexEntry(Uri uri, string type, string commitId, DateTime commitTs, string id, NuGetVersion version)
         {
-            Uri = uri;
-            _type = type;
-            CommitId = commitId;
-            CommitTimeStamp = commitTs;
-            Id = id;
-            Version = version;
-        }
+            Uri = uri ?? throw new ArgumentNullException(nameof(uri));
 
-        [JsonConstructor]
-        public CatalogIndexEntry()
-        {
-        }
+            if (string.IsNullOrWhiteSpace(type))
+            {
+                throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(type));
+            }
 
-        [JsonProperty("@id")]
-        public Uri Uri { get; private set; }
+            Types = new[] { type };
+            IsDelete = type == "nuget:PackageDelete";
 
-        [JsonProperty("@type")]
-        [JsonConverter(typeof(CatalogTypeConverter))]
-        public IEnumerable<string> Types
-        {
-            get
+            if (string.IsNullOrWhiteSpace(commitId))
             {
-                return new string[] { _type };
+                throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(commitId));
             }
-            set
+
+            CommitId = commitId;
+            CommitTimeStamp = commitTs;
+
+            if (string.IsNullOrWhiteSpace(id))
             {
-                _type = value.Single();
+                throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id));
             }
+
+            Id = id;
+            Version = version ?? throw new ArgumentNullException(nameof(version));
         }
 
-        [JsonProperty("nuget:id")]
-        public string Id { get; private set; }
+        [JsonProperty("@id")]
+        [JsonRequired]
+        public Uri Uri { get; private set; }
 
-        [JsonProperty("nuget:version")]
-        public NuGetVersion Version { get; private set; }
+        [JsonProperty("@type")]
+        [JsonRequired]
+        [JsonConverter(typeof(CatalogTypeConverter))]
+        public IEnumerable<string> Types { get; private set; }
 
         [JsonProperty("commitId")]
+        [JsonRequired]
         public string CommitId { get; private set; }
 
         [JsonProperty("commitTimeStamp")]
+        [JsonRequired]
         public DateTime CommitTimeStamp { get; private set; }
 
-        public int CompareTo(CatalogIndexEntry other)
-        {
-            return CommitTSComparer.Compare(this, other);
-        }
+        [JsonProperty("nuget:id")]
+        [JsonRequired]
+        public string Id { get; private set; }
 
-        public bool IsDelete()
-        {
-            return _type == "nuget:PackageDelete";
-        }
+        [JsonProperty("nuget:version")]
+        [JsonRequired]
+        public NuGetVersion Version { get; private set; }
 
-        // common comparers for sorting and creating sets from these entries
-        public static CatalogIndexEntryIdComparer IdComparer
-        {
-            get
-            {
-                return new CatalogIndexEntryIdComparer();
-            }
-        }
+        [JsonIgnore]
+        public bool IsDelete { get; }
 
-        public static CatalogIndexEntryIdVersionComparer IdVersionComparer
+        public int CompareTo(CatalogIndexEntry other)
         {
-            get
+            if (other == null)
             {
-                return new CatalogIndexEntryIdVersionComparer();
+                throw new ArgumentNullException(nameof(other));
             }
+
+            return _commitTimeStampComparer.Compare(this, other);
         }
 
-        public static CatalogIndexEntryDateComparer CommitTSComparer
+        public static CatalogIndexEntry Create(JToken token)
         {
-            get
+            if (token == null)
             {
-                return new CatalogIndexEntryDateComparer();
+                throw new ArgumentNullException(nameof(token));
             }
-        }
-    }
-
-    public class CatalogIndexEntryDateComparer : IComparer<CatalogIndexEntry>
-    {
-        public int Compare(CatalogIndexEntry x, CatalogIndexEntry y)
-        {
-            return x.CommitTimeStamp.CompareTo(y.CommitTimeStamp);
-        }
-    }
 
+            var uri = new Uri(token["@id"].ToString());
+            var type = token["@type"].ToString();
+            var commitId = token["commitId"].ToString();
+            var commitTimeStamp = DateTime.ParseExact(
+                token["commitTimeStamp"].ToString(),
+                "yyyy-MM-ddTHH:mm:ss.FFFFFFFZ",
+                DateTimeFormatInfo.CurrentInfo,
+                DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
 
-    public class CatalogIndexEntryIdComparer : IEqualityComparer<CatalogIndexEntry>
-    {
-        public bool Equals(CatalogIndexEntry x, CatalogIndexEntry y)
-        {
-            return StringComparer.OrdinalIgnoreCase.Equals(x.Id, y.Id);
-        }
+            var packageId = token["nuget:id"].ToString();
+            var packageVersion = NuGetVersion.Parse(token["nuget:version"].ToString());
 
-        public int GetHashCode(CatalogIndexEntry obj)
-        {
-            return obj.Id.ToLowerInvariant().GetHashCode();
+            return new CatalogIndexEntry(uri, type, commitId, commitTimeStamp, packageId, packageVersion);
         }
     }
 
-    public class CatalogIndexEntryIdVersionComparer : IEqualityComparer<CatalogIndexEntry>
+    public class CatalogIndexEntryDateComparer : IComparer<CatalogIndexEntry>
     {
-        const string PackageIdFormat = "{0}.{1}";
-        public bool Equals(CatalogIndexEntry x, CatalogIndexEntry y)
-        {
-            return x.Id.Equals(y.Id, StringComparison.OrdinalIgnoreCase) && x.Version.Equals(y.Version);
-        }
-
-        public int GetHashCode(CatalogIndexEntry obj)
+        public int Compare(CatalogIndexEntry x, CatalogIndexEntry y)
         {
-            return String.Format(CultureInfo.InvariantCulture, PackageIdFormat, obj.Id.ToLowerInvariant(), obj.Version.ToNormalizedString().ToLowerInvariant()).GetHashCode();
+            return x.CommitTimeStamp.CompareTo(y.CommitTimeStamp);
         }
     }
 }

src/Catalog/CatalogItem.cs

@@ -1,9 +1,8 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-using NuGet.Services.Metadata.Catalog.Persistence;
+
 using System;
-using System.Collections.Generic;
-using System.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
 using VDS.RDF;
 
 namespace NuGet.Services.Metadata.Catalog

src/Catalog/FixPackageHashHandler.cs

@@ -0,0 +1,74 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Net.Http;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// Adds the Content MD5 property to package blobs that are missing it.
+    /// </summary>
+    public class FixPackageHashHandler : IPackagesContainerHandler
+    {
+        private readonly HttpClient _httpClient;
+        private readonly ITelemetryService _telemetryService;
+        private readonly ILogger<FixPackageHashHandler> _logger;
+
+        public FixPackageHashHandler(
+            HttpClient httpClient,
+            ITelemetryService telemetryService,
+            ILogger<FixPackageHashHandler> logger)
+        {
+            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+            _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob)
+        {
+            await blob.FetchAttributesAsync(CancellationToken.None);
+
+            // Skip the package if it has a Content MD5 hash
+            if (blob.ContentMD5 != null)
+            {
+                _telemetryService.TrackPackageAlreadyHasHash(packageEntry.Id, packageEntry.Version);
+                return;
+            }
+
+            // Download the blob and calculate its hash. We use HttpClient to download blobs as Azure Blob Sotrage SDK
+            // occassionally hangs. See: https://github.com/Azure/azure-storage-net/issues/470
+            string hash;
+            using (var hashAlgorithm = MD5.Create())
+            using (var packageStream = await _httpClient.GetStreamAsync(blob.Uri))
+            {
+                var hashBytes = hashAlgorithm.ComputeHash(packageStream);
+
+                hash = Convert.ToBase64String(hashBytes);
+            }
+
+            blob.ContentMD5 = hash;
+
+            var condition = AccessCondition.GenerateIfMatchCondition(blob.ETag);
+            await blob.SetPropertiesAsync(
+                condition,
+                options: null,
+                operationContext: null);
+
+            _telemetryService.TrackPackageHashFixed(packageEntry.Id, packageEntry.Version);
+
+            _logger.LogWarning(
+                "Updated package {PackageId} {PackageVersion}, set hash to '{Hash}' using ETag {ETag}",
+                packageEntry.Id,
+                packageEntry.Version,
+                hash,
+                blob.ETag);
+        }
+    }
+}

src/Catalog/ICatalogIndexProcessor.cs

@@ -0,0 +1,21 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// A processor that runs on entries found on a catalog page.
+    /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-page
+    /// </summary>
+    public interface ICatalogIndexProcessor
+    {
+        /// <summary>
+        /// Process a single entry from a catalog page.
+        /// </summary>
+        /// <param name="catalogEntry">The catalog index entry that should be processed.</param>
+        /// <returns>A task that completes once the entry has been processed.</returns>
+        Task ProcessCatalogIndexEntryAsync(CatalogIndexEntry catalogEntry);
+    }
+}

src/Catalog/IPackagesContainerHandler.cs

@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog
+{
+    /// <summary>
+    /// A handler that is run on packages in the packages container.
+    /// </summary>
+    public interface IPackagesContainerHandler
+    {
+        /// <summary>
+        /// Handle a package in the packages container.
+        /// </summary>
+        /// <param name="packageEntry">The package's catalog index entry.</param>
+        /// <param name="blob">The package's blob in the packages container.</param>
+        /// <returns>A task that completes once the package has been handled.</returns>
+        Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob);
+    }
+}

src/Catalog/LogEvents.cs

@@ -2,11 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace NuGet.Services.Metadata.Catalog
 {