Merge pull request #698 from NuGet/dev

Merge branch 'dev' into master

nuget.config

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
+    <clear />
     <add key="nugetbuild" value="https://dotnet.myget.org/F/nuget-build/api/v3/index.json" />
     <add key="nuget" value="https://api.nuget.org/v3/index.json" />
   </packageSources>

src/Catalog/Icons/ExternalIconCopyResult.cs

@@ -9,24 +9,51 @@ public class ExternalIconCopyResult
     {
         public static ExternalIconCopyResult Success(Uri sourceUrl, Uri storageUrl)
         {
+            if (sourceUrl == null)
+            {
+                throw new ArgumentNullException(nameof(sourceUrl));
+            }
+
+            if (storageUrl == null)
+            {
+                throw new ArgumentNullException(nameof(storageUrl));
+            }
+
             return new ExternalIconCopyResult
             {
                 SourceUrl = sourceUrl,
                 StorageUrl = storageUrl,
+                Expiration = null,            // successes don't expire
             };
         }
 
-        public static ExternalIconCopyResult Fail(Uri sourceUrl)
+        public static ExternalIconCopyResult Fail(Uri sourceUrl, TimeSpan validityPeriod)
         {
+            if (sourceUrl == null)
+            {
+                throw new ArgumentNullException(nameof(sourceUrl));
+            }
+
+            if (validityPeriod < TimeSpan.Zero)
+            {
+                throw new ArgumentOutOfRangeException(nameof(validityPeriod), $"{nameof(validityPeriod)} cannot be negative");
+            }
+
             return new ExternalIconCopyResult
             {
                 SourceUrl = sourceUrl,
-                StorageUrl = null
+                StorageUrl = null,
+                Expiration = DateTimeOffset.UtcNow.Add(validityPeriod),
             };
         }
 
         public Uri SourceUrl { get; set; }
         public Uri StorageUrl { get; set; }
+
+        /// <summary>
+        /// Expiration time for the fail cache item.
+        /// </summary>
+        public DateTimeOffset? Expiration { get; set; }
         public bool IsCopySucceeded => StorageUrl != null;
     }
 }

src/Catalog/Icons/IconCopyResultCache.cs

@@ -23,13 +23,16 @@ public class IconCopyResultCache : IIconCopyResultCache, IIconCopyResultCachePer
         private ConcurrentDictionary<Uri, SemaphoreSlim> _uriSemaphores = null;
 
         private readonly IStorage _auxStorage;
+        private readonly TimeSpan _failCacheTime;
         private readonly ILogger<IconCopyResultCache> _logger;
 
         public IconCopyResultCache(
             IStorage auxStorage,
+            TimeSpan failCacheTime,
             ILogger<IconCopyResultCache> logger)
         {
             _auxStorage = auxStorage ?? throw new ArgumentNullException(nameof(auxStorage));
+            _failCacheTime = failCacheTime;
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
             _uriSemaphores = new ConcurrentDictionary<Uri, SemaphoreSlim>();
@@ -73,12 +76,15 @@ public ExternalIconCopyResult Get(Uri iconUrl)
                 throw new InvalidOperationException("Object was not initialized");
             }
 
-            if (_externalIconCopyResults.TryGetValue(iconUrl, out var result))
+            if (!_externalIconCopyResults.TryGetValue(iconUrl, out var result))
             {
-                return result;
+                return null;
             }
-
-            return null;
+            if (!result.IsCopySucceeded && (!result.Expiration.HasValue || result.Expiration.Value < DateTimeOffset.UtcNow))
+            {
+                return null;
+            }
+            return result;
         }
 
         public async Task<Uri> SaveExternalIcon(Uri originalIconUrl, Uri storageUrl, IStorage mainDestinationStorage, IStorage cacheStorage, CancellationToken cancellationToken)
@@ -144,7 +150,7 @@ public void SaveExternalCopyFailure(Uri iconUrl)
                 throw new InvalidOperationException("Object was not initialized");
             }
 
-            Set(iconUrl, ExternalIconCopyResult.Fail(iconUrl));
+            Set(iconUrl, ExternalIconCopyResult.Fail(iconUrl, _failCacheTime));
         }
 
         private void Set(Uri iconUrl, ExternalIconCopyResult newItem)

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -294,12 +294,6 @@
     <PackageReference Include="NuGet.Services.Sql">
       <Version>2.58.0</Version>
     </PackageReference>
-    <PackageReference Include="NuGet.Protocol">
-      <Version>5.0.0-preview1.5665</Version>
-    </PackageReference>
-    <PackageReference Include="NuGet.Packaging">
-      <Version>5.0.0-preview1.5665</Version>
-    </PackageReference>
     <PackageReference Include="NuGet.StrongName.json-ld.net">
       <Version>1.0.6</Version>
     </PackageReference>
@@ -314,9 +308,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="NuGet.Packaging.Core">
-      <Version>5.0.0-preview1.5665</Version>
-    </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
       <Version>2.58.0</Version>
     </PackageReference>

src/Ng/Jobs/Catalog2IconJob.cs

@@ -19,6 +19,7 @@ namespace Ng.Jobs
     public class Catalog2IconJob : LoopingNgJob
     {
         private const int DegreeOfParallelism = 120;
+        private const string FailCacheTime = "failCacheTime";
         private IconsCollector _collector;
         private DurableCursor _front;
 
@@ -33,6 +34,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
 
             var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
             var packageStorageBase = arguments.GetOrThrow<string>(Arguments.ContentBaseAddress);
+            var failCacheTime = arguments.GetOrDefault(FailCacheTime, TimeSpan.FromHours(1));
             var auxStorageFactory = CreateAuxStorageFactory(arguments, verbose);
             var targetStorageFactory = CreateTargetStorageFactory(arguments, verbose);
             var packageStorage = new AzureStorage(
@@ -52,7 +54,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
             var catalogClient = new CatalogClient(simpleHttpClient, LoggerFactory.CreateLogger<CatalogClient>());
             var httpResponseProvider = new HttpClientWrapper(httpClient);
             var externalIconProvider = new ExternalIconContentProvider(httpResponseProvider, LoggerFactory.CreateLogger<ExternalIconContentProvider>());
-            var iconCopyResultCache = new IconCopyResultCache(auxStorageFactory.Create(), LoggerFactory.CreateLogger<IconCopyResultCache>());
+            var iconCopyResultCache = new IconCopyResultCache(auxStorageFactory.Create(), failCacheTime, LoggerFactory.CreateLogger<IconCopyResultCache>());
 
             var leafProcessor = new CatalogLeafDataProcessor(
                 packageStorage,

src/Ng/Ng.csproj

@@ -171,9 +171,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="NuGet.Protocol">
-      <Version>5.0.0-preview1.5665</Version>
-    </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
       <Version>2.58.0</Version>
     </PackageReference>

src/Ng/Program.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using System.Net;
@@ -98,6 +99,12 @@ public static async Task MainAsync(string[] args)
 
                 Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
             }
+            catch (KeyNotFoundException knfe)
+            {
+                _logger?.LogError("An expected key was not found. One possible cause of this is required argument has not been provided: {Exception}", knfe);
+
+                Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
+            }
             catch (Exception e)
             {
                 _logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);

src/NuGet.Indexing/Extraction/CatalogPackageReader.cs

@@ -89,16 +89,11 @@ public override Task<byte[]> GetArchiveHashAsync(HashAlgorithmName hashAlgorithm
         }
 
         public override bool CanVerifySignedPackages(SignedPackageVerifierSettings verifierSettings)
-        {
-            return false;
-        }
-
-        public override string GetContentHashForSignedPackage(CancellationToken token)
         {
             throw new NotImplementedException();
         }
 
-        public override string GetContentHash(CancellationToken token)
+        public override string GetContentHash(CancellationToken token, Func<string> GetUnsignedPackageHash = null)
         {
             throw new NotImplementedException();
         }

src/NuGet.Indexing/NuGet.Indexing.csproj

@@ -181,9 +181,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="NuGet.Packaging">
-      <Version>5.0.0-preview1.5665</Version>
-    </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
       <Version>2.58.0</Version>
     </PackageReference>

src/NuGet.Protocol.Catalog/Models/AlternatePackage.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Newtonsoft.Json;
+
+namespace NuGet.Protocol.Catalog
+{
+    /// <summary>
+    /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#alternate-package
+    /// </summary>
+    public class AlternatePackage
+    {
+        [JsonProperty("@id")]
+        public string Url { get; set; }
+
+        [JsonProperty("@type")]
+        public string Type { get; set; }
+
+        [JsonProperty("id")]
+        public string Id { get; set; }
+
+        [JsonProperty("range")]
+        public string Range { get; set; }
+    }
+}

src/NuGet.Protocol.Catalog/Models/BasePackageDependencyGroup.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace NuGet.Protocol.Catalog
+{
+    public abstract class BasePackageDependencyGroup<TDependency> where TDependency : PackageDependency
+    {
+        [JsonProperty("@id")]
+        public string Url { get; set; }
+
+        [JsonProperty("@type")]
+        public string Type { get; set; }
+
+        [JsonProperty("dependencies")]
+        public List<TDependency> Dependencies { get; set; }
+
+        [JsonProperty("targetFramework")]
+        public string TargetFramework { get; set; }
+    }
+}

src/NuGet.Protocol.Catalog/Models/PackageDependency.cs

@@ -8,6 +8,12 @@ namespace NuGet.Protocol.Catalog
 {
     public class PackageDependency
     {
+        [JsonProperty("@id")]
+        public string Url { get; set; }
+
+        [JsonProperty("@type")]
+        public string Type { get; set; }
+
         [JsonProperty("id")]
         public string Id { get; set; }
 

src/NuGet.Protocol.Catalog/Models/PackageDependencyGroup.cs

@@ -1,17 +1,9 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System.Collections.Generic;
-using Newtonsoft.Json;
-
 namespace NuGet.Protocol.Catalog
 {
-    public class PackageDependencyGroup
+    public class PackageDependencyGroup : BasePackageDependencyGroup<PackageDependency>
     {
-        [JsonProperty("targetFramework")]
-        public string TargetFramework { get; set; }
-
-        [JsonProperty("dependencies")]
-        public List<PackageDependency> Dependencies { get; set; }
     }
 }

src/NuGet.Protocol.Catalog/Models/PackageDeprecation.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace NuGet.Protocol.Catalog
+{
+    /// <summary>
+    /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-deprecation
+    /// </summary>
+    public class PackageDeprecation
+    {
+        [JsonProperty("@id")]
+        public string Url { get; set; }
+
+        [JsonProperty("@type")]
+        public string Type { get; set; }
+
+        [JsonProperty("alternatePackage")]
+        public AlternatePackage AlternatePackage { get; set; }
+
+        [JsonProperty("message")]
+        public string Message { get; set; }
+
+        [JsonProperty("reasons")]
+        public List<string> Reasons { get; set; }
+    }
+}

src/NuGet.Protocol.Catalog/Models/PackageDetailsCatalogLeaf.cs

@@ -24,6 +24,9 @@ public class PackageDetailsCatalogLeaf : CatalogLeaf
         [JsonProperty("dependencyGroups")]
         public List<PackageDependencyGroup> DependencyGroups { get; set; }
 
+        [JsonProperty("deprecation")]
+        public PackageDeprecation Deprecation { get; set; }
+
         [JsonProperty("description")]
         public string Description { get; set; }
 

src/NuGet.Protocol.Catalog/NuGet.Protocol.Catalog.csproj

@@ -64,7 +64,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Protocol">
-      <Version>4.8.0</Version>
+      <Version>5.0.0-preview1.5707</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>
@@ -76,6 +76,8 @@
     <Compile Include="ICatalogLeafProcessor.cs" />
     <Compile Include="ICursor.cs" />
     <Compile Include="ISimpleHttpClient.cs" />
+    <Compile Include="Models\AlternatePackage.cs" />
+    <Compile Include="Models\BasePackageDependencyGroup.cs" />
     <Compile Include="Models\CatalogIndex.cs" />
     <Compile Include="Models\CatalogLeaf.cs" />
     <Compile Include="Models\CatalogLeafItem.cs" />
@@ -84,6 +86,7 @@
     <Compile Include="Models\CatalogPageContext.cs" />
     <Compile Include="Models\CatalogPageItem.cs" />
     <Compile Include="Models\ContextTypeDescription.cs" />
+    <Compile Include="Models\PackageDeprecation.cs" />
     <Compile Include="Models\ICatalogLeafItem.cs" />
     <Compile Include="Models\ModelExtensions.cs" />
     <Compile Include="Models\PackageDeleteCatalogLeaf.cs" />

src/NuGet.Services.V3/NuGet.Services.V3.csproj

@@ -39,6 +39,11 @@
     <Compile Include="CommitCollectorConfiguration.cs" />
     <Compile Include="DefaultBlobRequestOptions.cs" />
     <Compile Include="DependencyInjectionExtensions.cs" />
+    <Compile Include="Registration\Models\ICommitted.cs" />
+    <Compile Include="Registration\Models\RegistrationPackageDependency.cs" />
+    <Compile Include="Registration\Models\RegistrationContainerContext.cs" />
+    <Compile Include="Registration\Models\RegistrationLeafContext.cs" />
+    <Compile Include="Registration\Models\RegistrationPackageDependencyGroup.cs" />
     <Compile Include="Support\Guard.cs" />
     <Compile Include="Support\IdAndValue.cs" />
     <Compile Include="IV3TelemetryService.cs" />

src/NuGet.Services.V3/Registration/Models/ICommitted.cs

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Protocol.Registration
+{
+    public interface ICommitted
+    {
+        string CommitId { get; set; }
+        DateTimeOffset CommitTimestamp { get; set; }
+    }
+}