Read DowloadData from downloads.v1.json (#593)

The goal is to leave the existing download.v1.json data structure Downloads as-is to ensure existing search is unaffected.
Instead of changing Downloads, an API is added to read this file into the more convenient DownloadData class which is flexible enough for comparison and maintaining original case of IDs.
Progress on NuGet/NuGetGallery#6458

src/NuGet.Indexing/Downloads.cs

@@ -51,6 +51,45 @@ public DownloadsByVersion this[int docId]
             }
         }
 
+        public static void Load(string name, ILoader loader, Action<string, string, long> addCount)
+        {
+            // The data in downloads.v1.json will be an array of Package records - which has Id, Array of Versions and download count.
+            // Sample.json : [["AutofacContrib.NSubstitute",["2.4.3.700",406],["2.5.0",137]],["Assman.Core",["2.0.7",138]]....
+            using (var jsonReader = loader.GetReader(name))
+            {
+                jsonReader.Read();
+
+                while (jsonReader.Read())
+                {
+                    if (jsonReader.TokenType == JsonToken.StartArray)
+                    {
+                        JToken record = JToken.ReadFrom(jsonReader);
+
+                        // The second entry in each record should be an array of versions, if not move on to next entry.
+                        // This is a check to safe guard against invalid entries.
+                        if (record.Count() == 2 && record[1].Type != JTokenType.Array)
+                        {
+                            continue;
+                        }
+
+                        var id = record[0].ToString();
+
+                        foreach (JToken token in record)
+                        {
+                            if (token != null && token.Count() == 2)
+                            {
+                                var version = token[0].ToString();
+
+                                var count = token[1].ToObject<long>();
+
+                                addCount.Invoke(id, version, count);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         public void Load(string name, ILoader loader, FrameworkLogger logger)
         {
             // The data in downloads.v1.json will be an array of Package records - which has Id, Array of Versions and download count.

src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs

@@ -41,6 +41,27 @@ public AuxiliaryFileClient(
 
         private ICloudBlobContainer Container => _lazyContainer.Value;
 
+        public async Task<DownloadData> LoadDownloadDataAsync()
+        {
+            var result = await LoadAuxiliaryFileAsync(
+                _options.Value.AuxiliaryDataStorageDownloadsPath,
+                etag: null,
+                loadData: loader =>
+                {
+                    var downloadData = new DownloadData();
+
+                    Downloads.Load(
+                        name: null,
+                        loader: loader,
+                        addCount: downloadData.SetDownloadCount);
+
+                    return downloadData;
+                });
+
+            // Discard the etag and other metadata since this API is only ever used to read the latest data.
+            return result.Data;
+        }
+
         public async Task<AuxiliaryFileResult<Downloads>> LoadDownloadsAsync(string etag)
         {
             return await LoadAuxiliaryFileAsync(
@@ -73,7 +94,11 @@ private async Task<AuxiliaryFileResult<T>> LoadAuxiliaryFileAsync<T>(
             string etag,
             Func<ILoader, T> loadData) where T : class
         {
-            _logger.LogInformation("Attempted to load blob {BlobName} with etag {ETag}.", blobName, etag);
+            _logger.LogInformation(
+                "Attempted to load blob {BlobName} as {TypeName} with etag {ETag}.",
+                blobName,
+                typeof(T).FullName,
+                etag);
 
             var stopwatch = Stopwatch.StartNew();
             var blob = Container.GetBlobReference(blobName);

src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryFileClient.cs

@@ -9,6 +9,7 @@ namespace NuGet.Services.AzureSearch.AuxiliaryFiles
 {
     public interface IAuxiliaryFileClient
     {
+        Task<DownloadData> LoadDownloadDataAsync();
         Task<AuxiliaryFileResult<Downloads>> LoadDownloadsAsync(string etag);
         Task<AuxiliaryFileResult<HashSet<string>>> LoadVerifiedPackagesAsync(string etag);
     }

tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs

@@ -20,6 +20,49 @@ namespace NuGet.Services.AzureSearch.AuxiliaryFiles
 {
     public class AuxiliaryFileClientFacts
     {
+        public class LoadDownloadDataAsync : BaseFacts
+        {
+            public LoadDownloadDataAsync(ITestOutputHelper output) : base(output)
+            {
+            }
+
+            [Fact]
+            public async Task ReadsContent()
+            {
+                var json = @"
+[
+    [
+        ""NuGet.Frameworks"",
+        [ ""1.0.0"", 406],
+        [ ""2.0.0-ALPHA"", 137]
+    ],
+    [
+        ""NuGet.Versioning"",
+        [""3.0.0"", 138]
+    ]
+]
+";
+                _blob
+                    .Setup(x => x.OpenReadAsync(It.IsAny<AccessCondition>()))
+                    .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json)));
+
+                var actual = await _target.LoadDownloadDataAsync();
+
+                Assert.NotNull(actual);
+                Assert.Equal(406, actual["NuGet.Frameworks"]["1.0.0"]);
+                Assert.Equal(137, actual["NuGet.Frameworks"]["2.0.0-alpha"]);
+                Assert.Equal(138, actual["nuget.versioning"]["3.0.0"]);
+                Assert.Equal(0, actual["nuget.versioning"].GetDownloadCount("4.0.0"));
+                Assert.Equal(0, actual.GetDownloadCount("something.else"));
+                _blobClient.Verify(x => x.GetContainerReference("my-container"), Times.Once);
+                _blobClient.Verify(x => x.GetContainerReference(It.IsAny<string>()), Times.Once);
+                _container.Verify(x => x.GetBlobReference("my-downloads.json"), Times.Once);
+                _container.Verify(x => x.GetBlobReference(It.IsAny<string>()), Times.Once);
+                _blob.Verify(x => x.OpenReadAsync(null), Times.Once);
+                _blob.Verify(x => x.OpenReadAsync(It.IsAny<AccessCondition>()), Times.Once);
+            }
+        }
+
         public class LoadDownloadsAsync : BaseFacts
         {
             public LoadDownloadsAsync(ITestOutputHelper output) : base(output)