From 2342616f00e38123581f5fc647e895b15a88df46 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 29 Jan 2020 09:46:53 -0800 Subject: [PATCH] Merge NuGet.Indexing into NuGet.Services.AzureSearch (#739) Also, remove ILoader abstraction since it is no longer needed Progress on https://github.com/NuGet/NuGetGallery/issues/7822 --- NuGet.Services.Metadata.sln | 28 ---- build.ps1 | 2 - src/Catalog/Helpers/Retry.cs | 1 - src/Ng/Ng.csproj | 4 - src/NuGet.Indexing/Downloads.cs | 148 ------------------ src/NuGet.Indexing/DownloadsByVersion.cs | 52 ------ src/NuGet.Indexing/ILoader.cs | 13 -- src/NuGet.Indexing/NuGet.Indexing.csproj | 116 -------------- src/NuGet.Indexing/Properties/AssemblyInfo.cs | 20 --- src/NuGet.Indexing/SemVerHelpers.cs | 24 --- .../AuxiliaryFiles/AuxiliaryFileClient.cs | 54 +------ .../AuxiliaryFiles/DownloadOverrides.cs | 21 ++- .../AuxiliaryFiles/DownloadsV1Reader.cs | 49 ++++++ .../JsonStringArrayFileParser.cs | 12 +- .../DependencyInjectionExtensions.cs | 2 - .../NuGet.Services.AzureSearch.csproj | 8 +- .../SearchService/IndexOperationBuilder.cs | 1 - .../SearchService}/NuGetQueryParser.cs | 2 +- .../SearchService/ParsedQuery.cs | 1 - .../SearchService}/QueryField.cs | 2 +- .../SearchService/SearchTextBuilder.cs | 1 - .../Controllers/SearchController.cs | 5 +- .../NuGet.Services.SearchService.csproj | 4 - .../NuGet.Services.V3.csproj | 4 - test.ps1 | 1 - tests/NgTests/NgTests.csproj | 4 - .../DownloadsByVersionTests.cs | 122 --------------- .../NuGet.IndexingTests.csproj | 94 ----------- .../Properties/AssemblyInfo.cs | 35 ----- .../JsonStringArrayFileParsingTests.cs | 3 +- .../NuGet.Services.AzureSearch.Tests.csproj | 5 +- .../IndexOperationBuilderFacts.cs | 6 +- 32 files changed, 85 insertions(+), 759 deletions(-) delete mode 100644 src/NuGet.Indexing/Downloads.cs delete mode 100644 src/NuGet.Indexing/DownloadsByVersion.cs delete mode 100644 src/NuGet.Indexing/ILoader.cs delete mode 100644 src/NuGet.Indexing/NuGet.Indexing.csproj delete mode 100644 src/NuGet.Indexing/Properties/AssemblyInfo.cs delete mode 100644 src/NuGet.Indexing/SemVerHelpers.cs create mode 100644 src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs rename src/{NuGet.Indexing => NuGet.Services.AzureSearch/AuxiliaryFiles}/JsonStringArrayFileParser.cs (85%) rename src/{NuGet.Indexing => NuGet.Services.AzureSearch/SearchService}/NuGetQueryParser.cs (99%) rename src/{NuGet.Indexing => NuGet.Services.AzureSearch/SearchService}/QueryField.cs (88%) delete mode 100644 tests/NuGet.IndexingTests/DownloadsByVersionTests.cs delete mode 100644 tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj delete mode 100644 tests/NuGet.IndexingTests/Properties/AssemblyInfo.cs rename tests/{NuGet.IndexingTests => NuGet.Services.AzureSearch.Tests/AuxiliaryFiles}/JsonStringArrayFileParsingTests.cs (96%) diff --git a/NuGet.Services.Metadata.sln b/NuGet.Services.Metadata.sln index b9d07c818..3b08d455b 100644 --- a/NuGet.Services.Metadata.sln +++ b/NuGet.Services.Metadata.sln @@ -18,10 +18,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CatalogTests", "tests\Catal EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ng", "src\Ng\Ng.csproj", "{5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Indexing", "src\NuGet.Indexing\NuGet.Indexing.csproj", "{DDB34145-870F-42C3-9663-A9390CEE1E35}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.IndexingTests", "tests\NuGet.IndexingTests\NuGet.IndexingTests.csproj", "{43665489-01F0-4F35-8A33-258F2231B2D8}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NgTests", "tests\NgTests\NgTests.csproj", "{05C1C78A-9966-4922-9065-A099023E7366}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Services", "Services", "{97E23323-BA7A-48F0-A578-858B82B6D8FB}" @@ -117,28 +113,6 @@ Global {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|Mixed Platforms.Build.0 = Release|Any CPU {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|x64.ActiveCfg = Release|Any CPU {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|x64.Build.0 = Release|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Debug|x64.ActiveCfg = Debug|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Release|Any CPU.Build.0 = Release|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {DDB34145-870F-42C3-9663-A9390CEE1E35}.Release|x64.ActiveCfg = Release|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Debug|x64.ActiveCfg = Debug|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Debug|x64.Build.0 = Debug|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Release|Any CPU.Build.0 = Release|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Release|x64.ActiveCfg = Release|Any CPU - {43665489-01F0-4F35-8A33-258F2231B2D8}.Release|x64.Build.0 = Release|Any CPU {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Any CPU.Build.0 = Debug|Any CPU {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -387,8 +361,6 @@ Global {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} = {5DE01C58-D5F7-482F-8256-A8333064384C} {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} - {DDB34145-870F-42C3-9663-A9390CEE1E35} = {5DE01C58-D5F7-482F-8256-A8333064384C} - {43665489-01F0-4F35-8A33-258F2231B2D8} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} {05C1C78A-9966-4922-9065-A099023E7366} = {F1C83FD9-A498-483E-ADFA-B55D82A14965} {717E9A81-75C5-418E-92ED-18CAC55BC345} = {5DE01C58-D5F7-482F-8256-A8333064384C} {1745A383-D0BE-484B-81EB-27B20F6AC6C5} = {5DE01C58-D5F7-482F-8256-A8333064384C} diff --git a/build.ps1 b/build.ps1 index 7c6778a94..62cd29261 100644 --- a/build.ps1 +++ b/build.ps1 @@ -75,7 +75,6 @@ Invoke-BuildStep 'Restoring solution packages' { ` Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { $assemblyInfos = ` - "src\NuGet.Indexing\Properties\AssemblyInfo.g.cs", ` "src\Catalog\Properties\AssemblyInfo.g.cs", ` "src\NuGet.ApplicationInsights.Owin\Properties\AssemblyInfo.g.cs", ` "src\Ng\Properties\AssemblyInfo.g.cs", ` @@ -115,7 +114,6 @@ Invoke-BuildStep 'Signing the binaries' { Invoke-BuildStep 'Creating artifacts' { $csprojPackages = ` - "src\NuGet.Indexing\NuGet.Indexing.csproj", ` "src\Catalog\NuGet.Services.Metadata.Catalog.csproj", ` "src\NuGet.ApplicationInsights.Owin\NuGet.ApplicationInsights.Owin.csproj", ` "src\NuGet.Services.Metadata.Catalog.Monitoring\NuGet.Services.Metadata.Catalog.Monitoring.csproj", ` diff --git a/src/Catalog/Helpers/Retry.cs b/src/Catalog/Helpers/Retry.cs index 94c9a7bc3..614a5b76d 100644 --- a/src/Catalog/Helpers/Retry.cs +++ b/src/Catalog/Helpers/Retry.cs @@ -10,7 +10,6 @@ namespace NuGet.Services.Metadata.Catalog.Helpers /// Can (and probably should) be replaced with Polly library if the project is updated to target .netfx 4.7.2. /// In current state Polly pulls a ton of System.* dependencies which we previously didn't have. /// - /// public class Retry { /// diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj index eb5dae27e..5673ffe24 100644 --- a/src/Ng/Ng.csproj +++ b/src/Ng/Ng.csproj @@ -107,10 +107,6 @@ {e97f23b8-ecb0-4afa-b00c-015c39395fef} NuGet.Services.Metadata.Catalog - - {ddb34145-870f-42c3-9663-a9390cee1e35} - NuGet.Indexing - {5ABE8807-2209-4948-9FC5-1980A507C47A} NuGet.Jobs.Catalog2Registration diff --git a/src/NuGet.Indexing/Downloads.cs b/src/NuGet.Indexing/Downloads.cs deleted file mode 100644 index f39b8321d..000000000 --- a/src/NuGet.Indexing/Downloads.cs +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using FrameworkLogger = Microsoft.Extensions.Logging.ILogger; - -namespace NuGet.Indexing -{ - public class Downloads - { - private IDictionary _downloads = - new Dictionary(StringComparer.OrdinalIgnoreCase); - - private IDictionary _byDocId = - new Dictionary(); - - public DownloadsByVersion this[string packageId] - { - get - { - DownloadsByVersion result = null; - _downloads.TryGetValue(packageId, out result); - - return result; - } - - set - { - _downloads[packageId] = value; - } - } - - public DownloadsByVersion this[int docId] - { - get - { - DownloadsByVersion result = null; - _byDocId.TryGetValue(docId, out result); - - return result; - } - - set - { - _byDocId[docId] = value; - } - } - - public static void Load(string name, ILoader loader, Action addCount) - { - // The data in downloads.v1.json will be an array of Package records - which has Id, Array of Versions and download count. - // Sample.json : [["AutofacContrib.NSubstitute",["2.4.3.700",406],["2.5.0",137]],["Assman.Core",["2.0.7",138]].... - using (var jsonReader = loader.GetReader(name)) - { - jsonReader.Read(); - - while (jsonReader.Read()) - { - if (jsonReader.TokenType == JsonToken.StartArray) - { - JToken record = JToken.ReadFrom(jsonReader); - - // The second entry in each record should be an array of versions, if not move on to next entry. - // This is a check to safe guard against invalid entries. - if (record.Count() == 2 && record[1].Type != JTokenType.Array) - { - continue; - } - - var id = record[0].ToString(); - - foreach (JToken token in record) - { - if (token != null && token.Count() == 2) - { - var version = token[0].ToString(); - - var count = token[1].ToObject(); - - addCount.Invoke(id, version, count); - } - } - } - } - } - } - - public void Load(string name, ILoader loader, FrameworkLogger logger) - { - // The data in downloads.v1.json will be an array of Package records - which has Id, Array of Versions and download count. - // Sample.json : [["AutofacContrib.NSubstitute",["2.4.3.700",406],["2.5.0",137]],["Assman.Core",["2.0.7",138]].... - using (var jsonReader = loader.GetReader(name)) - { - try - { - jsonReader.Read(); - - while (jsonReader.Read()) - { - try - { - if (jsonReader.TokenType == JsonToken.StartArray) - { - JToken record = JToken.ReadFrom(jsonReader); - string id = String.Intern(record[0].ToString().ToLowerInvariant()); - - // The second entry in each record should be an array of versions, if not move on to next entry. - // This is a check to safe guard against invalid entries. - if (record.Count() == 2 && record[1].Type != JTokenType.Array) - { - continue; - } - - if (!_downloads.ContainsKey(id)) - { - _downloads.Add(id, new DownloadsByVersion()); - } - var versions = _downloads[id]; - - foreach (JToken token in record) - { - if (token != null && token.Count() == 2) - { - string version = String.Intern(token[0].ToString().ToLowerInvariant()); - versions[version] = token[1].ToObject(); - } - } - } - } - catch (JsonReaderException ex) - { - logger.LogInformation("Invalid entry found in downloads.v1.json. Exception Message : {0}", ex.Message); - } - } - } - catch (JsonReaderException ex) - { - logger.LogError("Data present in downloads.v1.json is invalid. Couldn't get download data.", ex); - } - } - } - } -} \ No newline at end of file diff --git a/src/NuGet.Indexing/DownloadsByVersion.cs b/src/NuGet.Indexing/DownloadsByVersion.cs deleted file mode 100644 index ce7b957ba..000000000 --- a/src/NuGet.Indexing/DownloadsByVersion.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; - -namespace NuGet.Indexing -{ - public class DownloadsByVersion - { - private readonly IDictionary _downloadsByVersion = - new Dictionary(StringComparer.OrdinalIgnoreCase); - - private int _total; - - /// - /// The total count of downloads across all versions - /// - /// - /// This is thread safe as long as set is not being called from multiple threads - /// - public int Total - { - get - { - return _total; - } - } - - public int this[string version] - { - get - { - int count = 0; - - _downloadsByVersion.TryGetValue(version, out count); - - return count; - } - - // Set is only ever called when the auxiliary data is reloaded, which is only ever done on one thread per instance. - set - { - int oldValue; - _downloadsByVersion.TryGetValue(version, out oldValue); - _downloadsByVersion[version] = value; - - _total = _total + value - oldValue; - } - } - } -} diff --git a/src/NuGet.Indexing/ILoader.cs b/src/NuGet.Indexing/ILoader.cs deleted file mode 100644 index 769f59f33..000000000 --- a/src/NuGet.Indexing/ILoader.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using Newtonsoft.Json; - -namespace NuGet.Indexing -{ - public interface ILoader - { - JsonReader GetReader(string name); - } -} diff --git a/src/NuGet.Indexing/NuGet.Indexing.csproj b/src/NuGet.Indexing/NuGet.Indexing.csproj deleted file mode 100644 index 747ec49fe..000000000 --- a/src/NuGet.Indexing/NuGet.Indexing.csproj +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - Debug - AnyCPU - {DDB34145-870F-42C3-9663-A9390CEE1E35} - Library - Properties - NuGet.Indexing - NuGet.Indexing - v4.7.2 - 512 - - true - - - true - win - - - .NET Foundation - https://github.com/NuGet/NuGet.Services.Metadata/blob/master/LICENSE - https://github.com/NuGet/NuGet.Services.Metadata - Lucene-based Search Indexing tools for NuGet - nuget;services;search;index;lucene - Copyright .NET Foundation - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - false - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0.3.0 - runtime; build; native; contentfiles; analyzers - all - - - 4.8.0 - runtime; build; native; contentfiles; analyzers - all - - - 2.66.0 - - - 4.4.5-dev-3213233 - - - 4.3.3 - - - 9.3.3 - - - - - - - - {e97f23b8-ecb0-4afa-b00c-015c39395fef} - NuGet.Services.Metadata.Catalog - - - - - ..\..\build - $(BUILD_SOURCESDIRECTORY)\build - $(NuGetBuildPath) - none - - - - \ No newline at end of file diff --git a/src/NuGet.Indexing/Properties/AssemblyInfo.cs b/src/NuGet.Indexing/Properties/AssemblyInfo.cs deleted file mode 100644 index 847c73b79..000000000 --- a/src/NuGet.Indexing/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Reflection; -using System.Runtime.CompilerServices; - -[assembly: AssemblyTitle("NuGet.Indexing")] -[assembly: AssemblyDescription("Lucene-based Search Indexing tools for NuGet")] - -[assembly: AssemblyCompany(".NET Foundation")] -[assembly: AssemblyProduct("NuGet Services")] -[assembly: AssemblyCopyright("\x00a9 .NET Foundation. All rights reserved.")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -#if SIGNED_BUILD -[assembly: InternalsVisibleTo("NuGet.IndexingTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] -#else -[assembly: InternalsVisibleTo("NuGet.IndexingTests")] -#endif \ No newline at end of file diff --git a/src/NuGet.Indexing/SemVerHelpers.cs b/src/NuGet.Indexing/SemVerHelpers.cs deleted file mode 100644 index a1fb4264c..000000000 --- a/src/NuGet.Indexing/SemVerHelpers.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using NuGet.Versioning; - -namespace NuGet.Indexing -{ - public class SemVerHelpers - { - public static readonly string SemVerLevelKeySemVer2 = "2"; - public static readonly NuGetVersion SemVer1Level = new NuGetVersion("1.0.0"); - public static readonly NuGetVersion SemVer2Level = new NuGetVersion("2.0.0"); - - public static bool ShouldIncludeSemVer2Results(NuGetVersion semVerLevel) - { - if (semVerLevel == null) - { - return false; - } - - return semVerLevel >= SemVer2Level; - } - } -} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs index 58454df75..a31f11d0c 100644 --- a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using NuGet.Indexing; using NuGetGallery; namespace NuGet.Services.AzureSearch.AuxiliaryFiles @@ -44,15 +43,10 @@ public async Task LoadDownloadDataAsync() { return await LoadAuxiliaryFileAsync( _options.Value.AuxiliaryDataStorageDownloadsPath, - loadData: loader => + loadData: reader => { var downloadData = new DownloadData(); - - Downloads.Load( - name: null, - loader: loader, - addCount: downloadData.SetDownloadCount); - + DownloadsV1Reader.Load(reader, downloadData.SetDownloadCount); return downloadData; }); } @@ -61,35 +55,26 @@ public async Task> LoadVerifiedPackagesAsync() { return await LoadAuxiliaryFileAsync( _options.Value.AuxiliaryDataStorageVerifiedPackagesPath, - loader => JsonStringArrayFileParser.Load( - fileName: null, - loader: loader, - logger: _logger)); + reader => JsonStringArrayFileParser.Load(reader, _logger)); } public async Task> LoadExcludedPackagesAsync() { return await LoadAuxiliaryFileAsync( _options.Value.AuxiliaryDataStorageExcludedPackagesPath, - loader => JsonStringArrayFileParser.Load( - fileName: null, - loader: loader, - logger: _logger)); + reader => JsonStringArrayFileParser.Load(reader, _logger)); } public async Task> LoadDownloadOverridesAsync() { return await LoadAuxiliaryFileAsync( _options.Value.AuxiliaryDataStorageDownloadOverridesPath, - loader => DownloadOverrides.Load( - fileName: null, - loader: loader, - logger: _logger)); + reader => DownloadOverrides.Load(reader, _logger)); } private async Task LoadAuxiliaryFileAsync( string blobName, - Func loadData) where T : class + Func loadData) where T : class { _logger.LogInformation( "Attempted to load blob {BlobName} as {TypeName}.", @@ -102,8 +87,7 @@ private async Task LoadAuxiliaryFileAsync( using (var textReader = new StreamReader(stream)) using (var jsonReader = new JsonTextReader(textReader)) { - var loader = new LoaderAdapter(jsonReader); - var data = loadData(loader); + var data = loadData(jsonReader); stopwatch.Stop(); _telemetryService.TrackAuxiliaryFileDownloaded(blobName, stopwatch.Elapsed); @@ -115,29 +99,5 @@ private async Task LoadAuxiliaryFileAsync( return data; }; } - - /// - /// This is an adapter implementation so that we can use the pre-existing auxiliary file reading code. It simply - /// returns a provided to the constructor and performs no additional network requests. - /// - private class LoaderAdapter : ILoader - { - private readonly JsonReader _jsonReader; - - public LoaderAdapter(JsonReader jsonReader) - { - _jsonReader = jsonReader ?? throw new ArgumentNullException(nameof(jsonReader)); - } - - public JsonReader GetReader(string name) - { - if (name != null) - { - throw new ArgumentException("The provided blob name should be null.", nameof(name)); - } - - return _jsonReader; - } - } } } diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadOverrides.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadOverrides.cs index dc82206b9..44d46b71c 100644 --- a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadOverrides.cs +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadOverrides.cs @@ -1,8 +1,10 @@ -using System; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; using System.Collections.Generic; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using NuGet.Indexing; namespace NuGet.Services.AzureSearch.AuxiliaryFiles { @@ -10,22 +12,19 @@ public static class DownloadOverrides { private static readonly JsonSerializer Serializer = new JsonSerializer(); - public static IReadOnlyDictionary Load(string fileName, ILoader loader, ILogger logger) + public static IReadOnlyDictionary Load(JsonReader reader, ILogger logger) { try { - using (var reader = loader.GetReader(fileName)) - { - var downloadOverrides = Serializer.Deserialize>(reader); + var downloadOverrides = Serializer.Deserialize>(reader); - return new Dictionary( - downloadOverrides, - StringComparer.OrdinalIgnoreCase); - } + return new Dictionary( + downloadOverrides, + StringComparer.OrdinalIgnoreCase); } catch (Exception ex) { - logger.LogError(0, ex, "Unable to load download overrides {FileName} due to exception", fileName); + logger.LogError(0, ex, "Unable to load download overrides due to exception"); throw; } } diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs new file mode 100644 index 000000000..7ece6bf3f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public static class DownloadsV1Reader + { + public static void Load(JsonReader jsonReader, Action addCount) + { + // The data in downloads.v1.json will be an array of Package records - which has Id, Array of Versions and download count. + // Sample.json : [["AutofacContrib.NSubstitute",["2.4.3.700",406],["2.5.0",137]],["Assman.Core",["2.0.7",138]].... + jsonReader.Read(); + + while (jsonReader.Read()) + { + if (jsonReader.TokenType == JsonToken.StartArray) + { + JToken record = JToken.ReadFrom(jsonReader); + + // The second entry in each record should be an array of versions, if not move on to next entry. + // This is a check to safe guard against invalid entries. + if (record.Count() == 2 && record[1].Type != JTokenType.Array) + { + continue; + } + + var id = record[0].ToString(); + + foreach (JToken token in record) + { + if (token != null && token.Count() == 2) + { + var version = token[0].ToString(); + + var count = token[1].ToObject(); + + addCount.Invoke(id, version, count); + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Indexing/JsonStringArrayFileParser.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/JsonStringArrayFileParser.cs similarity index 85% rename from src/NuGet.Indexing/JsonStringArrayFileParser.cs rename to src/NuGet.Services.AzureSearch/AuxiliaryFiles/JsonStringArrayFileParser.cs index 82c782a88..46f7dcbbe 100644 --- a/src/NuGet.Indexing/JsonStringArrayFileParser.cs +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/JsonStringArrayFileParser.cs @@ -6,9 +6,8 @@ using System.IO; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using FrameworkLogger = Microsoft.Extensions.Logging.ILogger; -namespace NuGet.Indexing +namespace NuGet.Services.AzureSearch.AuxiliaryFiles { public static class JsonStringArrayFileParser { @@ -19,18 +18,15 @@ public static class JsonStringArrayFileParser /// The loader that should be used to fetch the file's content /// The logger /// A case-insensitive set of all the strings in the json array - public static HashSet Load(string fileName, ILoader loader, FrameworkLogger logger) + public static HashSet Load(JsonReader reader, ILogger logger) { try { - using (var reader = loader.GetReader(fileName)) - { - return Parse(reader); - } + return Parse(reader); } catch (Exception ex) { - logger.LogError("Unable to load {FileName} as deserialization threw: {Exception}", fileName, ex); + logger.LogError(ex, "Unable to load JSON string array."); throw; } diff --git a/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs b/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs index 6669d9732..5800e75f8 100644 --- a/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs +++ b/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs @@ -11,8 +11,6 @@ using Microsoft.Extensions.Options; using Microsoft.Rest; using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Protocol; using NuGet.Protocol.Catalog; using NuGet.Services.AzureSearch.Auxiliary2AzureSearch; diff --git a/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj index 87cfc1494..64e4d71a9 100644 --- a/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj +++ b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj @@ -56,6 +56,8 @@ + + @@ -177,6 +179,8 @@ + + @@ -243,10 +247,6 @@ {e97f23b8-ecb0-4afa-b00c-015c39395fef} NuGet.Services.Metadata.Catalog - - {DDB34145-870F-42C3-9663-A9390CEE1E35} - NuGet.Indexing - {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} NuGet.Protocol.Catalog diff --git a/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs index e46ab49bf..654acba4d 100644 --- a/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs +++ b/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs @@ -3,7 +3,6 @@ using System; using System.Linq; -using NuGet.Indexing; using NuGet.Packaging; using NuGet.Versioning; diff --git a/src/NuGet.Indexing/NuGetQueryParser.cs b/src/NuGet.Services.AzureSearch/SearchService/NuGetQueryParser.cs similarity index 99% rename from src/NuGet.Indexing/NuGetQueryParser.cs rename to src/NuGet.Services.AzureSearch/SearchService/NuGetQueryParser.cs index d11cc3008..11c533ab7 100644 --- a/src/NuGet.Indexing/NuGetQueryParser.cs +++ b/src/NuGet.Services.AzureSearch/SearchService/NuGetQueryParser.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Text; -namespace NuGet.Indexing +namespace NuGet.Services.AzureSearch.SearchService { public class NuGetQueryParser { diff --git a/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs b/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs index 2ef96465e..bb6dcfd04 100644 --- a/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs +++ b/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using NuGet.Indexing; namespace NuGet.Services.AzureSearch.SearchService { diff --git a/src/NuGet.Indexing/QueryField.cs b/src/NuGet.Services.AzureSearch/SearchService/QueryField.cs similarity index 88% rename from src/NuGet.Indexing/QueryField.cs rename to src/NuGet.Services.AzureSearch/SearchService/QueryField.cs index 4e8a0496e..9f150d218 100644 --- a/src/NuGet.Indexing/QueryField.cs +++ b/src/NuGet.Services.AzureSearch/SearchService/QueryField.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -namespace NuGet.Indexing +namespace NuGet.Services.AzureSearch.SearchService { public enum QueryField { diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs index 485879518..1ad324302 100644 --- a/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Text.RegularExpressions; using Microsoft.Extensions.Options; -using NuGet.Indexing; using NuGet.Packaging; using NuGet.Services.Metadata.Catalog; using NuGet.Versioning; diff --git a/src/NuGet.Services.SearchService/Controllers/SearchController.cs b/src/NuGet.Services.SearchService/Controllers/SearchController.cs index fb0e365a6..966d7bc70 100644 --- a/src/NuGet.Services.SearchService/Controllers/SearchController.cs +++ b/src/NuGet.Services.SearchService/Controllers/SearchController.cs @@ -8,7 +8,6 @@ using System.Threading.Tasks; using System.Web.Http; using System.Web.Http.Description; -using NuGet.Indexing; using NuGet.Services.AzureSearch.SearchService; using NuGet.Versioning; @@ -16,6 +15,8 @@ namespace NuGet.Services.SearchService.Controllers { public class SearchController : ApiController { + private static readonly NuGetVersion SemVer2Level = new NuGetVersion("2.0.0"); + private const int DefaultSkip = 0; private const int DefaultTake = SearchParametersBuilder.DefaultTake; @@ -192,7 +193,7 @@ private static bool GetIncludeSemVer2(string semVerLevel) } else { - return SemVerHelpers.ShouldIncludeSemVer2Results(semVerLevelVersion); + return semVerLevelVersion >= SemVer2Level; } } } diff --git a/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj b/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj index ba5e75eca..98487d8f0 100644 --- a/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj +++ b/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj @@ -97,10 +97,6 @@ - - {ddb34145-870f-42c3-9663-a9390cee1e35} - NuGet.Indexing - {1a53fe3d-8041-4773-942f-d73aef5b82b2} NuGet.Services.AzureSearch diff --git a/src/NuGet.Services.V3/NuGet.Services.V3.csproj b/src/NuGet.Services.V3/NuGet.Services.V3.csproj index 18bf1dda2..2cb704fc2 100644 --- a/src/NuGet.Services.V3/NuGet.Services.V3.csproj +++ b/src/NuGet.Services.V3/NuGet.Services.V3.csproj @@ -80,10 +80,6 @@ {e97f23b8-ecb0-4afa-b00c-015c39395fef} NuGet.Services.Metadata.Catalog - - {DDB34145-870F-42C3-9663-A9390CEE1E35} - NuGet.Indexing - {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} NuGet.Protocol.Catalog diff --git a/test.ps1 b/test.ps1 index 5914037fa..3fcd57755 100644 --- a/test.ps1 +++ b/test.ps1 @@ -26,7 +26,6 @@ Function Run-Tests { $TestAssemblies = ` "tests\NgTests\bin\$Configuration\NgTests.dll", ` - "tests\NuGet.IndexingTests\bin\$Configuration\NuGet.IndexingTests.dll", ` "tests\CatalogTests\bin\$Configuration\CatalogTests.dll", ` "tests\CatalogMetadataTests\bin\$Configuration\CatalogMetadataTests.dll", ` "tests\NuGet.Protocol.Catalog.Tests\bin\$Configuration\NuGet.Protocol.Catalog.Tests.dll", ` diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj index bec520a57..a1cf0f3ac 100644 --- a/tests/NgTests/NgTests.csproj +++ b/tests/NgTests/NgTests.csproj @@ -184,10 +184,6 @@ {1745a383-d0be-484b-81eb-27b20f6ac6c5} NuGet.Services.Metadata.Catalog.Monitoring - - {43665489-01F0-4F35-8A33-258F2231B2D8} - NuGet.IndexingTests - diff --git a/tests/NuGet.IndexingTests/DownloadsByVersionTests.cs b/tests/NuGet.IndexingTests/DownloadsByVersionTests.cs deleted file mode 100644 index 83afecbfc..000000000 --- a/tests/NuGet.IndexingTests/DownloadsByVersionTests.cs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Collections.Generic; -using NuGet.Indexing; -using Xunit; - -namespace NuGet.IndexingTests -{ - public class DownloadsByVersionTests - { - [Theory] - [MemberData(nameof(RollingTotalTestData))] - public void RollingTotalCountTest(Dictionary versionDownloads, int expectedTotal) - { - var newDownloadsByVersion = new DownloadsByVersion(); - var rollingTotal = 0; - foreach (var version in versionDownloads) - { - newDownloadsByVersion[version.Key] = version.Value; - rollingTotal += version.Value; - Assert.Equal(rollingTotal, newDownloadsByVersion.Total); - Assert.Equal(newDownloadsByVersion[version.Key], version.Value); - } - - Assert.Equal(expectedTotal, newDownloadsByVersion.Total); - } - - [Theory] - [MemberData(nameof(UpdateTotalTestData))] - public void UpdateTotalTest(Dictionary originalData, Dictionary updatedVersionDownloads, int expectedTotal) - { - var newDownloadsByVersion = new DownloadsByVersion(); - foreach (var entry in originalData) - { - newDownloadsByVersion[entry.Key] = entry.Value; - } - - foreach (var version in updatedVersionDownloads) - { - newDownloadsByVersion[version.Key] = version.Value; - } - - Assert.Equal(expectedTotal, newDownloadsByVersion.Total); - } - - [Fact] - public void ReturnZeroForUnknownTest() - { - var newDownloadsByVersion = new DownloadsByVersion(); - Assert.Equal(0, newDownloadsByVersion["not.real.version"]); - } - - public static IEnumerable RollingTotalTestData - { - get - { - // simple - yield return new object[] - { - new Dictionary { - { "1.0.0", 10 }, - { "1.0.1", 10 }, - { "2.0.0", 20 } - }, - 40 - }; - } - } - - public static IEnumerable UpdateTotalTestData - { - get - { - // simple overwrite - yield return new object[] - { - new Dictionary { - { "1.0.0", 10 }, - { "1.0.1", 10 }, - { "2.0.0", 20 } - }, - new Dictionary { - { "2.0.0", 30 } - }, - 50 - }; - - // overwrite + new - yield return new object[] - { - new Dictionary { - { "1.0.0", 10 }, - { "1.0.1", 10 }, - { "2.0.0", 20 } - }, - new Dictionary { - { "2.0.0", 30 }, - { "3.0.0", 100 } - }, - 150 - }; - - // overwrite all - yield return new object[] - { - new Dictionary { - { "1.0.0", 10 }, - { "1.0.1", 10 }, - { "2.0.0", 20 } - }, - new Dictionary { - { "1.0.0", 100 }, - { "1.0.1", 100 }, - { "2.0.0", 100 } - }, - 300 - }; - } - } - } -} diff --git a/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj b/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj deleted file mode 100644 index e38153072..000000000 --- a/tests/NuGet.IndexingTests/NuGet.IndexingTests.csproj +++ /dev/null @@ -1,94 +0,0 @@ - - - - - Debug - AnyCPU - {43665489-01F0-4F35-8A33-258F2231B2D8} - Library - Properties - NuGet.IndexingTests - NuGet.IndexingTests - v4.7.2 - 512 - - - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - true - true - - - - - - - - - - - - - - - - - - - - - - {ddb34145-870f-42c3-9663-a9390cee1e35} - NuGet.Indexing - - - - - - - - 4.10.1 - - - 2.66.0 - - - 2.4.1 - - - 2.4.1 - runtime; build; native; contentfiles; analyzers - all - - - - - - ..\..\build - $(BUILD_SOURCESDIRECTORY)\build - $(NuGetBuildPath) - none - - - \ No newline at end of file diff --git a/tests/NuGet.IndexingTests/Properties/AssemblyInfo.cs b/tests/NuGet.IndexingTests/Properties/AssemblyInfo.cs deleted file mode 100644 index 44999f380..000000000 --- a/tests/NuGet.IndexingTests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Reflection; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("NuGet.IndexingTests")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("NuGet.IndexingTests")] -[assembly: AssemblyCopyright("Copyright © 2015")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("43665489-01f0-4f35-8a33-258f2231b2d8")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/NuGet.IndexingTests/JsonStringArrayFileParsingTests.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/JsonStringArrayFileParsingTests.cs similarity index 96% rename from tests/NuGet.IndexingTests/JsonStringArrayFileParsingTests.cs rename to tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/JsonStringArrayFileParsingTests.cs index 71a4a7d44..7b8bfb70c 100644 --- a/tests/NuGet.IndexingTests/JsonStringArrayFileParsingTests.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/JsonStringArrayFileParsingTests.cs @@ -6,10 +6,9 @@ using System.IO; using System.Linq; using Newtonsoft.Json; -using NuGet.Indexing; using Xunit; -namespace NuGet.IndexingTests +namespace NuGet.Services.AzureSearch.AuxiliaryFiles { public class JsonStringArrayFileParserTest { diff --git a/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj b/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj index 65355c9c6..8514881f1 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj +++ b/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj @@ -42,6 +42,7 @@ + @@ -93,10 +94,6 @@ {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} NuGet.Services.Metadata.Catalog - - {DDB34145-870F-42C3-9663-A9390CEE1E35} - NuGet.Indexing - {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} NuGet.Protocol.Catalog diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs index 79b869666..83cb1ac96 100644 --- a/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs @@ -1,8 +1,10 @@ -using System.Collections.Generic; +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; using System.Linq; using Microsoft.Azure.Search.Models; using Moq; -using NuGet.Indexing; using Xunit; namespace NuGet.Services.AzureSearch.SearchService