From b2133755c3cbc4967d01d039cd210fd64d26898f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Mon, 22 Jul 2019 11:09:52 -0700 Subject: [PATCH] [Azure Search] Update relevancy tests (#588) I've split tests into two categories: 1. "First results" - These tests that the expected results are first 2. "Top results" - These tests that the expected results are found I ensured that our tests align with the most frequently selected results, as per our search telemetry. Addresses https://github.com/nuget/nugetgallery/issues/6980 --- .../Relevancy/V3RelevancyFunctionalTests.cs | 95 +++++++++++-------- .../Support/NuGetSearchFunctionalTestBase.cs | 17 +++- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs index 16595585e..f33183245 100644 --- a/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -14,65 +16,74 @@ public V3RelevancyFunctionalTests(CommonFixture fixture, ITestOutputHelper testO { } - [RelevancyFact] - public async Task Json() + [RelevancyTheory] + [MemberData(nameof(EnsureFirstResultsData))] + public async Task EnsureFirstResults(string searchTerm, string[] expectedFirstResults) { - var results = await SearchAsync("json"); + var results = await SearchAsync(searchTerm, take: 10); - Assert.True(results.Count > 2); - Assert.Equal("json", results[0]); - Assert.Equal("newtonsoft.json", results[1]); - } - - [RelevancyFact] - public async Task NewtonsoftJson() - { - var results = await SearchAsync("Newtonsoft.Json"); + Assert.True(results.Count > expectedFirstResults.Length); - Assert.NotEmpty(results); - Assert.Equal("newtonsoft.json", results[0]); + for (var i = 0; i < expectedFirstResults.Length; i++) + { + Assert.True(expectedFirstResults[i] == results[i], $"Expected result '{expectedFirstResults[i]}' at index #{i} for query '{searchTerm}'"); + } } - [RelevancyFact] - public async Task Log() + public static IEnumerable EnsureFirstResultsData() { - var results = await SearchAsync("Log"); + // Test that common queries have the most frequently selected results at the top. + // These results were determined using the "BrowserSearchPage" and "BrowserSearchSelection" metrics + // on the Gallery's Application Insights telemetry. + // TODO: Reduce exact match boost and update "json.net", "entity framework", "redis", and "csv" tests. + // See: https://github.com/NuGet/NuGetGallery/issues/7330 + yield return new object[] { "newtonsoft.json", new[] { "newtonsoft.json" } }; + yield return new object[] { "newtonsoft", new[] { "newtonsoft.json" } }; + yield return new object[] { "json.net", new[] { "json.net", "newtonsoft.json" } }; + yield return new object[] { "json", new[] { "json", "newtonsoft.json" } }; - Assert.NotEmpty(results); - Assert.Contains("log4net", results); + yield return new object[] { "tags:\"aws-sdk-v3\"", new[] { "awssdk.core", "awssdk.s3" } }; - // TODO: These should be on the first page! - //Assert.Contains("nlog", results); - //Assert.Contains("serilog", results); - //Assert.Contains("microsoft.extensions.logging, results); - } + yield return new object[] { "entityframework", new[] { "entityframework" } }; + yield return new object[] { "entity framework", new[] { "entity", "entityframework" } }; + yield return new object[] { "EntityFrameworkCore", new[] { "microsoft.entityframeworkcore" } }; + yield return new object[] { "microsoft.entityframeworkcore", new[] { "microsoft.entityframeworkcore" } }; + yield return new object[] { "mysql", new[] { "mysql.data" } }; - [RelevancyFact] - public async Task EntityFrameworkCore() - { - var results = await SearchAsync("EntityFrameworkCore"); + yield return new object[] { "microsoft.aspnetcore.app", new[] { "microsoft.aspnetcore.app" } }; + yield return new object[] { "microsoft.extensions.logging", new[] { "microsoft.extensions.logging" } }; - Assert.Equal("microsoft.entityframeworkcore", results[0]); + yield return new object[] { "xunit", new[] { "xunit" } }; + yield return new object[] { "nunit", new[] { "nunit" } }; + yield return new object[] { "dapper", new[] { "dapper" } }; + yield return new object[] { "log4net", new[] { "log4net" } }; + yield return new object[] { "automapper", new[] { "automapper" } }; + yield return new object[] { "csv", new[] { "csv", "csvhelper" } }; + yield return new object[] { "bootstrap", new[] { "bootstrap" } }; + yield return new object[] { "moq", new[] { "moq" } }; + yield return new object[] { "serilog", new[] { "serilog" } }; + yield return new object[] { "redis", new[] { "redis", "stackexchange.redis", "microsoft.extensions.caching.redis" } }; } - [RelevancyFact] - public async Task MicrosoftExtensions() + [RelevancyTheory] + [MemberData(nameof(EnsureTopResultsData))] + public async Task EnsureTopResults(string searchTerm, string[] expectedTopResults) { - var results = await SearchAsync("Microsoft.Extensions"); + var results = await SearchAsync(searchTerm, take: 10); + + Assert.True(results.Count > expectedTopResults.Length); - Assert.Contains("microsoft.extensions.logging", results); - Assert.Contains("microsoft.extensions.configuration", results); - Assert.Contains("microsoft.extensions.dependencyinjection", results); + foreach (var expectedTopResult in expectedTopResults) + { + Assert.True(results.Contains(expectedTopResult), $"Expected result '{expectedTopResult}' for query '{searchTerm}'"); + } } - [RelevancyFact] - public async Task Mvc() + public static IEnumerable EnsureTopResultsData() { - var results = await SearchAsync("mvc"); - - Assert.NotEmpty(results); - Assert.Equal("microsoft.aspnet.mvc", results[0]); - Assert.Contains("microsoft.aspnetcore.mvc", results); + // The following were chosen arbitrarily without telemetry. + yield return new object[] { "Microsoft.Extensions", new[] { "microsoft.extensions.logging", "microsoft.extensions.configuration", "microsoft.extensions.dependencyinjection" } }; + yield return new object[] { "mvc", new[] { "microsoft.aspnet.mvc", "microsoft.aspnetcore.mvc" } }; } } } diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs index 180d71135..381c12f56 100644 --- a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs @@ -40,10 +40,25 @@ public NuGetSearchFunctionalTestBase(CommonFixture fixture, ITestOutputHelper te /// Whether prerelease results should be included. /// Whether semver2 results should be included. /// The package ids' that matches the query, lowercased. - protected async Task> SearchAsync(string query, bool includePrerelease = true, bool includeSemVer2 = true) + protected async Task> SearchAsync( + string query, + int? skip = 0, + int? take = 20, + bool includePrerelease = true, + bool includeSemVer2 = true) { var requestUri = $"/query?q={HttpUtility.UrlEncode(query)}"; + if (skip.HasValue) + { + requestUri += $"&skip={skip}"; + } + + if (take.HasValue) + { + requestUri += $"&skip={take}"; + } + if (includePrerelease) { requestUri += "&prerelease=true";