Support specific/custom test orders and a deterministic order in whic…

…h test classes are executed. (#1851) * Add support for executing tests in a specific order. The specific order is only supported within a class for now. * Add support for ordering test classes.
PomeloFoundation · Feb 27, 2024 · cdcaeac · cdcaeac
1 parent 5a3f9ca
commit cdcaeac
Show file tree

Hide file tree

Showing 10 changed files with 232 additions and 8 deletions.
diff --git a/test/EFCore.MySql.FunctionalTests/AssemblyInfo.cs b/test/EFCore.MySql.FunctionalTests/AssemblyInfo.cs
@@ -1,17 +1,15 @@
-// #define FIXED_TEST_ORDER
-
 using Xunit;
 
 //
 // Optional: Control the test execution order.
 //           This can be helpful for diffing etc.
 //
 
-#if FIXED_TEST_ORDER
+#if FIXED_TEST_ORDER || SPECIFIC_TEST_ORDER
 
 [assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true, MaxParallelThreads = 1)]
-[assembly: TestCollectionOrderer("Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit.MySqlTestCollectionOrderer", "Pomelo.EntityFrameworkCore.MySql.FunctionalTests")]
 [assembly: TestCaseOrderer("Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit.MySqlTestCaseOrderer", "Pomelo.EntityFrameworkCore.MySql.FunctionalTests")]
+[assembly: TestCollectionOrderer("Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit.MySqlTestCollectionOrderer", "Pomelo.EntityFrameworkCore.MySql.FunctionalTests")]
 
 #endif
 

diff --git a/test/EFCore.MySql.FunctionalTests/EFCore.MySql.FunctionalTests.csproj b/test/EFCore.MySql.FunctionalTests/EFCore.MySql.FunctionalTests.csproj
@@ -7,10 +7,16 @@
     <PreserveCompilationContext>true</PreserveCompilationContext>
     <DefaultItemExcludes>$(DefaultItemExcludes);*.trx</DefaultItemExcludes>
   </PropertyGroup>
+
+  <PropertyGroup>
+    <FixedTestOrder Condition="'$(FixedTestOrder)' == ''">false</FixedTestOrder>
+    <SpecificTestOrder Condition="'$(SpecificTestOrder)' == ''">false</SpecificTestOrder>
+  </PropertyGroup>
 
   <PropertyGroup>
     <MSBuildWarningsAsMessages>$(MSBuildWarningsAsMessages);$(NoWarn)</MSBuildWarningsAsMessages>
     <DefineConstants Condition="'$(FixedTestOrder)' == 'true'">$(DefineConstants);FIXED_TEST_ORDER</DefineConstants>
+    <DefineConstants Condition="'$(SpecificTestOrder)' == 'true'">$(DefineConstants);SPECIFIC_TEST_ORDER</DefineConstants>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/IMySqlTestClassOrderer.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/IMySqlTestClassOrderer.cs
@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+using Xunit.Abstractions;
+
+namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit;
+
+public interface IMySqlTestClassOrderer
+{
+    IEnumerable<ITestClass> OrderTestClasses(IEnumerable<ITestClass> testClasses);
+}
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlTestCaseOrderer.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlTestCaseOrderer.cs
@@ -1,15 +1,51 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 using Xunit.Abstractions;
 using Xunit.Sdk;
 
 namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit
 {
-    public class MySqlTestCaseOrderer : ITestCaseOrderer
+    public class MySqlTestCaseOrderer : ITestCaseOrderer, IMySqlTestClassOrderer
     {
+#if SPECIFIC_TEST_ORDER
+        private static readonly bool _isSpecificTestCaseOrderingEnabled = true;
+#else
+        private static readonly bool _isSpecificTestCaseOrderingEnabled = false;
+#endif
+
+        private static string[] _specificTestCaseDisplayNamesInOrder;
+        public static string[] SpecificTestCaseDisplayNamesInOrder
+            => _specificTestCaseDisplayNamesInOrder ??= _isSpecificTestCaseOrderingEnabled &&
+                                                        Path.GetFullPath(@"..\..\..\TestResults\SpecificTestOrder.txt") is var path &&
+                                                        File.Exists(path)
+                ? File.ReadLines(path)
+                    .Select(s => Regex.Match(s, @"^(?:\W*)([^\u200B]+)").Groups[1].Value)
+                    .Where(s => !string.IsNullOrEmpty(s))
+                    .Distinct()
+                    .ToArray()
+                : [];
+
+        private readonly Dictionary<string, int> _specificTestCaseDisplayNamesWithIndex;
+
+        public MySqlTestCaseOrderer()
+        {
+            _specificTestCaseDisplayNamesWithIndex = SpecificTestCaseDisplayNamesInOrder
+                .Select((s, i) => (s, i))
+                .ToDictionary(t => t.s, t => t.i);
+        }
+
         public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases)
             where TTestCase : ITestCase
-            => testCases.OrderBy(c => c.TestMethod.Method.Name, StringComparer.OrdinalIgnoreCase);
+            => testCases
+                .OrderBy(c => _specificTestCaseDisplayNamesWithIndex.GetValueOrDefault(c.DisplayName, int.MaxValue))
+                .ThenBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase)
+                .ThenBy(c => c.DisplayName, StringComparer.Ordinal)
+                .ThenBy(c => c.UniqueID);
+
+        public IEnumerable<ITestClass> OrderTestClasses(IEnumerable<ITestClass> testClasses)
+            => testClasses.OrderBy(c => c.Class.Name);
     }
 }
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlTestCollectionOrderer.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlTestCollectionOrderer.cs
@@ -9,6 +9,9 @@ namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit
     public class MySqlTestCollectionOrderer : ITestCollectionOrderer
     {
         public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollection> testCollections)
-            => testCollections.OrderBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase);
+            => testCollections
+                .OrderBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase)
+                .ThenBy(c => c.DisplayName, StringComparer.Ordinal)
+                .ThenBy(c => c.UniqueID);
     }
 }
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestAssemblyRunner.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestAssemblyRunner.cs
@@ -0,0 +1,34 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit;
+
+public class MySqlXunitTestAssemblyRunner : XunitTestAssemblyRunner
+{
+    public MySqlXunitTestAssemblyRunner(
+        ITestAssembly testAssembly,
+        IEnumerable<IXunitTestCase> testCases,
+        IMessageSink diagnosticMessageSink,
+        IMessageSink executionMessageSink,
+        ITestFrameworkExecutionOptions executionOptions)
+        : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions)
+    {
+    }
+
+    protected override Task<RunSummary> RunTestCollectionAsync(
+        IMessageBus messageBus,
+        ITestCollection testCollection,
+        IEnumerable<IXunitTestCase> testCases,
+        CancellationTokenSource cancellationTokenSource)
+        => new MySqlXunitTestCollectionRunner(
+            testCollection,
+            testCases,
+            DiagnosticMessageSink,
+            messageBus,
+            TestCaseOrderer,
+            new ExceptionAggregator(Aggregator),
+            cancellationTokenSource).RunAsync();
+}
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestCollectionRunner.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestCollectionRunner.cs
@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit;
+
+public class MySqlXunitTestCollectionRunner : XunitTestCollectionRunner
+{
+    public MySqlXunitTestCollectionRunner(
+        ITestCollection testCollection,
+        IEnumerable<IXunitTestCase> testCases,
+        IMessageSink diagnosticMessageSink,
+        IMessageBus messageBus,
+        ITestCaseOrderer testCaseOrderer,
+        ExceptionAggregator aggregator,
+        CancellationTokenSource cancellationTokenSource)
+        : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, cancellationTokenSource)
+    {
+    }
+
+    protected override async Task<RunSummary> RunTestClassesAsync()
+    {
+        var summary = new RunSummary();
+
+        var testsGroupedByClass = TestCases.GroupBy(tc => tc.TestMethod.TestClass, TestClassComparer.Instance);
+
+        // Explicitly order test classes.
+        if (TestCaseOrderer is IMySqlTestClassOrderer testClassOrderer)
+        {
+            var testClassesWithIndex = testClassOrderer
+                .OrderTestClasses(testsGroupedByClass.Select(g => g.Key))
+                .Select((s, i) => (s, i))
+                .ToDictionary(t => t.s, t => t.i);
+
+            testsGroupedByClass = testsGroupedByClass
+                .OrderBy(g => testClassesWithIndex.GetValueOrDefault(g.Key, int.MaxValue));
+        }
+
+        foreach (var testCasesByClass in testsGroupedByClass)
+        {
+            summary.Aggregate(await RunTestClassAsync(testCasesByClass.Key, (IReflectionTypeInfo)testCasesByClass.Key.Class, testCasesByClass));
+            if (CancellationTokenSource.IsCancellationRequested)
+                break;
+        }
+
+        return summary;
+    }
+}
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestFramework.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestFramework.cs
@@ -1,4 +1,6 @@
-using Xunit.Abstractions;
+using System;
+using System.Reflection;
+using Xunit.Abstractions;
 using Xunit.Sdk;
 
 namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit
@@ -11,5 +13,8 @@ public MySqlXunitTestFramework(IMessageSink messageSink) : base(messageSink)
 
         protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo)
             => new MySqlXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink);
+
+        protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName)
+            => new MySqlXunitTestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink);
     }
 }
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestFrameworkDiscoverer.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestFrameworkDiscoverer.cs
@@ -39,5 +39,56 @@ protected virtual IEnumerable<ITestCondition> GetTestConditions<TType>(ITypeInfo
             => type.GetCustomAttributes(typeof(TType))
                 .Select(attribute => (TType)Activator.CreateInstance(typeof(TType), attribute.GetConstructorArguments().ToArray()))
                 .Cast<ITestCondition>();
+
+        protected override bool FindTestsForMethod(
+            ITestMethod testMethod,
+            bool includeSourceInformation,
+            IMessageBus messageBus,
+            ITestFrameworkDiscoveryOptions discoveryOptions)
+            => base.FindTestsForMethod(
+                testMethod,
+                includeSourceInformation,
+                new FindTestsForMethodMessageBus(messageBus),
+                discoveryOptions);
+
+        private class FindTestsForMethodMessageBus : IMessageBus
+        {
+            private readonly IMessageBus _messageBus;
+            private static readonly HashSet<string> _testCaseDisplayNamesInOrder = MySqlTestCaseOrderer.SpecificTestCaseDisplayNamesInOrder.ToHashSet();
+
+            public FindTestsForMethodMessageBus(IMessageBus messageBus)
+                => _messageBus = messageBus;
+
+            public void Dispose()
+                => _messageBus.Dispose();
+
+            public bool QueueMessage(IMessageSinkMessage message)
+            {
+                // Intercept TestCaseDiscoveryMessage messages to filter specific test cases independent of the discoverer, so we can filter
+                // all test cases and not just the ones for which we specify our own discoverers.
+                if (_testCaseDisplayNamesInOrder.Count > 0 &&
+                    message is TestCaseDiscoveryMessage testCaseDiscoveryMessage)
+                {
+                    var displayName = GetFullyQualifiedDisplayName(testCaseDiscoveryMessage.TestCase);
+
+                    if (!_testCaseDisplayNamesInOrder.Contains(displayName))
+                    {
+                        return true;
+                    }
+                }
+
+                return _messageBus.QueueMessage(message);
+            }
+
+            private static string GetFullyQualifiedDisplayName(ITestCase testCase)
+            {
+                var className = testCase.TestMethod.TestClass.Class.Name;
+                var displayName = testCase.DisplayName;
+
+                return !displayName.StartsWith(className)
+                    ? $"{className}.{displayName}"
+                    : displayName;
+            }
+        }
     }
 }
diff --git a/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestFrameworkExecutor.cs b/test/EFCore.MySql.FunctionalTests/TestUtilities/Xunit/MySqlXunitTestFrameworkExecutor.cs
@@ -0,0 +1,31 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Pomelo.EntityFrameworkCore.MySql.FunctionalTests.TestUtilities.Xunit;
+
+public class MySqlXunitTestFrameworkExecutor : XunitTestFrameworkExecutor
+{
+    public MySqlXunitTestFrameworkExecutor(
+        AssemblyName assemblyName,
+        ISourceInformationProvider sourceInformationProvider,
+        IMessageSink diagnosticMessageSink)
+        : base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
+    {
+    }
+
+    protected override async void RunTestCases(
+        IEnumerable<IXunitTestCase> testCases,
+        IMessageSink executionMessageSink,
+        ITestFrameworkExecutionOptions executionOptions)
+    {
+        using var assemblyRunner = new MySqlXunitTestAssemblyRunner(
+            TestAssembly,
+            testCases,
+            DiagnosticMessageSink,
+            executionMessageSink,
+            executionOptions);
+        await assemblyRunner.RunAsync();
+    }
+}