feat: Use of environment variables for controlling the active mutation (

#3122) * feat: restore use of env variables * feat: disable coverage analysis optimisation when no coverage is reported * fix: address PR comments * Restores F# test projects settings * fix: minor design change * Update src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs --------- Co-authored-by: Rouke Broersma <[email protected]>
stryker-mutator · Dec 20, 2024 · 035fce3 · 035fce3
1 parent 05c74d4
commit 035fce3
Show file tree

Hide file tree

Showing 10 changed files with 79 additions and 46 deletions.
diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnerPoolTests.cs b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnerPoolTests.cs
@@ -257,7 +257,7 @@ public void IdentifyNonCoveredMutants()
     }
 
     [TestMethod]
-    public void WorksWhenAllMutantsAreIgnoredPool()
+    public void ShouldIgnoreCoverageAnalysisWhenEmpty()
     {
         var options = new StrykerOptions
         {
@@ -270,7 +270,7 @@ public void WorksWhenAllMutantsAreIgnoredPool()
 
         var analyzer = new CoverageAnalyser(options);
         analyzer.DetermineTestCoverage(SourceProjectInfo, runner, new[] { Mutant, OtherMutant }, TestGuidsList.NoTest());
-        Mutant.CoveringTests.Count.ShouldBe(0);
+        Mutant.CoveringTests.IsEveryTest.ShouldBeTrue();
     }
 
     [TestMethod]
@@ -666,8 +666,9 @@ public void DetectUnexpectedCase()
 
         var mockVsTest = BuildVsTestRunnerPool(options, out var runner);
 
+        var testResult = BuildCoverageTestResult("T0", new[] { "0;", "" });
         var buildCase = BuildCase("unexpected", TestFrameworks.NUnit);
-        SetupMockCoverageRun(mockVsTest, new[] { new VsTest.TestResult(buildCase) { Outcome = VsTest.TestOutcome.Passed } });
+        SetupMockCoverageRun(mockVsTest, new[] { new VsTest.TestResult(buildCase) { Outcome = VsTest.TestOutcome.Passed }, testResult });
 
         var analyzer = new CoverageAnalyser(options);
         analyzer.DetermineTestCoverage(SourceProjectInfo, runner, new[] { Mutant, OtherMutant }, TestGuidsList.NoTest());

diff --git a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs
@@ -30,21 +30,33 @@ public void DetermineTestCoverage(IProjectAndTests project, ITestRunner runner,
         if (!_options.OptimizationMode.HasFlag(OptimizationModes.SkipUncoveredMutants) &&
             !_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest))
         {
-            foreach (var mutant in mutants)
-            {
-                mutant.CoveringTests = TestGuidsList.EveryTest();
-                mutant.AssessingTests = TestGuidsList.EveryTest();
-            }
+            AssumeAllTestsAreNeeded(mutants);
 
             return;
         }
 
         ParseCoverage(runner.CaptureCoverage(project), mutants, new TestGuidsList(resultFailingTests.GetGuids()));
     }
 
+    private static void AssumeAllTestsAreNeeded(IEnumerable<IMutant> mutants)
+    {
+        foreach (var mutant in mutants)
+        {
+            mutant.CoveringTests = TestGuidsList.EveryTest();
+            mutant.AssessingTests = TestGuidsList.EveryTest();
+        }
+    }
+
     private void ParseCoverage(IEnumerable<CoverageRunResult> coverage, IEnumerable<IMutant> mutantsToScan,
         TestGuidsList failedTests)
     {
+        if (coverage.Sum(c => c.MutationsCovered.Count) == 0)
+        {
+            _logger.LogError("It looks like the test coverage capture failed. Disable coverage based optimisation.");
+            AssumeAllTestsAreNeeded(mutantsToScan);
+            return;
+        }
+
         var dubiousTests = new HashSet<Guid>();
         var trustedTests = new HashSet<Guid>();
         var testIds = new HashSet<Guid>();

diff --git a/src/Stryker.Core/Stryker.Core/Initialisation/InitialisationProcess.cs b/src/Stryker.Core/Stryker.Core/Initialisation/InitialisationProcess.cs
@@ -158,7 +158,7 @@ private InitialTestRun InitialTest(IStrykerOptions options, SourceProjectInfo pr
         }
 
         throw new InputException(
-            "No test has been detected. Make sure your test project contains test and is compatible with VsTest." +
+            "No test result reported. Make sure your test project contains test and is compatible with VsTest." +
             string.Join(Environment.NewLine, projectInfo.Warnings));
     }
 

diff --git a/src/Stryker.Core/Stryker.Core/InjectedHelpers/MutantControl.cs b/src/Stryker.Core/Stryker.Core/InjectedHelpers/MutantControl.cs
@@ -48,14 +48,23 @@ public static bool IsActive(int id)
             if (ActiveMutant == ActiveMutantNotInitValue)
             {
 #pragma warning disable CS8600
-                string environmentVariable = System.Environment.GetEnvironmentVariable("ActiveMutation");
-                if (string.IsNullOrEmpty(environmentVariable))
+                // get the environment variable storing the mutation id
+                string environmentVariableName = System.Environment.GetEnvironmentVariable("STRYKER_MUTANT_ID_CONTROL_VAR");
+                if (environmentVariableName != null)
                 {
-                    ActiveMutant = -1;
+                    string environmentVariable = System.Environment.GetEnvironmentVariable(environmentVariableName);
+                    if (string.IsNullOrEmpty(environmentVariable))
+                    {
+                        ActiveMutant = -1;
+                    }
+                    else
+                    {
+                        ActiveMutant = int.Parse(environmentVariable);
+                    }
                 }
                 else
                 {
-                    ActiveMutant = int.Parse(environmentVariable);
+                    ActiveMutant = -1;
                 }
             }
 

diff --git a/src/Stryker.Core/Stryker.Core/Mutants/TestGuidsList.cs b/src/Stryker.Core/Stryker.Core/Mutants/TestGuidsList.cs
@@ -77,7 +77,7 @@ public TestGuidsList Excluding(TestGuidsList testsToSkip)
 
     public static TestGuidsList NoTest() => NoTestAtAll;
 
-    public IEnumerable<Guid> GetGuids() => _testGuids;
+    public IEnumerable<Guid> GetGuids() => _testGuids ?? [];
 
     public bool ContainsAny(ITestGuids other)
     {

diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs
@@ -33,7 +33,7 @@ public sealed class VsTestContextInformation : IDisposable
     private TestFrameworks _testFramework;
 
     /// <summary>
-    ///     Discovered tests (VsTest format)
+    /// Discovered tests (VsTest format)
     /// </summary>
     public IDictionary<Guid, VsTestDescription> VsTests { get; private set; }
 
@@ -43,7 +43,7 @@ public sealed class VsTestContextInformation : IDisposable
     public IDictionary<string, ISet<Guid>> TestsPerSource { get; } = new Dictionary<string, ISet<Guid>>();
 
     /// <summary>
-    ///     Tests (Stryker format)
+    /// Tests (Stryker format)
     /// </summary>
     public TestSet Tests { get; } = new();
 
@@ -93,19 +93,23 @@ public void Dispose()
     }
 
     /// <summary>
-    ///     Starts a new VsTest instance and returns a wrapper to control it.
+    /// Starts a new VsTest instance and returns a wrapper to control it.
     /// </summary>
     /// <param name="runnerId">Name of the instance to create (used in log files)</param>
+    /// <param name="controlVariable">name of the env variable storing the active mutation id</param>
     /// <returns>a <see cref="IVsTestConsoleWrapper" /> controlling the created instance.</returns>
-    public IVsTestConsoleWrapper BuildVsTestWrapper(string runnerId)
+    public IVsTestConsoleWrapper BuildVsTestWrapper(string runnerId, string controlVariable)
     {
-        var vsTestConsole = _wrapperBuilder(DetermineConsoleParameters(runnerId));
+        var env = DetermineConsoleParameters(runnerId);
+        // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes
+        env.EnvironmentVariables["DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX"]="2";
+        // we define a per runner control variable to prevent conflict
+        env.EnvironmentVariables["STRYKER_MUTANT_ID_CONTROL_VAR"] = controlVariable;
+        var vsTestConsole = _wrapperBuilder(env);
         try
         {
-            // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes
-            Environment.SetEnvironmentVariable("DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX", "2");
             vsTestConsole.StartSession();
-            vsTestConsole.InitializeExtensions(Enumerable.Empty<string>());
+            vsTestConsole.InitializeExtensions([]);
         }
         catch (Exception e)
         {
@@ -188,7 +192,7 @@ public bool AddTestSource(string source, string frameworkVersion = null, string
 
     private void DiscoverTestsInSources(string newSource, string frameworkVersion = null, string platform = null)
     {
-        var wrapper = BuildVsTestWrapper("TestDiscoverer");
+        var wrapper = BuildVsTestWrapper("TestDiscoverer", "NOT_NEEDED");
         var messages = new List<string>();
         var handler = new DiscoveryEventHandler(messages);
         var settings = GenerateRunSettingsForDiscovery(frameworkVersion, platform);
@@ -223,6 +227,13 @@ private void DiscoverTestsInSources(string newSource, string frameworkVersion =
         Tests.RegisterTests(VsTests.Values.Select(t => t.Description));
     }
 
+    internal void RegisterDiscoveredTest(VsTestDescription vsTestDescription)
+    {
+        VsTests[vsTestDescription.Id] = vsTestDescription;
+        Tests.RegisterTest(vsTestDescription.Description);
+        TestsPerSource[vsTestDescription.Case.Source].Add(vsTestDescription.Id);
+    }
+
     private void DetectTestFrameworks(ICollection<VsTestDescription> tests)
     {
         if (tests == null)
@@ -262,7 +273,7 @@ private string GenerateCoreSettings(int maxCpu, string frameworkVersion, string
         return
             $@"
 <MaxCpuCount>{Math.Max(0, maxCpu)}</MaxCpuCount>
-{frameworkConfig}{platformConfig}{testCaseFilter}  <InIsolation>true</InIsolation>
+{frameworkConfig}{platformConfig}{testCaseFilter} 
 <DisableAppDomain>true</DisableAppDomain>";
     }
 
@@ -309,4 +320,5 @@ public string GenerateRunSettings(int? timeout, bool forCoverage, Dictionary<int
 
         return runSettings;
     }
+
 }
diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestDescription.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestDescription.cs
@@ -3,13 +3,12 @@
 using System.Linq;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Stryker.Core.Mutants;
-using Stryker.Core.TestRunners;
 
 namespace Stryker.Core.TestRunners.VsTest;
 
 public sealed class VsTestDescription
 {
-    private readonly ICollection<TestResult> _initialResults = new List<TestResult>();
+    private readonly ICollection<TestResult> _initialResults = [];
     private int _subCases;
 
     public VsTestDescription(TestCase testCase)

diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs
@@ -35,6 +35,7 @@ public sealed class VsTestRunner : IDisposable
     private const int MaxAttempts = 3;
 
     private string RunnerId => $"Runner {_id}";
+    private string ControlVariableName => $"ACTIVE_MUTATION_{_id}";
 
     public VsTestRunner(VsTestContextInformation context,
         int id,
@@ -43,7 +44,7 @@ public VsTestRunner(VsTestContextInformation context,
         _context = context;
         _id = id;
         _logger = logger ?? ApplicationLogging.LoggerFactory.CreateLogger<VsTestRunner>();
-        _vsTestConsole = _context.BuildVsTestWrapper(RunnerId);
+        _vsTestConsole = _context.BuildVsTestWrapper(RunnerId, ControlVariableName);
     }
 
     public TestRunResult InitialTest(IProjectAndTests project)
@@ -60,8 +61,7 @@ public TestRunResult InitialTest(IProjectAndTests project)
             if (!_context.VsTests.ContainsKey(result.TestCase.Id))
             {
                 var vsTestDescription = new VsTestDescription(result.TestCase);
-                _context.VsTests[result.TestCase.Id] = vsTestDescription;
-                _context.Tests.RegisterTest(vsTestDescription.Description);
+                _context.RegisterDiscoveredTest(vsTestDescription);
                 _logger.LogWarning(
                     "{RunnerId}: Initial test run encounter an unexpected test case ({DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if you have doubts.",
                 RunnerId, result.TestCase.DisplayName);
@@ -84,7 +84,7 @@ public TestRunResult TestMultipleMutants(IProjectAndTests project, ITimeoutValue
         if (testCases?.Count == 0)
         {
             return new TestRunResult(_context.VsTests.Values, TestGuidsList.NoTest(), TestGuidsList.NoTest(),
-                TestGuidsList.NoTest(), "Mutants are not covered by any test!", Enumerable.Empty<string>(),
+                TestGuidsList.NoTest(), "Mutants are not covered by any test!", [],
                 TimeSpan.Zero);
         }
 
@@ -115,7 +115,7 @@ void HandleUpdate(IRunResults handler)
                 : new WrappedGuidsEnumeration(handlerTestResults.Select(t => t.TestCase.Id));
             var failedTest = new WrappedGuidsEnumeration(handlerTestResults
                 .Where(tr => tr.Outcome == TestOutcome.Failed)
-            .Select(t => t.TestCase.Id));
+                .Select(t => t.TestCase.Id));
             var timedOutTest = new WrappedGuidsEnumeration(handler.TestsInTimeout?.Select(t => t.Id));
             var remainingMutants = update?.Invoke(mutants, failedTest, tests, timedOutTest);
 
@@ -180,7 +180,7 @@ private TestRunResult BuildTestRunResult(IRunResults testResults, int expectedTe
         // ranTests is the list of test that have been executed. We detect the special case where all (existing and found) tests have been executed.
         // this is needed when the tests list is not stable (mutations can generate variation for theories) and also helps for performance
         // so we assume that if executed at least as much test as have been detected, it means all tests have been executed
-        // EXCEPT when no test have been found. Otherwise, an empty test project would transform non covered mutants to survivors.
+        // EXCEPT when no test have been found. Otherwise, an empty test project would transform non-covered mutants to survivors.
         var ranTests = compressAll && totalCountOfTests > 0 && ranTestsCount >= totalCountOfTests
             ? (ITestGuids)TestGuidsList.EveryTest()
             : new WrappedGuidsEnumeration(testCases);
@@ -240,16 +240,21 @@ public IRunResults RunCoverageSession(ITestGuids testsToRun, IProjectAndTests pr
         foreach (var source in projectAndTests.TestProjectsInfo.AnalyzerResults)
         {
             var testForSource = _context.TestsPerSource[source.GetAssemblyPath()];
-            var testsForAssembly = new TestGuidsList(tests.GetGuids()?.Where(testForSource.Contains));
+            var testsForAssembly = new TestGuidsList(tests.GetGuids().Where(testForSource.Contains));
             if (!tests.IsEveryTest && testsForAssembly.Count == 0)
             {
                 // skip empty assemblies
                 continue;
             }
             var runSettings = _context.GenerateRunSettings(timeOut, forCoverage, mutantTestsMap,
                 projectAndTests.HelperNamespace, source.TargetFramework, source.TargetPlatform());
-
             _logger.LogTrace("{RunnerId}: testing assembly {source}.", RunnerId, source);
+            var activeId = -1;
+            if (mutantTestsMap is { Count: 1 })
+            {
+                activeId = mutantTestsMap.Keys.First();
+            }
+            Environment.SetEnvironmentVariable(ControlVariableName, activeId.ToString());
             RunVsTest(tests, source.GetAssemblyPath(), runSettings, options, timeOut, runEventHandler);
 
             if (_currentSessionCancelled)
@@ -281,13 +286,15 @@ private void RunVsTest(ITestGuids tests, string source, string runSettings, Test
                 {
                     if (tests.IsEveryTest)
                     {
-                        _vsTestConsole.RunTestsWithCustomTestHost(new[] { source }, runSettings, options, eventHandler,
+                        _vsTestConsole.RunTestsWithCustomTestHost([source], runSettings, options, eventHandler,
                             strykerVsTestHostLauncher);
                     }
                     else
                     {
+                        var actualTestCases = tests.GetGuids().Select(t => _context.VsTests[t].Case).ToList();
+                        var testCases = actualTestCases;
                         _vsTestConsole.RunTestsWithCustomTestHost(
-                            tests.GetGuids().Select(t => _context.VsTests[t].Case),
+                            testCases,
                             runSettings, options, eventHandler, strykerVsTestHostLauncher);
                     }
                 });
@@ -371,7 +378,7 @@ private void PrepareVsTestConsole()
             }
         }
 
-        _vsTestConsole = _context.BuildVsTestWrapper($"{RunnerId}-{_instanceCount}");
+        _vsTestConsole = _context.BuildVsTestWrapper($"{RunnerId}-{_instanceCount}", ControlVariableName);
     }
 
     #region IDisposable Support

diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs
@@ -188,8 +188,7 @@ private bool ConvertSingleResult(TestResult testResult, ISet<Guid> seenTestCases
             // ==> we need it to use this test against every mutation
             _logger.LogDebug("VsTestRunner: No coverage data for {TestCase}.", testResult.TestCase.DisplayName);
             seenTestCases.Add(testDescription.Id);
-            coverageRunResult = new CoverageRunResult(testCaseId, CoverageConfidence.Dubious, Enumerable.Empty<int>(),
-                Enumerable.Empty<int>(), Enumerable.Empty<int>());
+            coverageRunResult = new CoverageRunResult(testCaseId, CoverageConfidence.Dubious, [], [], []);
         }
         else
         {

diff --git a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs
@@ -29,7 +29,6 @@ public class CoverageCollector : InProcDataCollection
         private IDataCollectionSink _dataSink;
         private bool _coverageOn;
         private int _activeMutation = -1;
-        private bool _reportFailure;
 
         private Action<string> _logger;
         private readonly IDictionary<string, int> _mutantTestedBy = new Dictionary<string, int>();
@@ -94,7 +93,6 @@ public void Initialize(IDataCollectionSink dataCollectionSink)
         {
             _dataSink = dataCollectionSink;
             _throwingListener = new ThrowingListener();
-            SetLogger(Console.WriteLine);
         }
 
         public void SetLogger(Action<string> logger) => _logger = logger;
@@ -276,11 +274,7 @@ private void PublishCoverageData(DataCollectionContext dataCollectionContext)
             {
                 // no test covered any mutations, so the controller was never properly initialized
                 _dataSink.SendData(dataCollectionContext, PropertyName, ";");
-                if (!_reportFailure)
-                {
-                    _dataSink.SendData(dataCollectionContext, CoverageLog, $"Did not find type {_controlClassName}. Mutated assembly may not be covered by any test.");
-                    _reportFailure = true;
-                }
+                _dataSink.SendData(dataCollectionContext, CoverageLog, $"Test {dataCollectionContext.TestCase.DisplayName} endend. No mutant covered so far.");
                 return;
             }