diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnerPoolTests.cs b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnerPoolTests.cs index f0f0233cd..22ee4a81f 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnerPoolTests.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnerPoolTests.cs @@ -257,7 +257,7 @@ public void IdentifyNonCoveredMutants() } [TestMethod] - public void WorksWhenAllMutantsAreIgnoredPool() + public void ShouldIgnoreCoverageAnalysisWhenEmpty() { var options = new StrykerOptions { @@ -270,7 +270,7 @@ public void WorksWhenAllMutantsAreIgnoredPool() var analyzer = new CoverageAnalyser(options); analyzer.DetermineTestCoverage(SourceProjectInfo, runner, new[] { Mutant, OtherMutant }, TestGuidsList.NoTest()); - Mutant.CoveringTests.Count.ShouldBe(0); + Mutant.CoveringTests.IsEveryTest.ShouldBeTrue(); } [TestMethod] @@ -666,8 +666,9 @@ public void DetectUnexpectedCase() var mockVsTest = BuildVsTestRunnerPool(options, out var runner); + var testResult = BuildCoverageTestResult("T0", new[] { "0;", "" }); var buildCase = BuildCase("unexpected", TestFrameworks.NUnit); - SetupMockCoverageRun(mockVsTest, new[] { new VsTest.TestResult(buildCase) { Outcome = VsTest.TestOutcome.Passed } }); + SetupMockCoverageRun(mockVsTest, new[] { new VsTest.TestResult(buildCase) { Outcome = VsTest.TestOutcome.Passed }, testResult }); var analyzer = new CoverageAnalyser(options); analyzer.DetermineTestCoverage(SourceProjectInfo, runner, new[] { Mutant, OtherMutant }, TestGuidsList.NoTest()); diff --git a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs index 152170f27..e10cb2a42 100644 --- a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs +++ b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs @@ -30,11 +30,7 @@ public void DetermineTestCoverage(IProjectAndTests project, ITestRunner runner, if (!_options.OptimizationMode.HasFlag(OptimizationModes.SkipUncoveredMutants) && !_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest)) { - foreach (var mutant in mutants) - { - mutant.CoveringTests = TestGuidsList.EveryTest(); - mutant.AssessingTests = TestGuidsList.EveryTest(); - } + AssumeAllTestsAreNeeded(mutants); return; } @@ -42,9 +38,25 @@ public void DetermineTestCoverage(IProjectAndTests project, ITestRunner runner, ParseCoverage(runner.CaptureCoverage(project), mutants, new TestGuidsList(resultFailingTests.GetGuids())); } + private static void AssumeAllTestsAreNeeded(IEnumerable mutants) + { + foreach (var mutant in mutants) + { + mutant.CoveringTests = TestGuidsList.EveryTest(); + mutant.AssessingTests = TestGuidsList.EveryTest(); + } + } + private void ParseCoverage(IEnumerable coverage, IEnumerable mutantsToScan, TestGuidsList failedTests) { + if (coverage.Sum(c => c.MutationsCovered.Count) == 0) + { + _logger.LogError("It looks like the test coverage capture failed. Disable coverage based optimisation."); + AssumeAllTestsAreNeeded(mutantsToScan); + return; + } + var dubiousTests = new HashSet(); var trustedTests = new HashSet(); var testIds = new HashSet(); diff --git a/src/Stryker.Core/Stryker.Core/Initialisation/InitialisationProcess.cs b/src/Stryker.Core/Stryker.Core/Initialisation/InitialisationProcess.cs index 088cd0c9c..7a63c6a9f 100644 --- a/src/Stryker.Core/Stryker.Core/Initialisation/InitialisationProcess.cs +++ b/src/Stryker.Core/Stryker.Core/Initialisation/InitialisationProcess.cs @@ -158,7 +158,7 @@ private InitialTestRun InitialTest(IStrykerOptions options, SourceProjectInfo pr } throw new InputException( - "No test has been detected. Make sure your test project contains test and is compatible with VsTest." + + "No test result reported. Make sure your test project contains test and is compatible with VsTest." + string.Join(Environment.NewLine, projectInfo.Warnings)); } diff --git a/src/Stryker.Core/Stryker.Core/InjectedHelpers/MutantControl.cs b/src/Stryker.Core/Stryker.Core/InjectedHelpers/MutantControl.cs index 38eaa1b54..7c85175d7 100644 --- a/src/Stryker.Core/Stryker.Core/InjectedHelpers/MutantControl.cs +++ b/src/Stryker.Core/Stryker.Core/InjectedHelpers/MutantControl.cs @@ -48,14 +48,23 @@ public static bool IsActive(int id) if (ActiveMutant == ActiveMutantNotInitValue) { #pragma warning disable CS8600 - string environmentVariable = System.Environment.GetEnvironmentVariable("ActiveMutation"); - if (string.IsNullOrEmpty(environmentVariable)) + // get the environment variable storing the mutation id + string environmentVariableName = System.Environment.GetEnvironmentVariable("STRYKER_MUTANT_ID_CONTROL_VAR"); + if (environmentVariableName != null) { - ActiveMutant = -1; + string environmentVariable = System.Environment.GetEnvironmentVariable(environmentVariableName); + if (string.IsNullOrEmpty(environmentVariable)) + { + ActiveMutant = -1; + } + else + { + ActiveMutant = int.Parse(environmentVariable); + } } else { - ActiveMutant = int.Parse(environmentVariable); + ActiveMutant = -1; } } diff --git a/src/Stryker.Core/Stryker.Core/Mutants/TestGuidsList.cs b/src/Stryker.Core/Stryker.Core/Mutants/TestGuidsList.cs index 63aca9a96..4d4588da5 100644 --- a/src/Stryker.Core/Stryker.Core/Mutants/TestGuidsList.cs +++ b/src/Stryker.Core/Stryker.Core/Mutants/TestGuidsList.cs @@ -77,7 +77,7 @@ public TestGuidsList Excluding(TestGuidsList testsToSkip) public static TestGuidsList NoTest() => NoTestAtAll; - public IEnumerable GetGuids() => _testGuids; + public IEnumerable GetGuids() => _testGuids ?? []; public bool ContainsAny(ITestGuids other) { diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs index d6494261a..c02b53253 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestContextInformation.cs @@ -33,7 +33,7 @@ public sealed class VsTestContextInformation : IDisposable private TestFrameworks _testFramework; /// - /// Discovered tests (VsTest format) + /// Discovered tests (VsTest format) /// public IDictionary VsTests { get; private set; } @@ -43,7 +43,7 @@ public sealed class VsTestContextInformation : IDisposable public IDictionary> TestsPerSource { get; } = new Dictionary>(); /// - /// Tests (Stryker format) + /// Tests (Stryker format) /// public TestSet Tests { get; } = new(); @@ -93,19 +93,23 @@ public void Dispose() } /// - /// Starts a new VsTest instance and returns a wrapper to control it. + /// Starts a new VsTest instance and returns a wrapper to control it. /// /// Name of the instance to create (used in log files) + /// name of the env variable storing the active mutation id /// a controlling the created instance. - public IVsTestConsoleWrapper BuildVsTestWrapper(string runnerId) + public IVsTestConsoleWrapper BuildVsTestWrapper(string runnerId, string controlVariable) { - var vsTestConsole = _wrapperBuilder(DetermineConsoleParameters(runnerId)); + var env = DetermineConsoleParameters(runnerId); + // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes + env.EnvironmentVariables["DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX"]="2"; + // we define a per runner control variable to prevent conflict + env.EnvironmentVariables["STRYKER_MUTANT_ID_CONTROL_VAR"] = controlVariable; + var vsTestConsole = _wrapperBuilder(env); try { - // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes - Environment.SetEnvironmentVariable("DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX", "2"); vsTestConsole.StartSession(); - vsTestConsole.InitializeExtensions(Enumerable.Empty()); + vsTestConsole.InitializeExtensions([]); } catch (Exception e) { @@ -188,7 +192,7 @@ public bool AddTestSource(string source, string frameworkVersion = null, string private void DiscoverTestsInSources(string newSource, string frameworkVersion = null, string platform = null) { - var wrapper = BuildVsTestWrapper("TestDiscoverer"); + var wrapper = BuildVsTestWrapper("TestDiscoverer", "NOT_NEEDED"); var messages = new List(); var handler = new DiscoveryEventHandler(messages); var settings = GenerateRunSettingsForDiscovery(frameworkVersion, platform); @@ -223,6 +227,13 @@ private void DiscoverTestsInSources(string newSource, string frameworkVersion = Tests.RegisterTests(VsTests.Values.Select(t => t.Description)); } + internal void RegisterDiscoveredTest(VsTestDescription vsTestDescription) + { + VsTests[vsTestDescription.Id] = vsTestDescription; + Tests.RegisterTest(vsTestDescription.Description); + TestsPerSource[vsTestDescription.Case.Source].Add(vsTestDescription.Id); + } + private void DetectTestFrameworks(ICollection tests) { if (tests == null) @@ -262,7 +273,7 @@ private string GenerateCoreSettings(int maxCpu, string frameworkVersion, string return $@" {Math.Max(0, maxCpu)} -{frameworkConfig}{platformConfig}{testCaseFilter} true +{frameworkConfig}{platformConfig}{testCaseFilter} true"; } @@ -309,4 +320,5 @@ public string GenerateRunSettings(int? timeout, bool forCoverage, Dictionary _initialResults = new List(); + private readonly ICollection _initialResults = []; private int _subCases; public VsTestDescription(TestCase testCase) diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index 0c5d275aa..de6e99665 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -35,6 +35,7 @@ public sealed class VsTestRunner : IDisposable private const int MaxAttempts = 3; private string RunnerId => $"Runner {_id}"; + private string ControlVariableName => $"ACTIVE_MUTATION_{_id}"; public VsTestRunner(VsTestContextInformation context, int id, @@ -43,7 +44,7 @@ public VsTestRunner(VsTestContextInformation context, _context = context; _id = id; _logger = logger ?? ApplicationLogging.LoggerFactory.CreateLogger(); - _vsTestConsole = _context.BuildVsTestWrapper(RunnerId); + _vsTestConsole = _context.BuildVsTestWrapper(RunnerId, ControlVariableName); } public TestRunResult InitialTest(IProjectAndTests project) @@ -60,8 +61,7 @@ public TestRunResult InitialTest(IProjectAndTests project) if (!_context.VsTests.ContainsKey(result.TestCase.Id)) { var vsTestDescription = new VsTestDescription(result.TestCase); - _context.VsTests[result.TestCase.Id] = vsTestDescription; - _context.Tests.RegisterTest(vsTestDescription.Description); + _context.RegisterDiscoveredTest(vsTestDescription); _logger.LogWarning( "{RunnerId}: Initial test run encounter an unexpected test case ({DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if you have doubts.", RunnerId, result.TestCase.DisplayName); @@ -84,7 +84,7 @@ public TestRunResult TestMultipleMutants(IProjectAndTests project, ITimeoutValue if (testCases?.Count == 0) { return new TestRunResult(_context.VsTests.Values, TestGuidsList.NoTest(), TestGuidsList.NoTest(), - TestGuidsList.NoTest(), "Mutants are not covered by any test!", Enumerable.Empty(), + TestGuidsList.NoTest(), "Mutants are not covered by any test!", [], TimeSpan.Zero); } @@ -115,7 +115,7 @@ void HandleUpdate(IRunResults handler) : new WrappedGuidsEnumeration(handlerTestResults.Select(t => t.TestCase.Id)); var failedTest = new WrappedGuidsEnumeration(handlerTestResults .Where(tr => tr.Outcome == TestOutcome.Failed) - .Select(t => t.TestCase.Id)); + .Select(t => t.TestCase.Id)); var timedOutTest = new WrappedGuidsEnumeration(handler.TestsInTimeout?.Select(t => t.Id)); var remainingMutants = update?.Invoke(mutants, failedTest, tests, timedOutTest); @@ -180,7 +180,7 @@ private TestRunResult BuildTestRunResult(IRunResults testResults, int expectedTe // ranTests is the list of test that have been executed. We detect the special case where all (existing and found) tests have been executed. // this is needed when the tests list is not stable (mutations can generate variation for theories) and also helps for performance // so we assume that if executed at least as much test as have been detected, it means all tests have been executed - // EXCEPT when no test have been found. Otherwise, an empty test project would transform non covered mutants to survivors. + // EXCEPT when no test have been found. Otherwise, an empty test project would transform non-covered mutants to survivors. var ranTests = compressAll && totalCountOfTests > 0 && ranTestsCount >= totalCountOfTests ? (ITestGuids)TestGuidsList.EveryTest() : new WrappedGuidsEnumeration(testCases); @@ -240,7 +240,7 @@ public IRunResults RunCoverageSession(ITestGuids testsToRun, IProjectAndTests pr foreach (var source in projectAndTests.TestProjectsInfo.AnalyzerResults) { var testForSource = _context.TestsPerSource[source.GetAssemblyPath()]; - var testsForAssembly = new TestGuidsList(tests.GetGuids()?.Where(testForSource.Contains)); + var testsForAssembly = new TestGuidsList(tests.GetGuids().Where(testForSource.Contains)); if (!tests.IsEveryTest && testsForAssembly.Count == 0) { // skip empty assemblies @@ -248,8 +248,13 @@ public IRunResults RunCoverageSession(ITestGuids testsToRun, IProjectAndTests pr } var runSettings = _context.GenerateRunSettings(timeOut, forCoverage, mutantTestsMap, projectAndTests.HelperNamespace, source.TargetFramework, source.TargetPlatform()); - _logger.LogTrace("{RunnerId}: testing assembly {source}.", RunnerId, source); + var activeId = -1; + if (mutantTestsMap is { Count: 1 }) + { + activeId = mutantTestsMap.Keys.First(); + } + Environment.SetEnvironmentVariable(ControlVariableName, activeId.ToString()); RunVsTest(tests, source.GetAssemblyPath(), runSettings, options, timeOut, runEventHandler); if (_currentSessionCancelled) @@ -281,13 +286,15 @@ private void RunVsTest(ITestGuids tests, string source, string runSettings, Test { if (tests.IsEveryTest) { - _vsTestConsole.RunTestsWithCustomTestHost(new[] { source }, runSettings, options, eventHandler, + _vsTestConsole.RunTestsWithCustomTestHost([source], runSettings, options, eventHandler, strykerVsTestHostLauncher); } else { + var actualTestCases = tests.GetGuids().Select(t => _context.VsTests[t].Case).ToList(); + var testCases = actualTestCases; _vsTestConsole.RunTestsWithCustomTestHost( - tests.GetGuids().Select(t => _context.VsTests[t].Case), + testCases, runSettings, options, eventHandler, strykerVsTestHostLauncher); } }); @@ -371,7 +378,7 @@ private void PrepareVsTestConsole() } } - _vsTestConsole = _context.BuildVsTestWrapper($"{RunnerId}-{_instanceCount}"); + _vsTestConsole = _context.BuildVsTestWrapper($"{RunnerId}-{_instanceCount}", ControlVariableName); } #region IDisposable Support diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs index ab307e796..4ede47d9a 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs @@ -188,8 +188,7 @@ private bool ConvertSingleResult(TestResult testResult, ISet seenTestCases // ==> we need it to use this test against every mutation _logger.LogDebug("VsTestRunner: No coverage data for {TestCase}.", testResult.TestCase.DisplayName); seenTestCases.Add(testDescription.Id); - coverageRunResult = new CoverageRunResult(testCaseId, CoverageConfidence.Dubious, Enumerable.Empty(), - Enumerable.Empty(), Enumerable.Empty()); + coverageRunResult = new CoverageRunResult(testCaseId, CoverageConfidence.Dubious, [], [], []); } else { diff --git a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs index c840b323a..583ff1051 100644 --- a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs +++ b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs @@ -29,7 +29,6 @@ public class CoverageCollector : InProcDataCollection private IDataCollectionSink _dataSink; private bool _coverageOn; private int _activeMutation = -1; - private bool _reportFailure; private Action _logger; private readonly IDictionary _mutantTestedBy = new Dictionary(); @@ -94,7 +93,6 @@ public void Initialize(IDataCollectionSink dataCollectionSink) { _dataSink = dataCollectionSink; _throwingListener = new ThrowingListener(); - SetLogger(Console.WriteLine); } public void SetLogger(Action logger) => _logger = logger; @@ -276,11 +274,7 @@ private void PublishCoverageData(DataCollectionContext dataCollectionContext) { // no test covered any mutations, so the controller was never properly initialized _dataSink.SendData(dataCollectionContext, PropertyName, ";"); - if (!_reportFailure) - { - _dataSink.SendData(dataCollectionContext, CoverageLog, $"Did not find type {_controlClassName}. Mutated assembly may not be covered by any test."); - _reportFailure = true; - } + _dataSink.SendData(dataCollectionContext, CoverageLog, $"Test {dataCollectionContext.TestCase.DisplayName} endend. No mutant covered so far."); return; }