Issue215: Export NUnit XML Test results

build.cake

@@ -13,7 +13,7 @@ var configuration = Argument("configuration", "Release");
 // SET PACKAGE VERSION
 //////////////////////////////////////////////////////////////////////
 
-var version = "3.11.2";
+var version = "3.12.0";
 var modifier = "";
 
 var dbgSuffix = configuration == "Debug" ? "-dbg" : "";

src/NUnit3TestAdapterInstall/source.extension.vsixmanifest

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <PackageManifest Version="2.0.0" xmlns="http://schemas.microsoft.com/developer/vsx-schema/2011" xmlns:d="http://schemas.microsoft.com/developer/vsx-schema-design/2011">
     <Metadata>
-        <Identity Id="NUnit.NUnit3TestAdapter" Version="3.11.2.0" Language="en-US" Publisher="Charlie Poole, Terje Sandstrom" />
+        <Identity Id="NUnit.NUnit3TestAdapter" Version="3.12.0.0" Language="en-US" Publisher="Charlie Poole, Terje Sandstrom" />
         <DisplayName>NUnit 3 Test Adapter</DisplayName>
         <Description xml:space="preserve">NUnit 3 adapter for running tests in Visual Studio. Works with NUnit 3.x.
 For running NUnit 2 tests, use the NUnit 2 adapter.</Description>

src/NUnitTestAdapter/AdapterSettings.cs

@@ -62,6 +62,8 @@ public interface IAdapterSettings
         string DefaultTestNamePattern { get; }
 
         VsTestCategoryType VsTestCategoryType { get; }
+        string TestOutputXml { get; }
+        bool UseTestOutputXml { get; }
 
         void Load(IDiscoveryContext context);
         void Load(string settingsXml);
@@ -135,7 +137,8 @@ public AdapterSettings(TestLogger logger)
         public string InternalTraceLevel { get; private set; }
 
         public string WorkDirectory { get; private set; }
-
+        public string TestOutputXml { get; private set; }
+        public bool UseTestOutputXml => !string.IsNullOrEmpty(TestOutputXml);
         public int DefaultTimeout { get; private set; }
 
         public int NumberOfTestWorkers { get; private set; }
@@ -182,7 +185,7 @@ public void Load(IDiscoveryContext context)
             if (context == null)
                 throw new ArgumentNullException(nameof(context), "Load called with null context");
 
-            Load(context?.RunSettings?.SettingsXml);
+            Load(context.RunSettings?.SettingsXml);
         }
 
         public void Load(string settingsXml)
@@ -209,7 +212,8 @@ public void Load(string settingsXml)
             DisableAppDomain = GetInnerTextAsBool(runConfiguration, nameof(DisableAppDomain), false);
             DisableParallelization = GetInnerTextAsBool(runConfiguration, nameof(DisableParallelization), false);
             DesignMode = GetInnerTextAsBool(runConfiguration, nameof(DesignMode), false);
-            CollectDataForEachTestSeparately = GetInnerTextAsBool(runConfiguration, nameof(CollectDataForEachTestSeparately), false);
+            CollectDataForEachTestSeparately =
+                GetInnerTextAsBool(runConfiguration, nameof(CollectDataForEachTestSeparately), false);
 
             TestProperties = new Dictionary<string, string>();
             foreach (XmlNode node in doc.SelectNodes("RunSettings/TestRunParameters/Parameter"))
@@ -221,14 +225,18 @@ public void Load(string settingsXml)
             }
 
             // NUnit settings
-            InternalTraceLevel = GetInnerTextWithLog(nunitNode, nameof(InternalTraceLevel), "Off", "Error", "Warning", "Info", "Verbose", "Debug");
+            InternalTraceLevel = GetInnerTextWithLog(nunitNode, nameof(InternalTraceLevel), "Off", "Error", "Warning",
+                "Info", "Verbose", "Debug");
             WorkDirectory = GetInnerTextWithLog(nunitNode, nameof(WorkDirectory));
             DefaultTimeout = GetInnerTextAsInt(nunitNode, nameof(DefaultTimeout), 0);
             NumberOfTestWorkers = GetInnerTextAsInt(nunitNode, nameof(NumberOfTestWorkers), -1);
             ShadowCopyFiles = GetInnerTextAsBool(nunitNode, nameof(ShadowCopyFiles), false);
             UseVsKeepEngineRunning = GetInnerTextAsBool(nunitNode, nameof(UseVsKeepEngineRunning), false);
             BasePath = GetInnerTextWithLog(nunitNode, nameof(BasePath));
             PrivateBinPath = GetInnerTextWithLog(nunitNode, nameof(PrivateBinPath));
+            var testOutput = GetInnerTextWithLog(nunitNode, nameof(TestOutputXml));
+            if (!string.IsNullOrEmpty(testOutput))
+                TestOutputXml = ValidatedPath(testOutput,nameof(TestOutputXml));
             RandomSeed = GetInnerTextAsNullableInt(nunitNode, nameof(RandomSeed));
             RandomSeedSpecified = RandomSeed.HasValue;
             if (!RandomSeedSpecified)
@@ -248,14 +256,15 @@ public void Load(string settingsXml)
                         VsTestCategoryType = VsTestCategoryType.MsTest;
                         break;
                     default:
-                        _logger.Warning($"Invalid value ({vsTestCategoryType}) for VsTestCategoryType, should be either NUnit or MsTest");
+                        _logger.Warning(
+                            $"Invalid value ({vsTestCategoryType}) for VsTestCategoryType, should be either NUnit or MsTest");
                         break;
                 }
 
 
 
 #if SUPPORT_REGISTRY_SETTINGS
-            // Legacy (CTP) registry settings override defaults
+// Legacy (CTP) registry settings override defaults
             var registry = RegistryCurrentUser.OpenRegistryCurrentUser(@"Software\nunit.org\VSAdapter");
             if (registry.Exist("ShadowCopy") && (registry.Read<int>("ShadowCopy") == 1))
                 ShadowCopyFiles = true;
@@ -266,17 +275,21 @@ public void Load(string settingsXml)
 #endif
 
 #if DEBUG && VERBOSE
-            // Force Verbosity to 1 under Debug
+// Force Verbosity to 1 under Debug
             Verbosity = 1;
 #endif
 
-            var inProcDataCollectorNode = doc.SelectSingleNode("RunSettings/InProcDataCollectionRunSettings/InProcDataCollectors");
-            InProcDataCollectorsAvailable = inProcDataCollectorNode != null && inProcDataCollectorNode.SelectNodes("InProcDataCollector").Count > 0;
+            var inProcDataCollectorNode =
+                doc.SelectSingleNode("RunSettings/InProcDataCollectionRunSettings/InProcDataCollectors");
+            InProcDataCollectorsAvailable = inProcDataCollectorNode != null &&
+                                            inProcDataCollectorNode.SelectNodes("InProcDataCollector").Count > 0;
 
             // Older versions of VS do not pass the CollectDataForEachTestSeparately configuration together with the LiveUnitTesting collector.
             // However, the adapter is expected to run in CollectDataForEachTestSeparately mode.
             // As a result for backwards compatibility reasons enable CollectDataForEachTestSeparately mode whenever LiveUnitTesting collector is being used.
-            var hasLiveUnitTestingDataCollector = inProcDataCollectorNode?.SelectSingleNode("InProcDataCollector[@uri='InProcDataCollector://Microsoft/LiveUnitTesting/1.0']") != null;
+            var hasLiveUnitTestingDataCollector =
+                inProcDataCollectorNode?.SelectSingleNode(
+                    "InProcDataCollector[@uri='InProcDataCollector://Microsoft/LiveUnitTesting/1.0']") != null;
 
             // TestPlatform can opt-in to run tests one at a time so that the InProcDataCollectors can collect the data for each one of them separately.
             // In that case, we need to ensure that tests do not run in parallel and the test started/test ended events are sent synchronously.
@@ -288,7 +301,8 @@ public void Load(string settingsXml)
                 {
                     if (!InProcDataCollectorsAvailable)
                     {
-                        _logger.Info("CollectDataForEachTestSeparately is set, which is used to make InProcDataCollectors collect data for each test separately. No InProcDataCollectors can be found, thus the tests will run slower unnecessarily.");
+                        _logger.Info(
+                            "CollectDataForEachTestSeparately is set, which is used to make InProcDataCollectors collect data for each test separately. No InProcDataCollectors can be found, thus the tests will run slower unnecessarily.");
                     }
                 }
             }
@@ -301,6 +315,20 @@ public void Load(string settingsXml)
 
             // Update NumberOfTestWorkers based on the DisableParallelization and NumberOfTestWorkers from runsettings.
             UpdateNumberOfTestWorkers();
+
+
+            string ValidatedPath(string path, string what)
+            {
+                try
+                {
+                    return Path.GetFullPath(path);
+                }
+                catch (Exception)
+                {
+                    _logger.Error($"   Invalid path for {what}: {path}");
+                    throw;
+                }
+            }
         }
 
         public void SaveRandomSeed(string dirname)

src/NUnitTestAdapter/NUnit3TestExecutor.cs

@@ -192,7 +192,7 @@ public void Initialize(IRunContext runContext, IFrameworkHandle frameworkHandle)
             TestLog.Debug("UseVsKeepEngineRunning: " + Settings.UseVsKeepEngineRunning);
 
             bool enableShutdown = true;
-            if (Settings.UseVsKeepEngineRunning )
+            if (Settings.UseVsKeepEngineRunning)
             {
                 enableShutdown = !runContext.KeepAlive;
             }
@@ -220,7 +220,7 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
             // No need to restore if the seed was in runsettings file
             if (!Settings.RandomSeedSpecified)
                 Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
-            DumpXml dumpXml=null;
+            DumpXml dumpXml = null;
             if (Settings.DumpXmlTestResults)
             {
                 dumpXml = new Dump.DumpXml(assemblyPath);
@@ -273,7 +273,8 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
                         {
                             try
                             {
-                                _activeRunner.Run(listener, filter);
+                                var results = _activeRunner.Run(listener, filter);
+                                GenerateTestOutput(results, assemblyPath);
                             }
                             catch (NullReferenceException)
                             {
@@ -329,6 +330,23 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
             }
         }
 
+
+        private void GenerateTestOutput(XmlNode testResults, string assemblypath)
+        {
+            if (!Settings.UseTestOutputXml)
+                return;
+#if NETCOREAPP1_0
+#else
+            var path = Path.Combine(Settings.TestOutputXml,$"{Path.GetFileNameWithoutExtension(assemblypath)}.xml");
+            var resultService = TestEngine.Services.GetService<IResultService>();
+            // Following null argument should work for nunit3 format. Empty array is OK as well.
+            // If you decide to handle other formats in the runsettings, it needs more work.
+            var resultWriter = resultService.GetResultWriter("nunit3", null);
+            resultWriter.WriteResultFile(testResults, path);
+            TestLog.Info($"   Test results written to {path}");
+#endif
+        }
+
         private NUnitTestFilterBuilder CreateTestFilterBuilder()
         {
 #if NETCOREAPP1_0
@@ -338,6 +356,6 @@ private NUnitTestFilterBuilder CreateTestFilterBuilder()
 #endif
         }
 
-        #endregion
+#endregion
     }
 }

src/NUnitTestAdapter/NUnitTestAdapter.cs

@@ -125,11 +125,11 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge
             TestLog = new TestLogger(messageLogger);
             Settings = new AdapterSettings(TestLog);
             TestLog.InitSettings(Settings);
-
             try
             {
                 Settings.Load(context);
                 TestLog.Verbosity = Settings.Verbosity;
+                CheckDirectories();
             }
             catch (Exception e)
             {
@@ -138,6 +138,16 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge
             }
         }
 
+        private void CheckDirectories()
+        {
+            bool ok = true;
+            if (Settings.UseTestOutputXml)
+            {
+                Directory.CreateDirectory(Settings.TestOutputXml);
+                TestLog.Info($"  Test Output folder checked/created : {Settings.TestOutputXml} ");
+            }
+        }
+
         protected ITestRunner GetRunnerFor(string assemblyName)
         {
             var package = CreateTestPackage(assemblyName);

src/NUnitTestAdapter/Properties/AssemblyInfo.cs

@@ -21,7 +21,7 @@
 [assembly: ComVisible(false)]
 
 [assembly: Guid("c0aad5e4-b486-49bc-b3e8-31e01be6fefe")]
-[assembly: AssemblyVersion("3.11.2.0")]
-[assembly: AssemblyFileVersion("3.11.2.0")]
+[assembly: AssemblyVersion("3.12.0.0")]
+[assembly: AssemblyFileVersion("3.12.0.0")]
 
 [assembly: InternalsVisibleTo("NUnit.VisualStudio.TestAdapter.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010029b97dea816272cc4ea44cf3cf666f8150d6dfe1274b6c2e6c4d54259b756888ec08ad6dd3ea0f540b30408b948ae5f39cf0c7b210abdec267b367ce1eccab97d5c6c02ee67090827ffd699544fa2add4849b45a1901eac08495bfee0397fba3946ff3912ce0b9a497818e418a77a0c8db4ca1780e7b6f6dd6911395fcc0faba")]

src/NUnitTestAdapter/TestConverter.cs

@@ -247,7 +247,7 @@ private static void FillResultFromOutputNodes(IEnumerable<XmlNode> outputNodes,
                     continue;
                 }
 
-                // Add stdErr/Progress messages from TestOutput element to vstest result
+                // Add stdErr/Progress messages from TestOutputXml element to vstest result
                 vsResult.Messages.Add(new TestResultMessage(
                     "error".Equals(stream, StringComparison.OrdinalIgnoreCase)
                         ? TestResultMessage.StandardErrorCategory

src/NUnitTestAdapterTests/AdapterSettingsTests.cs

@@ -51,29 +51,33 @@ public void NullContextThrowsException()
         public void DefaultSettings(string xml)
         {
             _settings.Load(xml);
-            Assert.That(_settings.MaxCpuCount, Is.EqualTo(-1));
-            Assert.Null(_settings.ResultsDirectory);
-            Assert.Null(_settings.TargetFrameworkVersion);
-            Assert.Null(_settings.TargetPlatform);
-            Assert.Null(_settings.TestAdapterPaths);
-            Assert.IsTrue(_settings.CollectSourceInformation);
-            Assert.IsEmpty(_settings.TestProperties);
-            Assert.Null(_settings.InternalTraceLevel);
-            Assert.Null(_settings.WorkDirectory);
-            Assert.That(_settings.NumberOfTestWorkers, Is.EqualTo(-1));
-            Assert.That(_settings.DefaultTimeout, Is.EqualTo(0));
-            Assert.That(_settings.Verbosity, Is.EqualTo(0));
-            Assert.False(_settings.ShadowCopyFiles);
-            Assert.False(_settings.UseVsKeepEngineRunning);
-            Assert.Null(_settings.BasePath);
-            Assert.Null(_settings.PrivateBinPath);
-            Assert.NotNull(_settings.RandomSeed);
-            Assert.False(_settings.SynchronousEvents);
-            Assert.Null(_settings.DomainUsage);
-            Assert.False(_settings.InProcDataCollectorsAvailable);
-            Assert.IsFalse(_settings.DisableAppDomain);
-            Assert.IsFalse(_settings.DisableParallelization);
-            Assert.IsFalse(_settings.DesignMode);
+            Assert.Multiple(() =>
+            {
+                Assert.That(_settings.MaxCpuCount, Is.EqualTo(-1));
+                Assert.Null(_settings.ResultsDirectory);
+                Assert.Null(_settings.TargetFrameworkVersion);
+                Assert.Null(_settings.TargetPlatform);
+                Assert.Null(_settings.TestAdapterPaths);
+                Assert.IsTrue(_settings.CollectSourceInformation);
+                Assert.IsEmpty(_settings.TestProperties);
+                Assert.Null(_settings.InternalTraceLevel);
+                Assert.Null(_settings.WorkDirectory);
+                Assert.That(_settings.NumberOfTestWorkers, Is.EqualTo(-1));
+                Assert.That(_settings.DefaultTimeout, Is.EqualTo(0));
+                Assert.That(_settings.Verbosity, Is.EqualTo(0));
+                Assert.False(_settings.ShadowCopyFiles);
+                Assert.False(_settings.UseVsKeepEngineRunning);
+                Assert.Null(_settings.BasePath);
+                Assert.Null(_settings.PrivateBinPath);
+                Assert.NotNull(_settings.RandomSeed);
+                Assert.False(_settings.SynchronousEvents);
+                Assert.Null(_settings.DomainUsage);
+                Assert.False(_settings.InProcDataCollectorsAvailable);
+                Assert.IsFalse(_settings.DisableAppDomain);
+                Assert.IsFalse(_settings.DisableParallelization);
+                Assert.IsFalse(_settings.DesignMode);
+                Assert.False(_settings.UseTestOutputXml);
+            });
         }
 
         [Test]
@@ -179,6 +183,15 @@ public void WorkDirectorySetting()
             Assert.That(_settings.WorkDirectory, Is.EqualTo("/my/work/dir"));
         }
 
+        [Test]
+        public void TestOutputSetting()
+        {
+            _settings.Load("<RunSettings><NUnit><TestOutputXml>/my/work/dir</TestOutputXml></NUnit></RunSettings>");
+            Assert.That(_settings.UseTestOutputXml);
+            Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\work\dir"));
+        }
+
+
         [Test]
         public void NumberOfTestWorkersSetting()
         {
@@ -255,7 +268,7 @@ public void RandomSeedSetting()
         public void DefaultTestNamePattern()
         {
             _settings.Load("<RunSettings><NUnit><DefaultTestNamePattern>{m}{a:1000}</DefaultTestNamePattern></NUnit></RunSettings>");
-            Assert.That(_settings.DefaultTestNamePattern,Is.EqualTo("{m}{a:1000}"));
+            Assert.That(_settings.DefaultTestNamePattern, Is.EqualTo("{m}{a:1000}"));
         }
 
         [Test]