Issue215: Export NUnit XML Test results

src/NUnitTestAdapter/AdapterSettings.cs

@@ -1,5 +1,5 @@
 // ***********************************************************************
-// Copyright (c) 2014-2017 Charlie Poole, Terje Sandstrom
+// Copyright (c) 2014-2019 Charlie Poole, Terje Sandstrom
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -62,6 +62,23 @@ public interface IAdapterSettings
         string DefaultTestNamePattern { get; }
 
         VsTestCategoryType VsTestCategoryType { get; }
+        string TestOutputXml { get; }
+        bool UseTestOutputXml { get; }
+
+        /// <summary>
+        /// True if test run is triggered in an IDE/Editor context.
+        /// </summary>
+        bool DesignMode { get; }
+
+        /// <summary>
+        /// If true, an adapter shouldn't create appdomains to run tests
+        /// </summary>
+        bool DisableAppDomain { get; }
+
+        /// <summary>
+        /// If true, an adapter should disable any test case parallelization
+        /// </summary>
+        bool DisableParallelization { get; }
 
         void Load(IDiscoveryContext context);
         void Load(string settingsXml);
@@ -135,7 +152,8 @@ public AdapterSettings(TestLogger logger)
         public string InternalTraceLevel { get; private set; }
 
         public string WorkDirectory { get; private set; }
-
+        public string TestOutputXml { get; private set; }
+        public bool UseTestOutputXml => !string.IsNullOrEmpty(TestOutputXml);
         public int DefaultTimeout { get; private set; }
 
         public int NumberOfTestWorkers { get; private set; }
@@ -182,7 +200,7 @@ public void Load(IDiscoveryContext context)
             if (context == null)
                 throw new ArgumentNullException(nameof(context), "Load called with null context");
 
-            Load(context?.RunSettings?.SettingsXml);
+            Load(context.RunSettings?.SettingsXml);
         }
 
         public void Load(string settingsXml)
@@ -209,7 +227,8 @@ public void Load(string settingsXml)
             DisableAppDomain = GetInnerTextAsBool(runConfiguration, nameof(DisableAppDomain), false);
             DisableParallelization = GetInnerTextAsBool(runConfiguration, nameof(DisableParallelization), false);
             DesignMode = GetInnerTextAsBool(runConfiguration, nameof(DesignMode), false);
-            CollectDataForEachTestSeparately = GetInnerTextAsBool(runConfiguration, nameof(CollectDataForEachTestSeparately), false);
+            CollectDataForEachTestSeparately =
+                GetInnerTextAsBool(runConfiguration, nameof(CollectDataForEachTestSeparately), false);
 
             TestProperties = new Dictionary<string, string>();
             foreach (XmlNode node in doc.SelectNodes("RunSettings/TestRunParameters/Parameter"))
@@ -221,14 +240,18 @@ public void Load(string settingsXml)
             }
 
             // NUnit settings
-            InternalTraceLevel = GetInnerTextWithLog(nunitNode, nameof(InternalTraceLevel), "Off", "Error", "Warning", "Info", "Verbose", "Debug");
+            InternalTraceLevel = GetInnerTextWithLog(nunitNode, nameof(InternalTraceLevel), "Off", "Error", "Warning",
+                "Info", "Verbose", "Debug");
             WorkDirectory = GetInnerTextWithLog(nunitNode, nameof(WorkDirectory));
             DefaultTimeout = GetInnerTextAsInt(nunitNode, nameof(DefaultTimeout), 0);
             NumberOfTestWorkers = GetInnerTextAsInt(nunitNode, nameof(NumberOfTestWorkers), -1);
             ShadowCopyFiles = GetInnerTextAsBool(nunitNode, nameof(ShadowCopyFiles), false);
             UseVsKeepEngineRunning = GetInnerTextAsBool(nunitNode, nameof(UseVsKeepEngineRunning), false);
             BasePath = GetInnerTextWithLog(nunitNode, nameof(BasePath));
             PrivateBinPath = GetInnerTextWithLog(nunitNode, nameof(PrivateBinPath));
+            var testOutput = GetInnerTextWithLog(nunitNode, nameof(TestOutputXml));
+            if (!string.IsNullOrEmpty(testOutput))
+                TestOutputXml = ValidatedPath(testOutput, nameof(TestOutputXml));
             RandomSeed = GetInnerTextAsNullableInt(nunitNode, nameof(RandomSeed));
             RandomSeedSpecified = RandomSeed.HasValue;
             if (!RandomSeedSpecified)
@@ -248,7 +271,8 @@ public void Load(string settingsXml)
                         VsTestCategoryType = VsTestCategoryType.MsTest;
                         break;
                     default:
-                        _logger.Warning($"Invalid value ({vsTestCategoryType}) for VsTestCategoryType, should be either NUnit or MsTest");
+                        _logger.Warning(
+                            $"Invalid value ({vsTestCategoryType}) for VsTestCategoryType, should be either NUnit or MsTest");
                         break;
                 }
 
@@ -270,13 +294,17 @@ public void Load(string settingsXml)
             Verbosity = 1;
 #endif
 
-            var inProcDataCollectorNode = doc.SelectSingleNode("RunSettings/InProcDataCollectionRunSettings/InProcDataCollectors");
-            InProcDataCollectorsAvailable = inProcDataCollectorNode != null && inProcDataCollectorNode.SelectNodes("InProcDataCollector").Count > 0;
+            var inProcDataCollectorNode =
+                doc.SelectSingleNode("RunSettings/InProcDataCollectionRunSettings/InProcDataCollectors");
+            InProcDataCollectorsAvailable = inProcDataCollectorNode != null &&
+                                            inProcDataCollectorNode.SelectNodes("InProcDataCollector").Count > 0;
 
             // Older versions of VS do not pass the CollectDataForEachTestSeparately configuration together with the LiveUnitTesting collector.
             // However, the adapter is expected to run in CollectDataForEachTestSeparately mode.
             // As a result for backwards compatibility reasons enable CollectDataForEachTestSeparately mode whenever LiveUnitTesting collector is being used.
-            var hasLiveUnitTestingDataCollector = inProcDataCollectorNode?.SelectSingleNode("InProcDataCollector[@uri='InProcDataCollector://Microsoft/LiveUnitTesting/1.0']") != null;
+            var hasLiveUnitTestingDataCollector =
+                inProcDataCollectorNode?.SelectSingleNode(
+                    "InProcDataCollector[@uri='InProcDataCollector://Microsoft/LiveUnitTesting/1.0']") != null;
 
             // TestPlatform can opt-in to run tests one at a time so that the InProcDataCollectors can collect the data for each one of them separately.
             // In that case, we need to ensure that tests do not run in parallel and the test started/test ended events are sent synchronously.
@@ -288,7 +316,8 @@ public void Load(string settingsXml)
                 {
                     if (!InProcDataCollectorsAvailable)
                     {
-                        _logger.Info("CollectDataForEachTestSeparately is set, which is used to make InProcDataCollectors collect data for each test separately. No InProcDataCollectors can be found, thus the tests will run slower unnecessarily.");
+                        _logger.Info(
+                            "CollectDataForEachTestSeparately is set, which is used to make InProcDataCollectors collect data for each test separately. No InProcDataCollectors can be found, thus the tests will run slower unnecessarily.");
                     }
                 }
             }
@@ -301,6 +330,24 @@ public void Load(string settingsXml)
 
             // Update NumberOfTestWorkers based on the DisableParallelization and NumberOfTestWorkers from runsettings.
             UpdateNumberOfTestWorkers();
+
+
+            string ValidatedPath(string path, string purpose)
+            {
+                try
+                {
+                    if (string.IsNullOrEmpty(WorkDirectory))
+                        return Path.GetFullPath(path);
+                    if (Path.IsPathRooted(path))
+                        return Path.GetFullPath(path);
+                    return Path.GetFullPath(Path.Combine(WorkDirectory, path));
+                }
+                catch (Exception)
+                {
+                    _logger.Error($"   Invalid path for {purpose}: {path}");
+                    throw;
+                }
+            }
         }
 
         public void SaveRandomSeed(string dirname)

src/NUnitTestAdapter/NUnit3TestExecutor.cs

@@ -1,5 +1,5 @@
 // ***********************************************************************
-// Copyright (c) 2011-2018 Charlie Poole, Terje Sandstrom
+// Copyright (c) 2011-2019 Charlie Poole, Terje Sandstrom
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -192,7 +192,7 @@ public void Initialize(IRunContext runContext, IFrameworkHandle frameworkHandle)
             TestLog.Debug("UseVsKeepEngineRunning: " + Settings.UseVsKeepEngineRunning);
 
             bool enableShutdown = true;
-            if (Settings.UseVsKeepEngineRunning )
+            if (Settings.UseVsKeepEngineRunning)
             {
                 enableShutdown = !runContext.KeepAlive;
             }
@@ -220,7 +220,7 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
             // No need to restore if the seed was in runsettings file
             if (!Settings.RandomSeedSpecified)
                 Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
-            DumpXml dumpXml=null;
+            DumpXml dumpXml = null;
             if (Settings.DumpXmlTestResults)
             {
                 dumpXml = new Dump.DumpXml(assemblyPath);
@@ -273,7 +273,8 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
                         {
                             try
                             {
-                                _activeRunner.Run(listener, filter);
+                                var results = _activeRunner.Run(listener, filter);
+                                GenerateTestOutput(results, assemblyPath);
                             }
                             catch (NullReferenceException)
                             {
@@ -329,6 +330,23 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
             }
         }
 
+
+        private void GenerateTestOutput(XmlNode testResults, string assemblyPath)
+        {
+            if (!Settings.UseTestOutputXml)
+                return;
+#if NETCOREAPP1_0
+#else
+            var path = Path.Combine(Settings.TestOutputXml, Path.ChangeExtension(Path.GetFileName(assemblyPath),".xml"));
+            var resultService = TestEngine.Services.GetService<IResultService>();
+            // Following null argument should work for nunit3 format. Empty array is OK as well.
+            // If you decide to handle other formats in the runsettings, it needs more work.
+            var resultWriter = resultService.GetResultWriter("nunit3", null);
+            resultWriter.WriteResultFile(testResults, path);
+            TestLog.Info($"   Test results written to {path}");
+#endif
+        }
+
         private NUnitTestFilterBuilder CreateTestFilterBuilder()
         {
 #if NETCOREAPP1_0

src/NUnitTestAdapter/NUnitTestAdapter.cs

@@ -1,5 +1,5 @@
 // ***********************************************************************
-// Copyright (c) 2011-2015 Charlie Poole, Terje Sandstrom
+// Copyright (c) 2011-2019 Charlie Poole, Terje Sandstrom
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -125,11 +125,11 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge
             TestLog = new TestLogger(messageLogger);
             Settings = new AdapterSettings(TestLog);
             TestLog.InitSettings(Settings);
-
             try
             {
                 Settings.Load(context);
                 TestLog.Verbosity = Settings.Verbosity;
+                CheckDirectories();
             }
             catch (Exception e)
             {
@@ -138,6 +138,15 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge
             }
         }
 
+        private void CheckDirectories()
+        {
+            if (Settings.UseTestOutputXml)
+            {
+                Directory.CreateDirectory(Settings.TestOutputXml);
+                TestLog.Info($"  Test Output folder checked/created : {Settings.TestOutputXml} ");
+            }
+        }
+
         protected ITestRunner GetRunnerFor(string assemblyName)
         {
             var package = CreateTestPackage(assemblyName);

src/NUnitTestAdapter/TestConverter.cs

@@ -247,7 +247,7 @@ private static void FillResultFromOutputNodes(IEnumerable<XmlNode> outputNodes,
                     continue;
                 }
 
-                // Add stdErr/Progress messages from TestOutput element to vstest result
+                // Add stdErr/Progress messages from TestOutputXml element to vstest result
                 vsResult.Messages.Add(new TestResultMessage(
                     "error".Equals(stream, StringComparison.OrdinalIgnoreCase)
                         ? TestResultMessage.StandardErrorCategory

src/NUnitTestAdapterTests/AdapterSettingsTests.cs

@@ -1,5 +1,5 @@
 // ***********************************************************************
-// Copyright (c) 2011-2017 Charlie Poole, Terje Sandstrom
+// Copyright (c) 2011-2019 Charlie Poole, Terje Sandstrom
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -21,6 +21,7 @@
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 // ***********************************************************************
 
+using System.IO;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
 using NUnit.Framework;
 using NUnit.VisualStudio.TestAdapter.Tests.Fakes;
@@ -29,14 +30,14 @@ namespace NUnit.VisualStudio.TestAdapter.Tests
 {
     public class AdapterSettingsTests
     {
-        private AdapterSettings _settings;
+        private IAdapterSettings _settings;
 
         [SetUp]
         public void SetUp()
         {
-            var testlogger = new TestLogger(new MessageLoggerStub());
-            _settings = new AdapterSettings(testlogger);
-            testlogger.InitSettings(_settings);
+            var testLogger = new TestLogger(new MessageLoggerStub());
+            _settings = new AdapterSettings(testLogger);
+            testLogger.InitSettings(_settings);
         }
 
         [Test]
@@ -51,29 +52,33 @@ public void NullContextThrowsException()
         public void DefaultSettings(string xml)
         {
             _settings.Load(xml);
-            Assert.That(_settings.MaxCpuCount, Is.EqualTo(-1));
-            Assert.Null(_settings.ResultsDirectory);
-            Assert.Null(_settings.TargetFrameworkVersion);
-            Assert.Null(_settings.TargetPlatform);
-            Assert.Null(_settings.TestAdapterPaths);
-            Assert.IsTrue(_settings.CollectSourceInformation);
-            Assert.IsEmpty(_settings.TestProperties);
-            Assert.Null(_settings.InternalTraceLevel);
-            Assert.Null(_settings.WorkDirectory);
-            Assert.That(_settings.NumberOfTestWorkers, Is.EqualTo(-1));
-            Assert.That(_settings.DefaultTimeout, Is.EqualTo(0));
-            Assert.That(_settings.Verbosity, Is.EqualTo(0));
-            Assert.False(_settings.ShadowCopyFiles);
-            Assert.False(_settings.UseVsKeepEngineRunning);
-            Assert.Null(_settings.BasePath);
-            Assert.Null(_settings.PrivateBinPath);
-            Assert.NotNull(_settings.RandomSeed);
-            Assert.False(_settings.SynchronousEvents);
-            Assert.Null(_settings.DomainUsage);
-            Assert.False(_settings.InProcDataCollectorsAvailable);
-            Assert.IsFalse(_settings.DisableAppDomain);
-            Assert.IsFalse(_settings.DisableParallelization);
-            Assert.IsFalse(_settings.DesignMode);
+            Assert.Multiple(() =>
+            {
+                Assert.That(_settings.MaxCpuCount, Is.EqualTo(-1));
+                Assert.Null(_settings.ResultsDirectory);
+                Assert.Null(_settings.TargetFrameworkVersion);
+                Assert.Null(_settings.TargetPlatform);
+                Assert.Null(_settings.TestAdapterPaths);
+                Assert.IsTrue(_settings.CollectSourceInformation);
+                Assert.IsEmpty(_settings.TestProperties);
+                Assert.Null(_settings.InternalTraceLevel);
+                Assert.Null(_settings.WorkDirectory);
+                Assert.That(_settings.NumberOfTestWorkers, Is.EqualTo(-1));
+                Assert.That(_settings.DefaultTimeout, Is.EqualTo(0));
+                Assert.That(_settings.Verbosity, Is.EqualTo(0));
+                Assert.False(_settings.ShadowCopyFiles);
+                Assert.False(_settings.UseVsKeepEngineRunning);
+                Assert.Null(_settings.BasePath);
+                Assert.Null(_settings.PrivateBinPath);
+                Assert.NotNull(_settings.RandomSeed);
+                Assert.False(_settings.SynchronousEvents);
+                Assert.Null(_settings.DomainUsage);
+                Assert.False(_settings.InProcDataCollectorsAvailable);
+                Assert.IsFalse(_settings.DisableAppDomain);
+                Assert.IsFalse(_settings.DisableParallelization);
+                Assert.IsFalse(_settings.DesignMode);
+                Assert.False(_settings.UseTestOutputXml);
+            });
         }
 
         [Test]
@@ -179,6 +184,40 @@ public void WorkDirectorySetting()
             Assert.That(_settings.WorkDirectory, Is.EqualTo("/my/work/dir"));
         }
 
+        /// <summary>
+        /// Workdir not set, TestOutput is relative
+        /// </summary>
+        [Test]
+        public void TestOutputSetting()
+        {
+            _settings.Load("<RunSettings><NUnit><TestOutputXml>/my/work/dir</TestOutputXml></NUnit></RunSettings>");
+            Assert.That(_settings.UseTestOutputXml);
+            Assert.Multiple(() =>
+            {
+                Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\work\dir"));
+                Assert.That(Path.IsPathRooted(_settings.TestOutputXml), Is.True);
+            });
+
+        }
+
+        /// <summary>
+        /// Workdir set, and is absolute,  TestOutputXml is relative
+        /// </summary>
+        [Test]
+        public void TestOutputSettingWithWorkDir()
+        {
+            _settings.Load(@"<RunSettings><NUnit><WorkDirectory>C:\Whatever</WorkDirectory><TestOutputXml>/my/testoutput/dir</TestOutputXml></NUnit></RunSettings>");
+            Assert.That(_settings.UseTestOutputXml);
+            Assert.Multiple(() =>
+            {
+                Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\testoutput\dir"));
+                Assert.That(_settings, Does.StartWith(@"C:\"));
+                Assert.That(Path.IsPathRooted(_settings.TestOutputXml), Is.True);
+            });
+
+        }
+
+
         [Test]
         public void NumberOfTestWorkersSetting()
         {
@@ -255,7 +294,7 @@ public void RandomSeedSetting()
         public void DefaultTestNamePattern()
         {
             _settings.Load("<RunSettings><NUnit><DefaultTestNamePattern>{m}{a:1000}</DefaultTestNamePattern></NUnit></RunSettings>");
-            Assert.That(_settings.DefaultTestNamePattern,Is.EqualTo("{m}{a:1000}"));
+            Assert.That(_settings.DefaultTestNamePattern, Is.EqualTo("{m}{a:1000}"));
         }
 
         [Test]