Add support for additional extension directories

src/NUnitConsole/nunit3-console.tests/CommandLineTests.cs

@@ -144,6 +144,7 @@ public void NoInputFiles()
         [TestCase("DisposeRunners", "dispose-runners")]
         [TestCase("TeamCity", "teamcity")]
         [TestCase("SkipNonTestAssemblies", "skipnontestassemblies")]
+        [TestCase("NoResult", "noresult")]
 #if NETFRAMEWORK
         [TestCase("RunAsX86", "x86")]
         [TestCase("ShadowCopyFiles", "shadowcopy")]
@@ -505,20 +506,6 @@ public void DefaultResultSpecification()
             Assert.That(spec.Transform, Is.Null);
         }
 
-        [Test]
-        public void NoResultSuppressesDefaultResultSpecification()
-        {
-            var options = ConsoleMocks.Options("test.dll", "-noresult");
-            Assert.That(options.ResultOutputSpecifications.Count, Is.EqualTo(0));
-        }
-
-        [Test]
-        public void NoResultSuppressesAllResultSpecifications()
-        {
-            var options = ConsoleMocks.Options("test.dll", "-result:results.xml", "-noresult", "-result:nunit2results.xml;format=nunit2");
-            Assert.That(options.ResultOutputSpecifications.Count, Is.EqualTo(0));
-        }
-
         [Test]
         public void InvalidResultSpecRecordsError()
         {
@@ -859,6 +846,13 @@ public void DeprecatedLabelsOptionsAreReplacedCorrectly(string oldOption, string
             Assert.That(options.DisplayTestLabels, Is.EqualTo(newOption));
         }
 
+        public void UserExtensionDirectoryTest()
+        {
+            ConsoleOptions options = ConsoleMocks.Options("--extensionDirectory=/a/b/c");
+            Assert.That(options.Validate);
+            Assert.That(options.ExtensionDirectories.Contains("/a/b/c"));
+        }
+
         private static IFileSystem GetFileSystemContainingFile(string fileName)
         {
             var fileSystem = new VirtualFileSystem();

src/NUnitConsole/nunit3-console.tests/ConsoleRunnerTests.cs

@@ -39,9 +39,9 @@ public void ThrowsNUnitEngineExceptionWhenTestResultsAreNotWriteable()
         }
 
         [Test]
-        public void ThrowsNUnitExceptionWhenTeamcityOptionIsSpecifiedButNotAvailable()
+        public void ThrowsRequiredExtensionExceptionWhenTeamcityOptionIsSpecifiedButNotAvailable()
         {
-            var ex = Assert.Throws<NUnitEngineException>(
+            var ex = Assert.Throws<RequiredExtensionException>(
                 () => new ConsoleRunner(_testEngine, ConsoleMocks.Options("mock-assembly.dll", "--teamcity"), new ColorConsoleWriter()));
 
             Assert.That(ex, Has.Message.Contains("teamcity"));

src/NUnitConsole/nunit3-console.tests/MakeTestPackageTests.cs

@@ -65,7 +65,7 @@ public void WhenOptionIsSpecified_PackageIncludesSetting(string option, string k
             var options = ConsoleMocks.Options("test.dll", option);
             var package = ConsoleRunner.MakeTestPackage(options);
 
-            Assert.That(package.Settings.ContainsKey(key), "Setting not included for {0}", option);
+            Assert.That(package.Settings.ContainsKey(key), $"Setting not included for {option}");
             Assert.That(package.Settings[key], Is.EqualTo(val), "NumberOfTestWorkers not set correctly for {0}", option);
         }
 

src/NUnitConsole/nunit3-console/ConsoleOptions.cs

@@ -21,8 +21,7 @@ public class ConsoleOptions : OptionSet
     {
         private static readonly string CURRENT_DIRECTORY_ON_ENTRY = Directory.GetCurrentDirectory();
 
-        private bool validated;
-        private bool noresult;
+        private bool _validated;
 
         /// <summary>
         /// An abstraction of the file system
@@ -41,19 +40,24 @@ internal ConsoleOptions(IDefaultOptionsProvider defaultOptionsProvider, IFileSys
                 Parse(args);
         }
 
-        // Action to Perform
+        // Action to Perform ( Default is to run the tests )
 
         public bool Explore { get; private set; }
 
         public bool ShowHelp { get; private set; }
 
         public bool ShowVersion { get; private set; }
 
+        public bool ListExtensions { get; private set; }
+
+        // Additional directories to be used to search for user extensions
+        public List<string> ExtensionDirectories { get; } = new List<string>();
+
         // Select tests
 
-        public IList<string> InputFiles { get; } = new List<string>();
+        public List<string> InputFiles { get; } = new List<string>();
 
-        public IList<string> TestList { get; } = new List<string>();
+        public List<string> TestList { get; } = new List<string>();
 
         public IDictionary<string, string> TestParameters { get; } = new Dictionary<string, string>();
 
@@ -99,13 +103,15 @@ public string WorkDirectory
         public string InternalTraceLevel { get; private set; }
         public bool InternalTraceLevelSpecified { get { return InternalTraceLevel != null; } }
 
+        public bool NoResult { get; private set; }
+
         private readonly List<OutputSpecification> resultOutputSpecifications = new List<OutputSpecification>();
-        public IList<OutputSpecification> ResultOutputSpecifications
+        public List<OutputSpecification> ResultOutputSpecifications
         {
             get
             {
-                if (noresult)
-                    return new OutputSpecification[0];
+                //if (noresult)
+                //    return new List<OutputSpecification>();
 
                 if (resultOutputSpecifications.Count == 0)
                     resultOutputSpecifications.Add(
@@ -115,7 +121,7 @@ public IList<OutputSpecification> ResultOutputSpecifications
             }
         }
 
-        public IList<OutputSpecification> ExploreOutputSpecifications { get; } = new List<OutputSpecification>();
+        public List<OutputSpecification> ExploreOutputSpecifications { get; } = new List<OutputSpecification>();
 
         public string ActiveConfig { get; private set; }
         public bool ActiveConfigSpecified { get { return ActiveConfig != null; } }
@@ -153,15 +159,13 @@ public IList<OutputSpecification> ResultOutputSpecifications
 
         public bool DebugAgent { get; private set; }
 
-        public bool ListExtensions { get; private set; }
-
         public bool PauseBeforeRun { get; private set; }
 
         public string PrincipalPolicy { get; private set; }
 
-        public IList<string> WarningMessages { get; } = new List<string>();
+        public List<string> WarningMessages { get; } = new List<string>();
 
-        public IList<string> ErrorMessages { get; } = new List<string>();
+        public List<string> ErrorMessages { get; } = new List<string>();
 
         private void ConfigureOptions()
         {
@@ -274,7 +278,7 @@ private void ConfigureOptions()
             });
 
             this.Add("noresult", "Don't save any test results.",
-                v => noresult = v != null);
+                v => NoResult = v != null);
 
             this.Add("labels=", "Specify whether to write test case names to the output. Values: Off, OnOutputOnly, Before, After, BeforeAndAfter",
                 v => {
@@ -383,6 +387,9 @@ private void ConfigureOptions()
             this.Add("list-extensions", "List all extension points and the extensions for each.",
                 v => ListExtensions = v != null);
 
+            this.Add("extensionDirectory=", "Specifies an additional directory to be examined for extensions. May be repeated.",
+                v => { ExtensionDirectories.Add(Path.GetFullPath(v)); });
+
             this.AddNetFxOnlyOption("set-principal-policy=", "Set PrincipalPolicy for the test domain.",
                 NetFxOnlyOption("set-principal-policy=", v => PrincipalPolicy = parser.RequiredValue(v, "--set-principal-policy", "UnauthenticatedPrincipal", "NoPrincipal", "WindowsPrincipal")));
 
@@ -413,11 +420,11 @@ private Action<string> NetFxOnlyOption(string optionName, Action<string> action)
 
         public bool Validate()
         {
-            if (!validated)
+            if (!_validated)
             {
                 CheckOptionCombinations();
 
-                validated = true;
+                _validated = true;
             }
 
             return ErrorMessages.Count == 0;

src/NUnitConsole/nunit3-console/ConsoleRunner.cs

@@ -11,6 +11,7 @@
 using NUnit.Engine.Extensibility;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Linq;
 
 namespace NUnit.ConsoleRunner
 {
@@ -26,7 +27,10 @@ public class ConsoleRunner
         private const int MAXIMUM_RETURN_CODE_ALLOWED = 100; // In case we are running on Unix
 
         private const string EVENT_LISTENER_EXTENSION_PATH = "/NUnit/Engine/TypeExtensions/ITestEventListener";
-        private const string TEAMCITY_EVENT_LISTENER = "NUnit.Engine.Listeners.TeamCityEventListener";
+        private const string TEAMCITY_EVENT_LISTENER_FULLNAME = "NUnit.Engine.Listeners.TeamCityEventListener";
+        private const string TEAMCITY_EVENT_LISTENER_NAME = "TeamCityEventListener";
+
+        private const string NUNIT_EXTENSION_DIRECTORIES = "NUNIT_EXTENSION_DIRECTORIES";
 
         public static readonly int OK = 0;
         public static readonly int INVALID_ARG = -1;
@@ -48,28 +52,47 @@ public class ConsoleRunner
 
         public ConsoleRunner(ITestEngine engine, ConsoleOptions options, ExtendedTextWriter writer)
         {
-            _engine = engine;
-            _options = options;
-            _outWriter = writer;
-
-            _workDirectory = options.WorkDirectory ?? Directory.GetCurrentDirectory();
-
-            if (!Directory.Exists(_workDirectory))
-                Directory.CreateDirectory(_workDirectory);
+            Guard.ArgumentNotNull(_engine = engine, nameof(engine));
+            Guard.ArgumentNotNull(_options = options, nameof(options));
+            Guard.ArgumentNotNull(_outWriter = writer, nameof(writer));
 
+            // NOTE: Accessing Services triggerss the engine to initialize all services
             _resultService = _engine.Services.GetService<IResultService>();
+            Guard.OperationValid(_resultService != null, "Internal Error: ResultService was not found");
+
             _filterService = _engine.Services.GetService<ITestFilterService>();
+            Guard.OperationValid(_filterService != null, "Internal Error: TestFilterService was not found");
+
             _extensionService = _engine.Services.GetService<IExtensionService>();
+            Guard.OperationValid(_extensionService != null, "Internal Error: ExtensionService was not found");
 
-            // TODO: Exit with error if any of the services are not found
+            var extensionPath = Environment.GetEnvironmentVariable(NUNIT_EXTENSION_DIRECTORIES);
+            if (!string.IsNullOrEmpty(extensionPath))
+                foreach (string extensionDirectory in extensionPath.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
+                    _extensionService.FindExtensionAssemblies(extensionDirectory);
+
+            foreach (string extensionDirectory in _options.ExtensionDirectories)
+               _extensionService.FindExtensionAssemblies(extensionDirectory);
+
+            // Trigger lazy loading of extensions
+            var dummy = _extensionService.Extensions;
+
+
+            _workDirectory = options.WorkDirectory;
+            if (_workDirectory != null)
+                Directory.CreateDirectory(_workDirectory);
+            else
+                _workDirectory = null;
 
             if (_options.TeamCity)
             {
                 bool teamcityInstalled = false;
                 foreach (var node in _extensionService.GetExtensionNodes(EVENT_LISTENER_EXTENSION_PATH))
-                    if (teamcityInstalled = node.TypeName == TEAMCITY_EVENT_LISTENER)
+                    if (teamcityInstalled = node.TypeName == TEAMCITY_EVENT_LISTENER_FULLNAME)
                         break;
-                if (!teamcityInstalled) throw new NUnitEngineException("Option --teamcity specified but the extension is not installed.");
+                //Guard.ArgumentValid(teamcityInstalled, "Option --teamcity specified but the extension is not installed.", "--teamCity");
+                if (!teamcityInstalled)
+                    throw new RequiredExtensionException(TEAMCITY_EVENT_LISTENER_NAME, "--teamcity");
             }
 
             // Enable TeamCityEventListener immediately, before the console is redirected
@@ -149,49 +172,49 @@ private int RunTests(TestPackage package, TestFilter filter)
         {
             var writer = new ColorConsoleWriter(!_options.NoColor);
 
-            foreach (var spec in _options.ResultOutputSpecifications)
-            {
-                var outputPath = Path.Combine(_workDirectory, spec.OutputPath);
+            if (!_options.NoResult)
+                foreach (var spec in _options.ResultOutputSpecifications)
+                {
+                    var outputPath = Path.Combine(_workDirectory, spec.OutputPath);
 
-                IResultWriter resultWriter;
+                    IResultWriter resultWriter;
 
-                try
-                {
-                    resultWriter = GetResultWriter(spec);
-                }
-                catch (Exception ex)
-                {
-                    throw new NUnitEngineException($"Error encountered in resolving output specification: {spec}", ex);
-                }
+                    try
+                    {
+                        resultWriter = GetResultWriter(spec);
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new NUnitEngineException($"Error encountered in resolving output specification: {spec}", ex);
+                    }
 
-                try
-                {
-                    var outputDirectory = Path.GetDirectoryName(outputPath);
-                    Directory.CreateDirectory(outputDirectory);
-                }
-                catch (Exception ex)
-                {
-                    writer.WriteLine(ColorStyle.Error, String.Format(
-                        "The directory in --result {0} could not be created",
-                        spec.OutputPath));
-                    writer.WriteLine(ColorStyle.Error, ExceptionHelper.BuildMessage(ex));
-                    return ConsoleRunner.UNEXPECTED_ERROR;
-                }
+                    try
+                    {
+                        var outputDirectory = Path.GetDirectoryName(outputPath);
+                        Directory.CreateDirectory(outputDirectory);
+                    }
+                    catch (Exception ex)
+                    {
+                        writer.WriteLine(ColorStyle.Error, String.Format(
+                            "The directory in --result {0} could not be created",
+                            spec.OutputPath));
+                        writer.WriteLine(ColorStyle.Error, ExceptionHelper.BuildMessage(ex));
+                        return ConsoleRunner.UNEXPECTED_ERROR;
+                    }
 
-                try
-                {
-                    resultWriter.CheckWritability(outputPath);
-                }
-                catch (Exception ex)
-                {
-                    throw new NUnitEngineException(
-                        String.Format(
-                            "The path specified in --result {0} could not be written to",
-                            spec.OutputPath), ex);
+                    try
+                    {
+                        resultWriter.CheckWritability(outputPath);
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new NUnitEngineException(
+                            String.Format(
+                                "The path specified in --result {0} could not be written to",
+                                spec.OutputPath), ex);
+                    }
                 }
 
-            }
-
             var labels = _options.DisplayTestLabels != null
                 ? _options.DisplayTestLabels.ToUpperInvariant()
                 : "ON";
@@ -310,9 +333,17 @@ private static string GetOSVersion()
 
         private void DisplayExtensionList()
         {
+            if (_options.ExtensionDirectories.Count > 0)
+            {
+                _outWriter.WriteLine(ColorStyle.SectionHeader, "User Extension Directories");
+                foreach (var dir in _options.ExtensionDirectories)
+                    _outWriter.WriteLine($"  {Path.GetFullPath(dir)}");
+                _outWriter.WriteLine();
+            }
+
             _outWriter.WriteLine(ColorStyle.SectionHeader, "Installed Extensions");
 
-            foreach (var ep in _extensionService?.ExtensionPoints ?? new IExtensionPoint[0])
+            foreach (var ep in _extensionService.ExtensionPoints)
             {
                 _outWriter.WriteLabelLine("  Extension Point: ", ep.Path);
                 foreach (var node in ep.Extensions)

src/NUnitConsole/nunit3-console/Program.cs

@@ -116,6 +116,11 @@ public static int Main(string[] args)
                     {
                         return new ConsoleRunner(engine, Options, OutWriter).Execute();
                     }
+                    catch (RequiredExtensionException ex)
+                    {
+                        OutWriter.WriteLine(ColorStyle.Error, ex.Message);
+                        return ConsoleRunner.INVALID_ARG;
+                    }
                     catch (TestSelectionParserException ex)
                     {
                         OutWriter.WriteLine(ColorStyle.Error, ex.Message);