Merge pull request #450 from nunit/jnm2/run_all_tests_with_adapter

Test in all target environments using the correct version of the NUnit VSTest adapter
nunit · Feb 18, 2018 · 323b598 · 323b598
2 parents 77c85b5 + f2a6f69
commit 323b598
Show file tree

Hide file tree

Showing 21 changed files with 217 additions and 130 deletions.
diff --git a/DisableAppDomain.runsettings b/DisableAppDomain.runsettings
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <DisableAppDomain>true</DisableAppDomain>
+  </RunConfiguration>
+</RunSettings>
diff --git a/NUnit3TestAdapter.sln b/NUnit3TestAdapter.sln
diff --git a/README.md b/README.md
@@ -10,17 +10,20 @@ The NUnit 3 Test Adapter is Open Source software released under the [MIT license
 
 ## Developing
 
-You need Visual Studio 2017 for building the adapter
+Visual Studio 2017 is required to build the adapter.
 
-You will get some failing tests (6) in VS, that is intended.  
+Use `.\build -t test` at the command line to run complete tests.
 
-Use command line (Cake build):  
+To run and debug tests on .NET Framework, load `DisableAppDomain.runsettings`.
 
-`build -t test` 
+Visual Studio’s Test Explorer only allows you to run tests against the first target in the test project
+[(upvote)](https://developercommunity.visualstudio.com/content/problem/150864/running-tests-in-a-csproj-with-multiple-targetfram.html).
+That makes command line is the easiest way to run .NET Core tests for now. If you need to frequently debug into .NET Core tests,
+you can temporarily switch the order of the `<TargetFrameworks>` in `NUnit.TestAdapter.Tests.csproj`.
 
-to get the right results, or exclude the tests from mock-assembly. 
+The `mock-assembly` tests are not for direct running.
 
-For more details see https://github.com/nunit/docs/wiki/Packaging-the-V3-Adapter  
+See https://github.com/nunit/docs/wiki/Packaging-the-V3-Adapter for more details.
 
 
 ## Announcement

diff --git a/build.cake b/build.cake
@@ -70,7 +70,8 @@ var BIN_DIR = PROJECT_DIR + "bin/" + configuration + "/";
 
 var ADAPTER_PROJECT = SRC_DIR + "NUnitTestAdapter/NUnit.TestAdapter.csproj";
 
-var NET35_BIN_DIR = SRC_DIR + "NUnitTestAdapter/bin/" + configuration + "/net35/";
+var ADAPTER_BIN_DIR_NET35 = SRC_DIR + $"NUnitTestAdapter/bin/{configuration}/net35/";
+var ADAPTER_BIN_DIR_NETCOREAPP10 = SRC_DIR + $"NUnitTestAdapter/bin/{configuration}/netcoreapp1.0/";
 
 var BIN_DIRS = new [] {
     PROJECT_DIR + "src/empty-assembly/bin",
@@ -83,10 +84,6 @@ var BIN_DIRS = new [] {
 // Solution
 var ADAPTER_SOLUTION = PROJECT_DIR + "NUnit3TestAdapter.sln";
 
-// Test Assemblies
-var TEST_NET35 = SRC_DIR + "NUnitTestAdapterTests/bin/" + configuration + "/net45/NUnit.VisualStudio.TestAdapter.Tests.exe";
-var TEST_NETCOREAPP10 = SRC_DIR + "NUnitTestAdapterTests/bin/" + configuration + "/netcoreapp1.0/publish/NUnit.VisualStudio.TestAdapter.Tests.dll";
-
 //////////////////////////////////////////////////////////////////////
 // CLEAN
 //////////////////////////////////////////////////////////////////////
@@ -131,66 +128,82 @@ Task("Build")
         Information("Building using MSBuild at " + msBuildPathX64);
         Information("Configuration is:"+configuration);
 
-        MSBuild(ADAPTER_SOLUTION, CreateSettings(msBuildPathX64));
-
-        Information("Publishing netcoreapp1.0 tests so that dependencies are present...");
-
-        MSBuild(PROJECT_DIR + "src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj", CreateSettings(msBuildPathX64)
-            .WithTarget("Publish")
-            .WithProperty("TargetFramework", "netcoreapp1.0")
-            .WithProperty("NoBuild", "true") // https://github.com/dotnet/cli/issues/5331#issuecomment-338392972
-            .WithRawArgument("/nologo"));
-
-        MSBuildSettings CreateSettings(FilePath toolPath) => new MSBuildSettings
+        MSBuild(ADAPTER_SOLUTION, new MSBuildSettings
         {
             Configuration = configuration,
-            ToolPath = toolPath,
+            ToolPath = msBuildPathX64,
             ToolVersion = MSBuildToolVersion.VS2017,
             EnvironmentVariables = new Dictionary<string, string>
             {
                 ["PackageVersion"] = packageVersion
             }
-        };
+        });
     });
 
 //////////////////////////////////////////////////////////////////////
 // TEST
 //////////////////////////////////////////////////////////////////////
 
-Task("TestAdapter")
-    .IsDependentOn("Build")
-    .Does(() =>
-    {
-        int result = StartProcess(TEST_NET35);
-        if (result != 0)
-            throw new Exception("TestAdapter failed");
-    });
+string GetTestAssemblyPath(string framework)
+{
+    return SRC_DIR + $"NUnitTestAdapterTests/bin/{configuration}/{framework}/NUnit.VisualStudio.TestAdapter.Tests.dll";
+}
 
-Task("TestAdapterNetCore")
-    .IsDependentOn("Build")
-    .Does(() =>
-    {
-        DotNetCoreExecute(TEST_NETCOREAPP10);
-    });
+foreach (var (framework, vstestFramework, adapterDir) in new[] {
+    ("net45", "Framework45", ADAPTER_BIN_DIR_NET35),
+    ("netcoreapp1.0", "FrameworkCore10", ADAPTER_BIN_DIR_NETCOREAPP10)
+})
+{
+    Task($"VSTest-{framework}")
+        .IsDependentOn("Build")
+        .Does(() =>
+        {
+            var settings = new VSTestSettings
+            {
+                TestAdapterPath = adapterDir,
+                // Enables the tests to run against the correct version of Microsoft.VisualStudio.TestPlatform.ObjectModel.dll.
+                // (The DLL they are compiled against depends on VS2012 at runtime.)
+                SettingsFile = File("DisableAppDomain.runsettings")
+            };
+
+            // https://github.com/Microsoft/vswhere/issues/126#issuecomment-360542783
+            var vstestInstallation = VSWhereLatest(new VSWhereLatestSettings
+            {
+                Requires = "Microsoft.VisualStudio.TestTools.TestPlatform.V1.CLI"
+            }.WithRawArgument("-products *"));
 
-Task("TestAdapterUsingVSTest")
-    .IsDependentOn("Build")
-    .Does(() =>
-    {
-        var settings = new VSTestSettings();
+            if (vstestInstallation != null) settings.ToolPath = vstestInstallation
+                .CombineWithFilePath(@"Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe");
 
-        // https://github.com/Microsoft/vswhere/issues/126#issuecomment-360542783
-        var vstestInstallation = VSWhereLatest(new VSWhereLatestSettings
-        {
-             Requires = "Microsoft.VisualStudio.TestTools.TestPlatform.V1.CLI"
-        }.WithRawArgument("-products *"));
+            VSTest(GetTestAssemblyPath(framework), settings);
+        });
 
-        if (vstestInstallation != null) settings.ToolPath = vstestInstallation
-            .CombineWithFilePath(@"Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe");
+    Task($"DotnetTest-{framework}")
+        .IsDependentOn("Build")
+        .Does(() =>
+        {
+            DotNetCoreTest(SRC_DIR + "NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj", new DotNetCoreTestSettings
+            {
+                Configuration = configuration,
+                Framework = framework,
+                NoBuild = true,
+                TestAdapterPath = adapterDir,
+                Settings = File("DisableAppDomain.runsettings")
+            });
+        });
 
-        VSTest(TEST_NET35, settings
-            .WithRawArgument("/TestAdapterPath:" + NET35_BIN_DIR));
-    });
+    Task($"DotnetVSTest-{framework}")
+        .IsDependentOn("Build")
+        .Does(() =>
+        {
+            DotNetCoreVSTest(GetTestAssemblyPath(framework), new DotNetCoreVSTestSettings
+            {
+                TestAdapterPath = adapterDir,
+                Framework = vstestFramework,
+                Settings = File("DisableAppDomain.runsettings")
+            });
+        });
+}
 
 //////////////////////////////////////////////////////////////////////
 // PACKAGE
@@ -214,13 +227,13 @@ Task("CreateWorkingImage")
         // dotnet publish doesn't work for .NET 3.5
         var net35Files = new FilePath[]
         {
-            NET35_BIN_DIR + "NUnit3.TestAdapter.dll",
-            NET35_BIN_DIR + "nunit.engine.dll",
-            NET35_BIN_DIR + "nunit.engine.api.dll",
-            NET35_BIN_DIR + "Mono.Cecil.dll",
-            NET35_BIN_DIR + "Mono.Cecil.Pdb.dll",
-            NET35_BIN_DIR + "Mono.Cecil.Mdb.dll",
-            NET35_BIN_DIR + "Mono.Cecil.Rocks.dll"
+            ADAPTER_BIN_DIR_NET35 + "NUnit3.TestAdapter.dll",
+            ADAPTER_BIN_DIR_NET35 + "nunit.engine.dll",
+            ADAPTER_BIN_DIR_NET35 + "nunit.engine.api.dll",
+            ADAPTER_BIN_DIR_NET35 + "Mono.Cecil.dll",
+            ADAPTER_BIN_DIR_NET35 + "Mono.Cecil.Pdb.dll",
+            ADAPTER_BIN_DIR_NET35 + "Mono.Cecil.Mdb.dll",
+            ADAPTER_BIN_DIR_NET35 + "Mono.Cecil.Rocks.dll"
         };
 
         var net35Dir = PACKAGE_IMAGE_DIR + "build/net35";
@@ -295,9 +308,12 @@ Task("Rebuild")
     .IsDependentOn("Build");
 
 Task("Test")
-    .IsDependentOn("TestAdapter")
-    .IsDependentOn("TestAdapterNetCore")
-    .IsDependentOn("TestAdapterUsingVSTest");
+    .IsDependentOn("VSTest-net45")
+    .IsDependentOn("VSTest-netcoreapp1.0")
+    .IsDependentOn("DotnetTest-net45")
+    .IsDependentOn("DotnetTest-netcoreapp1.0")
+    .IsDependentOn("DotnetVSTest-net45")
+    .IsDependentOn("DotnetVSTest-netcoreapp1.0");
 
 Task("Package")
     .IsDependentOn("PackageZip")

diff --git a/src/Common.props b/src/Common.props
@@ -0,0 +1,8 @@
+<Project>
+
+  <PropertyGroup>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>..\NUnitAdapter.snk</AssemblyOriginatorKeyFile>
+  </PropertyGroup>
+
+</Project>
diff --git a/src/NUnit3AdapterExternalTests/NUnit3AdapterExternalTests.csproj b/src/NUnit3AdapterExternalTests/NUnit3AdapterExternalTests.csproj
@@ -1,4 +1,6 @@
-<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="..\Common.props" />
   <PropertyGroup>
     <AssemblyName>NUnit.VisualStudio.TestAdapter.ExternalTests</AssemblyName>
     <RootNamespace>NUnit.VisualStudio.TestAdapter.Tests</RootNamespace>
@@ -13,4 +15,5 @@
   <ItemGroup>
     <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
   </ItemGroup>
+
 </Project>
diff --git a/src/NUnitTestAdapter/NUnitAdapter.snk → src/NUnitAdapter.snk b/src/NUnitTestAdapter/NUnitAdapter.snk → src/NUnitAdapter.snk
diff --git a/src/NUnitTestAdapter/NUnit.TestAdapter.csproj b/src/NUnitTestAdapter/NUnit.TestAdapter.csproj
@@ -1,4 +1,6 @@
-<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="..\Common.props" />
   <PropertyGroup>
     <AssemblyName>NUnit3.TestAdapter</AssemblyName>
     <RootNamespace>NUnit.VisualStudio.TestAdapter</RootNamespace>
@@ -15,8 +17,6 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <SignAssembly>true</SignAssembly>
-    <AssemblyOriginatorKeyFile>NUnitAdapter.snk</AssemblyOriginatorKeyFile>
     <PackageId>NUnit3TestAdapter</PackageId>
     <Authors>Charlie Poole, Terje Sandstrom</Authors>
   </PropertyGroup>
@@ -25,14 +25,10 @@
     <Content Include="..\..\LICENSE.txt" Link="LICENSE.txt" />
   </ItemGroup>
 
-  <ItemGroup>
-    <None Include="NUnitAdapter.snk" />
-  </ItemGroup>
-
   <ItemGroup Condition="'$(TargetFramework)' == 'net35'">
     <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="11.0.0" PrivateAssets="All" />
     <PackageReference Include="Mono.Cecil" Version="0.9.6.4" />
-    <PackageReference Include="nunit.engine" Version="3.7.0" PrivateAssets="All" />
+    <PackageReference Include="nunit.engine" Version="3.7.0" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp1.0'">
@@ -43,4 +39,10 @@
     <PackageReference Include="nunit.engine.netstandard" Version="3.7.0" />
   </ItemGroup>
 
+  <Target Name="PreventTestPlatformObjectModelCopyLocal" AfterTargets="ResolveReferences">
+    <ItemGroup>
+      <ReferenceCopyLocalPaths Remove="@(ReferenceCopyLocalPaths)" Condition="'%(Filename)' == 'Microsoft.VisualStudio.TestPlatform.ObjectModel'" />
+    </ItemGroup>
+  </Target>
+
 </Project>
diff --git a/src/NUnitTestAdapter/NUnitTestAdapter.cs b/src/NUnitTestAdapter/NUnitTestAdapter.cs
@@ -74,6 +74,8 @@ protected NUnitTestAdapter()
 
         #endregion
 
+        internal event Action<TestEngineClass> InternalEngineCreated;
+
         #region Properties
 
         public AdapterSettings Settings { get; private set; }
@@ -117,7 +119,9 @@ public static bool IsRunningUnderIDE
         // Discover or Execute method must call this method.
         protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogger)
         {
-            TestEngine = new TestEngineClass();
+            var engine = new TestEngineClass();
+            InternalEngineCreated?.Invoke(engine);
+            TestEngine = engine;
             TestLog = new TestLogger(messageLogger);
             Settings = new AdapterSettings(TestLog);
             TestLog.InitSettings(Settings);

diff --git a/src/NUnitTestAdapter/Properties/AssemblyInfo.cs b/src/NUnitTestAdapter/Properties/AssemblyInfo.cs
@@ -5,6 +5,7 @@
 using System;
 using System.Reflection;
 using System.Resources;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 [assembly: AssemblyTitle("NUnit 3 Test Adapter for Visual Studio")]
@@ -22,3 +23,5 @@
 [assembly: Guid("c0aad5e4-b486-49bc-b3e8-31e01be6fefe")]
 [assembly: AssemblyVersion("3.10.0.21")]
 [assembly: AssemblyFileVersion("3.10.0.21")]
+
+[assembly: InternalsVisibleTo("NUnit.VisualStudio.TestAdapter.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010029b97dea816272cc4ea44cf3cf666f8150d6dfe1274b6c2e6c4d54259b756888ec08ad6dd3ea0f540b30408b948ae5f39cf0c7b210abdec267b367ce1eccab97d5c6c02ee67090827ffd699544fa2add4849b45a1901eac08495bfee0397fba3946ff3912ce0b9a497818e418a77a0c8db4ca1780e7b6f6dd6911395fcc0faba")]
diff --git a/src/NUnitTestAdapterTests/Fakes/FakeTestData.cs b/src/NUnitTestAdapterTests/Fakes/FakeTestData.cs
@@ -153,12 +153,7 @@ private static void FakeTestCase() { } // LineNumber should be this line
         public static readonly string AssemblyPath = 
             typeof(FakeTestData).GetTypeInfo().Assembly.ManifestModule.FullyQualifiedName;
 
-        public static readonly string CodeFile = Path.Combine(Path.GetDirectoryName(AssemblyPath),
-#if NETCOREAPP1_0
-            @"..\..\..\..\Fakes\FakeTestData.cs");
-#else
-            @"..\..\..\Fakes\FakeTestData.cs");
-#endif
+        public static readonly string CodeFile = Path.Combine(Path.GetDirectoryName(AssemblyPath), @"..\..\..\Fakes\FakeTestData.cs");
 
         // NOTE: If the location of the FakeTestCase method defined 
         // above changes, update the value of LineNumber.

diff --git a/src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj b/src/NUnitTestAdapterTests/NUnit.TestAdapter.Tests.csproj
@@ -1,13 +1,13 @@
-<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="..\Common.props" />
   <PropertyGroup>
     <RootNamespace>NUnit.VisualStudio.TestAdapter.Tests</RootNamespace>
     <AssemblyName>NUnit.VisualStudio.TestAdapter.Tests</AssemblyName>
     <TargetFrameworks>net45;netcoreapp1.0</TargetFrameworks>
-    <!--<TargetFramework>net45</TargetFramework>-->
     <DebugType Condition="'$(TargetFramework)' != '' AND '$(TargetFramework)' != 'netcoreapp1.0'">Full</DebugType>
-    <OutputType>exe</OutputType>
-    <OutputTypeEx>exe</OutputTypeEx>
   </PropertyGroup>
+
   <ItemGroup>
     <Content Include="..\native-assembly\NativeTests.dll" Link="NativeTests.dll">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -17,8 +17,6 @@
   <ItemGroup>
     <PackageReference Include="NSubstitute" Version="3.1.0" />
     <PackageReference Include="NUnit" Version="3.9.0" />
-    <PackageReference Include="nunit3testadapter" Version="3.9.0" />
-    <PackageReference Include="NUnitLite" Version="3.9.0" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
@@ -27,6 +25,7 @@
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">
     <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="15.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -45,4 +44,11 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Compile>
   </ItemGroup>
+
+  <Target Name="PreventTestPlatformObjectModelCopyLocal" AfterTargets="ResolveReferences">
+    <ItemGroup>
+      <ReferenceCopyLocalPaths Remove="@(ReferenceCopyLocalPaths)" Condition="'%(Filename)' == 'Microsoft.VisualStudio.TestPlatform.ObjectModel'" />
+    </ItemGroup>
+  </Target>
+
 </Project>