Skip to content

Commit

Permalink
Merge pull request #1473 from nunit/issue-1465
Browse files Browse the repository at this point in the history 
Eliminate unnecessary errors when processing non-test assemblies
  • Loading branch information
@CharliePoole
CharliePoole authored Sep 7, 2024
2 parents 355432d + e743826 commit d884dd9
Show file tree
Hide file tree
Showing 8 changed files with 158 additions and 30 deletions.
46 changes: 45 additions & 1 deletion build.cake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,17 @@ var StandardRunnerTests = new List<PackageTest>
Net462Test,
Net462X86Test,
Net462PlusNet462Test,
Net462ExeTest,
NetCore31Test,
Net60Test,
Net70Test,
Net80Test,
Net60PlusNet80Test,
Net462PlusNet60Test,
NUnitProjectTest,
V2ResultWriterTest
V2ResultWriterTest,
VSProjectLoaderTest_Project,
VSProjectLoaderTest_Solution
};

// Tests run for the NETCORE runner package
Expand Down Expand Up @@ -97,6 +100,26 @@ static ExpectedResult MockAssemblyX86ExpectedResult(params string[] runtimes)
return result;
}

static ExpectedResult MockAssemblySolutionResult = new ExpectedResult("Failed")
{
Total = 37 * 5,
Passed = 23 * 5,
Failed = 5 * 5,
Warnings = 1 * 5,
Inconclusive = 1 * 5,
Skipped = 7 * 5,
Assemblies = new ExpectedAssemblyResult[]
{
new ExpectedAssemblyResult("mock-assembly.dll", "net-4.6.2"),
new ExpectedAssemblyResult("mock-assembly.dll", "netcore-3.1"),
new ExpectedAssemblyResult("mock-assembly.dll", "netcore-6.0"),
new ExpectedAssemblyResult("mock-assembly.dll", "netcore-7.0"),
new ExpectedAssemblyResult("mock-assembly.dll", "netcore-8.0"),
new ExpectedAssemblyResult("notest-assembly.dll", "net-4.6.2"),
new ExpectedAssemblyResult("notest-assembly.dll", "netcore-3.1"),
new ExpectedAssemblyResult("notest-assembly.dll", "netstandard-2.0")
}
};

static PackageTest Net462Test = new PackageTest(
1, "Net462Test",
Expand All @@ -110,6 +133,12 @@ static PackageTest Net462X86Test = new PackageTest(
"net462/mock-assembly-x86.dll",
MockAssemblyX86ExpectedResult("net-4.6.2"));

static PackageTest Net462ExeTest = new PackageTest(
1, "Net462ExeTest",
"Run nunit.engine.core.tests.exe under .NET 4.6.2",
"net462/nunit.engine.core.tests.exe",
new ExpectedResult("Passed") { Assemblies = new[] { new ExpectedAssemblyResult("nunit.engine.core.tests.exe")}});

static PackageTest Net462PlusNet462Test = new PackageTest(
1, "Net462PlusNet462Test",
"Run two copies of mock-assembly together",
Expand Down Expand Up @@ -180,6 +209,7 @@ static PackageTest Net462PlusNet60Test = new PackageTest(

static ExtensionSpecifier NUnitProjectLoader = KnownExtensions.NUnitProjectLoader.SetVersion("3.8.0");
static ExtensionSpecifier NUnitV2ResultWriter = KnownExtensions.NUnitV2ResultWriter.SetVersion("3.8.0");
static ExtensionSpecifier VSProjectLoader = KnownExtensions.VSProjectLoader.SetVersion("3.9.0");

static PackageTest NUnitProjectTest = new PackageTest(
1, "NUnitProjectTest",
Expand All @@ -195,6 +225,20 @@ static PackageTest V2ResultWriterTest = new PackageTest(
MockAssemblyExpectedResult("netcore-6.0"),
NUnitV2ResultWriter);

static PackageTest VSProjectLoaderTest_Project = new PackageTest(
1, "VSProjectLoaderTest_Project",
"Run mock-assembly using the .csproj file",
"../../src/NUnitEngine/mock-assembly/mock-assembly.csproj --config=Release",
MockAssemblyExpectedResult("net462", "netcore-3.1", "netcore-6.0", "netcore-7.0", "netcore-8.0"),
VSProjectLoader);

static PackageTest VSProjectLoaderTest_Solution = new PackageTest(
1, "VSProjectLoaderTest_Solution",
"Run mock-assembly using the .sln file",
"../../src/NUnitEngine/mock-assembly/mock-assembly.sln --config=Release",
MockAssemblySolutionResult,
VSProjectLoader);

//////////////////////////////////////////////////////////////////////
// LISTS OF FILES USED IN CHECKING PACKAGES
//////////////////////////////////////////////////////////////////////
Expand Down
1 change: 1 addition & 0 deletions src/NUnitEngine/mock-assembly/mock-assembly.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<PropertyGroup>
<RootNamespace>NUnit.Tests</RootNamespace>
<TargetFrameworks>net462;netcoreapp3.1;net6.0;net7.0;net8.0</TargetFrameworks>
<OutputPath>..\..\..\bin\$(Configuration)\</OutputPath>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>..\..\nunit.snk</AssemblyOriginatorKeyFile>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
Expand Down
31 changes: 31 additions & 0 deletions src/NUnitEngine/mock-assembly/mock-assembly.sln
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.8.34525.116
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "mock-assembly", "mock-assembly.csproj", "{FB3F6F62-37AB-4193-87A9-197C2E0CBC1B}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "notest-assembly", "..\notest-assembly\notest-assembly.csproj", "{F412390F-E7C4-41B4-A84E-8FC4DC3FB2F7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{FB3F6F62-37AB-4193-87A9-197C2E0CBC1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FB3F6F62-37AB-4193-87A9-197C2E0CBC1B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FB3F6F62-37AB-4193-87A9-197C2E0CBC1B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FB3F6F62-37AB-4193-87A9-197C2E0CBC1B}.Release|Any CPU.Build.0 = Release|Any CPU
{F412390F-E7C4-41B4-A84E-8FC4DC3FB2F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F412390F-E7C4-41B4-A84E-8FC4DC3FB2F7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F412390F-E7C4-41B4-A84E-8FC4DC3FB2F7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F412390F-E7C4-41B4-A84E-8FC4DC3FB2F7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {1BD3B7CF-501E-4806-870A-328B37C21752}
EndGlobalSection
EndGlobal
3 changes: 2 additions & 1 deletion src/NUnitEngine/notest-assembly/notest-assembly.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

<PropertyGroup>
<RootNamespace>notest_assembly</RootNamespace>
<TargetFrameworks>net462;netcoreapp3.1</TargetFrameworks>
<TargetFrameworks>net462;netstandard2.0;netcoreapp3.1</TargetFrameworks>
<OutputPath>..\..\..\bin\$(Configuration)\</OutputPath>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
<NuGetAudit>false</NuGetAudit>
</PropertyGroup>
Expand Down
10 changes: 5 additions & 5 deletions src/NUnitEngine/nunit.engine.core.tests/Services/ExtensionManagerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,22 +164,22 @@ public void SkipsGracefullyLoadingOtherFrameworkExtensionAssembly()
[TestCaseSource(nameof(ValidCombos))]
public void ValidTargetFrameworkCombinations(FrameworkCombo combo)
{
Assert.That(() => ExtensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
Assert.That(() => _extensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
Is.True);
}

[TestCaseSource(nameof(InvalidTargetFrameworkCombos))]
public void InvalidTargetFrameworkCombinations(FrameworkCombo combo)
{
Assert.That(() => ExtensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
Assert.That(() => _extensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
Is.False);
}

[TestCaseSource(nameof(InvalidRunnerCombos))]
public void InvalidRunnerTargetFrameworkCombinations(FrameworkCombo combo)
{
Assert.That(() => ExtensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly),
Throws.Exception.TypeOf<NUnitEngineException>().And.Message.Contains("not .NET Standard"));
var ex = Assert.Catch(() => _extensionManager.CanLoadTargetFramework(combo.RunnerAssembly, combo.ExtensionAssembly));
Assert.That(ex.Message.Contains("not .NET Standard"));
}

// ExtensionAssembly is internal, so cannot be part of the public test parameters
Expand Down Expand Up @@ -217,7 +217,7 @@ public static IEnumerable<TestCaseData> ValidCombos()
var extNetStandard = new ExtensionAssembly(Path.Combine(GetSiblingDirectory("netstandard2.0"), "nunit.engine.dll"), false);

yield return new TestCaseData(new FrameworkCombo(netFramework, extNetFramework)).SetName("ValidCombo(.NET Framework, .NET Framework)");
yield return new TestCaseData(new FrameworkCombo(netFramework, extNetStandard)).SetName("ValidCombo(.NET Framework, .NET Standard)");
//yield return new TestCaseData(new FrameworkCombo(netFramework, extNetStandard)).SetName("ValidCombo(.NET Framework, .NET Standard)");
#endif
}

Expand Down
6 changes: 5 additions & 1 deletion src/NUnitEngine/nunit.engine.core/Services/DriverService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,11 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string
: targetFramework.Split(new char[] { ',' })[0];

if (platform == "Silverlight" || platform == ".NETPortable" || platform == ".NETStandard" || platform == ".NETCompactFramework")
return new InvalidAssemblyFrameworkDriver(assemblyPath, platform + " test assemblies are not supported by this version of the engine");
if (skipNonTestAssemblies)
return new SkippedAssemblyFrameworkDriver(assemblyPath);
else
return new InvalidAssemblyFrameworkDriver(assemblyPath, platform +
" test assemblies are not supported by this version of the engine");
}

try
Expand Down
87 changes: 69 additions & 18 deletions src/NUnitEngine/nunit.engine.core/Services/ExtensionManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,38 +484,89 @@ internal void FindExtensionsInAssembly(ExtensionAssembly assembly)
}
}

private const string NETFRAMEWORK = ".NETFramework";
private const string NETCOREAPP = ".NETCoreApp";
private const string NETSTANDARD = ".NETStandard";

/// <summary>
/// Checks that the target framework of the current runner can load the extension assembly. For example, .NET Core
/// cannot load .NET Framework assemblies and vice-versa.
/// </summary>
/// <param name="runnerAsm">The executing runner</param>
/// <param name="extensionAsm">The extension we are attempting to load</param>
internal static bool CanLoadTargetFramework(Assembly runnerAsm, ExtensionAssembly extensionAsm)
internal bool CanLoadTargetFramework(Assembly runnerAsm, ExtensionAssembly extensionAsm)
{
if (runnerAsm == null)
return true;

string extensionFrameworkName = AssemblyDefinition.ReadAssembly(extensionAsm.FilePath).GetFrameworkName();
string runnerFrameworkName = AssemblyDefinition.ReadAssembly(runnerAsm.Location).GetFrameworkName();
if (runnerFrameworkName?.StartsWith(".NETStandard") == true)
{
throw new NUnitEngineException($"{runnerAsm.FullName} test runner must target .NET Core or .NET Framework, not .NET Standard");
}
else if (runnerFrameworkName?.StartsWith(".NETCoreApp") == true)
var runnerFrameworkName = GetTargetRuntime(runnerAsm.Location);
var extensionFrameworkName = GetTargetRuntime(extensionAsm.FilePath);

switch (runnerFrameworkName.Identifier)
{
if (extensionFrameworkName?.StartsWith(".NETStandard") != true && extensionFrameworkName?.StartsWith(".NETCoreApp") != true)
{
log.Info($".NET Core runners require .NET Core or .NET Standard extension for {extensionAsm.FilePath}");
return false;
}
case NETSTANDARD:
throw new Exception($"{runnerAsm.FullName} test runner must target .NET Core or .NET Framework, not .NET Standard");

case NETCOREAPP:
switch (extensionFrameworkName.Identifier)
{
case NETSTANDARD:
case NETCOREAPP:
return true;
case NETFRAMEWORK:
default:
log.Info($".NET Core runners require .NET Core or .NET Standard extension for {extensionAsm.FilePath}");
return false;
}
case NETFRAMEWORK:
default:
switch (extensionFrameworkName.Identifier)
{
case NETFRAMEWORK:
return runnerFrameworkName.Version.Major == 4 || extensionFrameworkName.Version.Major < 4;
// For .NET Framework calling .NET Standard, we only support if framework is 4.7.2 or higher
case NETSTANDARD:
return extensionFrameworkName.Version >= new Version(4, 7, 2);
case NETCOREAPP:
default:
log.Info($".NET Framework runners cannot load .NET Core extension {extensionAsm.FilePath}");
return false;
}
}
else if (extensionFrameworkName?.StartsWith(".NETCoreApp") == true)

//string extensionFrameworkName = AssemblyDefinition.ReadAssembly(extensionAsm.FilePath).GetFrameworkName();
//string runnerFrameworkName = AssemblyDefinition.ReadAssembly(runnerAsm.Location).GetFrameworkName();
//if (runnerFrameworkName?.StartsWith(".NETStandard") == true)
//{
// throw new NUnitEngineException($"{runnerAsm.FullName} test runner must target .NET Core or .NET Framework, not .NET Standard");
//}
//else if (runnerFrameworkName?.StartsWith(".NETCoreApp") == true)
//{
// if (extensionFrameworkName?.StartsWith(".NETStandard") != true && extensionFrameworkName?.StartsWith(".NETCoreApp") != true)
// {
// log.Info($".NET Core runners require .NET Core or .NET Standard extension for {extensionAsm.FilePath}");
// return false;
// }
//}
//else if (extensionFrameworkName?.StartsWith(".NETCoreApp") == true)
//{
// log.Info($".NET Framework runners cannot load .NET Core extension {extensionAsm.FilePath}");
// return false;
//}

//return true;
}

private System.Runtime.Versioning.FrameworkName GetTargetRuntime(string filePath)
{
var assemblyDef = AssemblyDefinition.ReadAssembly(filePath);
var frameworkName = assemblyDef.GetFrameworkName();
if (string.IsNullOrEmpty(frameworkName))
{
log.Info($".NET Framework runners cannot load .NET Core extension {extensionAsm.FilePath}");
return false;
var runtimeVersion = assemblyDef.GetRuntimeVersion();
frameworkName = $".NETFramework,Version=v{runtimeVersion.ToString(3)}";
}

return true;
return new System.Runtime.Versioning.FrameworkName(frameworkName);
}

public void Dispose()
Expand Down
4 changes: 0 additions & 4 deletions src/NUnitEngine/nunit.engine/Runners/MasterTestRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,14 +331,10 @@ private void EnsurePackagesAreExpanded(TestPackage package)
if (package == null) throw new ArgumentNullException("package");

foreach (var subPackage in package.SubPackages)
{
EnsurePackagesAreExpanded(subPackage);
}

if (package.SubPackages.Count == 0 && IsProjectPackage(package))
{
_projectService.ExpandProjectPackage(package);
}
}

private bool IsProjectPackage(TestPackage package)
Expand Down

0 comments on commit d884dd9

Please sign in to comment.