Skip to content

Commit

Permalink
Implement Class Cleanup Lifecycle selection (#968)
Browse files Browse the repository at this point in the history 
* Implement Class Cleanup Lifecycle selection

Co-authored-by: Johan Henkens <[email protected]>
  • Loading branch information
@Haplois
Haplois and Johan Henkens authored Oct 27, 2021
1 parent 8005e0d commit 477ad52
Show file tree
Hide file tree
Showing 62 changed files with 1,594 additions and 883 deletions.
4 changes: 2 additions & 2 deletions scripts/build/TestFx.Sign.targets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
</ItemGroup>

<!-- Signing and Localization. -->
<ItemGroup Condition=" '$(IsTest)' == '' or '$(IsTest)' == 'false' ">
<ItemGroup>
<FilesToSign Include="$(OutDir)\$(AssemblyName).dll" Condition=" '$(IsVsixProj)' == '' or '$(IsVsixProj)' != 'true' ">
<Authenticode>Microsoft400</Authenticode>
<StrongName>StrongName</StrongName>
Expand All @@ -24,7 +24,7 @@
</SignFilesDependsOn>
</ItemGroup>

<Target Name="GatherLocalizedOutputsForSigning" DependsOnTargets="TestFxLocalization">
<Target Name="GatherLocalizedOutputsForSigning" DependsOnTargets="TestFxLocalization" Condition=" '$(IsTest)' == 'false' and '$(IsLocalizationEnabled)' == 'true' ">
<ItemGroup>
<FilesToSign Include="$(OutDir)\**\$(AssemblyName).resources.dll">
<Authenticode>Microsoft400</Authenticode>
Expand Down
6 changes: 2 additions & 4 deletions scripts/build/TestFx.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
<RepoRoot Condition=" '$(RepoRoot)' == '' ">$([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'LICENSE'))'))</RepoRoot>
<ArtifactsBinDir Condition=" '$(ArtifactsBinDir)' == '' ">$(RepoRoot)artifacts\$(Configuration)\</ArtifactsBinDir>
<IsLocalizationEnabled Condition=" '$(UpdateXlf)' == 'true' or '$(IsLocalizedBuild)' == 'true' ">true</IsLocalizationEnabled>
<IsTest Condition="$(MSBuildProjectDirectory.Contains('\test\'))">true</IsTest>
<IsTest Condition=" '$(IsTest)' == '' ">false</IsTest>
</PropertyGroup>

<Import Project="$(RepoRoot)eng\Versions.props" />
Expand All @@ -19,10 +21,6 @@
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>

<PropertyGroup>
<IsTest Condition="$(MSBuildProjectDirectory.Contains('\test\'))">true</IsTest>
</PropertyGroup>

<!-- Code analysis settings -->
<PropertyGroup>
<RunCodeAnalysis>false</RunCodeAnalysis>
Expand Down
2 changes: 1 addition & 1 deletion scripts/build/TestFx.targets
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<!-- Import localization specific Targets if enabled. -->
<Import Project="$(MSBuildThisFileDirectory)TestFx.Loc.props" Condition=" ('$(IsTest)' == '' or '$(IsTest)' == 'false') and '$(IsLocalizationEnabled)' == 'true' "/>
<Import Project="$(MSBuildThisFileDirectory)TestFx.Loc.props" Condition=" '$(IsTest)' == 'false' and '$(IsLocalizationEnabled)' == 'true' "/>
<Import Project="$(RepoRoot)scripts\build\TestFx.Sign.targets" Condition=" '$(TestFxSigningTargetsImported)' != 'true' " />

<!-- StyleCop settings. -->
Expand Down
6 changes: 6 additions & 0 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@ internal TestAssemblySettings GetSettings(string source)

testAssemblySettings.CanParallelizeAssembly = !this.reflectHelper.IsDoNotParallelizeSet(testAssembly);

var classCleanupSequencingAttribute = this.reflectHelper.GetClassCleanupAttribute(testAssembly);
if (classCleanupSequencingAttribute != null)
{
testAssemblySettings.ClassCleanupLifecycle = classCleanupSequencingAttribute.CleanupBehavior;
}

return testAssemblySettings;
}
}
Expand Down
109 changes: 72 additions & 37 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestClassInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
/// </summary>
public class TestClassInfo
{
private readonly object testClassExecuteSyncObject;
private MethodInfo classCleanupMethod;
private MethodInfo classInitializeMethod;
private MethodInfo testCleanupMethod;
private MethodInfo testInitializeMethod;
private object testClassExecuteSyncObject;

/// <summary>
/// Initializes a new instance of the <see cref="TestClassInfo"/> class.
Expand Down Expand Up @@ -110,6 +110,11 @@ internal set
/// </summary>
public bool IsClassInitializeExecuted { get; internal set; }

/// <summary>
/// Gets a value indicating whether class cleanup has executed.
/// </summary>
public bool IsClassCleanupExecuted { get; internal set; }

/// <summary>
/// Gets a stack of class cleanup methods to be executed.
/// </summary>
Expand All @@ -120,6 +125,11 @@ internal set
/// </summary>
public Exception ClassInitializationException { get; internal set; }

/// <summary>
/// Gets the exception thrown during <see cref="ClassCleanupAttribute"/> method invocation.
/// </summary>
public Exception ClassCleanupException { get; internal set; }

/// <summary>
/// Gets the class cleanup method.
/// </summary>
Expand Down Expand Up @@ -332,58 +342,83 @@ public void RunClassInitialize(TestContext testContext)
/// <summary>
/// Run class cleanup methods
/// </summary>
/// <param name="classCleanupLifecycle">The current lifecyle position that ClassCleanup is executing from</param>
/// <returns>
/// Any exception that can be thrown as part of a class cleanup as warning messages.
/// </returns>
[SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")]
public string RunClassCleanup()
public string RunClassCleanup(ClassCleanupBehavior classCleanupLifecycle = ClassCleanupBehavior.EndOfAssembly)
{
if (this.ClassCleanupMethod is null && !this.BaseClassInitAndCleanupMethods.Any(p => p.Item2 != null))
if (this.ClassCleanupMethod is null && this.BaseClassInitAndCleanupMethods.All(p => p.Item2 == null))
{
return null;
}

if (this.IsClassInitializeExecuted || this.ClassInitializeMethod is null)
if (!this.IsClassCleanupExecuted)
{
MethodInfo classCleanupMethod = null;

try
lock (this.testClassExecuteSyncObject)
{
classCleanupMethod = this.ClassCleanupMethod;
classCleanupMethod?.InvokeAsSynchronousTask(null);
var baseClassCleanupQueue = new Queue<MethodInfo>(this.BaseClassCleanupMethodsStack);
while (baseClassCleanupQueue.Count > 0)
if (this.IsClassCleanupExecuted)
{
classCleanupMethod = baseClassCleanupQueue.Dequeue();
classCleanupMethod?.InvokeAsSynchronousTask(null);
return null;
}

return null;
}
catch (Exception exception)
{
var realException = exception.InnerException ?? exception;
if (this.IsClassInitializeExecuted || this.ClassInitializeMethod is null)
{
MethodInfo classCleanupMethod = null;

string errorMessage;
try
{
classCleanupMethod = this.ClassCleanupMethod;
classCleanupMethod?.InvokeAsSynchronousTask(null);
var baseClassCleanupQueue = new Queue<MethodInfo>(this.BaseClassCleanupMethodsStack);
while (baseClassCleanupQueue.Count > 0)
{
classCleanupMethod = baseClassCleanupQueue.Dequeue();
classCleanupMethod?.InvokeAsSynchronousTask(null);
}

// special case AssertFailedException to trim off part of the stack trace
if (realException is AssertFailedException ||
realException is AssertInconclusiveException)
{
errorMessage = realException.Message;
}
else
{
errorMessage = StackTraceHelper.GetExceptionMessage(realException);
}
this.IsClassCleanupExecuted = true;

return null;
}
catch (Exception exception)
{
var realException = exception.InnerException ?? exception;
this.ClassCleanupException = realException;

string errorMessage;

// special case AssertFailedException to trim off part of the stack trace
if (realException is AssertFailedException ||
realException is AssertInconclusiveException)
{
errorMessage = realException.Message;
}
else
{
errorMessage = StackTraceHelper.GetExceptionMessage(realException);
}

var exceptionStackTraceInfo = realException.TryGetStackTraceInformation();

return string.Format(
CultureInfo.CurrentCulture,
Resource.UTA_ClassCleanupMethodWasUnsuccesful,
classCleanupMethod.DeclaringType.Name,
classCleanupMethod.Name,
errorMessage,
StackTraceHelper.GetStackTraceInformation(realException)?.ErrorStackTrace);
errorMessage = string.Format(
CultureInfo.CurrentCulture,
Resource.UTA_ClassCleanupMethodWasUnsuccesful,
classCleanupMethod.DeclaringType.Name,
classCleanupMethod.Name,
errorMessage,
exceptionStackTraceInfo?.ErrorStackTrace);

if (classCleanupLifecycle == ClassCleanupBehavior.EndOfClass)
{
var testFailedException = new TestFailedException(UnitTestOutcome.Failed, errorMessage, exceptionStackTraceInfo);
this.ClassCleanupException = testFailedException;
throw testFailedException;
}

return errorMessage;
}
}
}
}

Expand Down
32 changes: 24 additions & 8 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,15 +226,15 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source);

// Default test set is filtered tests based on user provided filter criteria
IEnumerable<TestCase> testsToRun = Enumerable.Empty<TestCase>();
ICollection<TestCase> testsToRun = new TestCase[0];
var filterExpression = this.TestMethodFilter.GetFilterExpression(runContext, frameworkHandle, out var filterHasError);
if (filterHasError)
{
// Bail out without processing everything else below.
return;
}

testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter));
testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter)).ToArray();

// this is done so that appropriate values of test context properties are set at source level
// and are merged with session level parameters
Expand All @@ -255,14 +255,13 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
}
catch (Exception ex)
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Could not create TestAssemblySettingsProvider instance in child app-domain",
ex);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Could not create TestAssemblySettingsProvider instance in child app-domain", ex);
}

var sourceSettings = (sourceSettingsProvider != null) ? sourceSettingsProvider.GetSettings(source) : new TestAssemblySettings();
var parallelWorkers = sourceSettings.Workers;
var parallelScope = sourceSettings.Scope;
this.InitializeClassCleanupManager(source, testRunner, testsToRun, sourceSettings);

if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue)
{
Expand Down Expand Up @@ -303,6 +302,7 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
case ExecutionScope.MethodLevel:
queue = new ConcurrentQueue<IEnumerable<TestCase>>(parallelizableTestSet.Select(t => new[] { t }));
break;

case ExecutionScope.ClassLevel:
queue = new ConcurrentQueue<IEnumerable<TestCase>>(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string));
break;
Expand Down Expand Up @@ -350,9 +350,24 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo

this.RunCleanup(frameworkHandle, testRunner);

PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executed tests belonging to source {0}",
source);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed tests belonging to source {0}", source);
}
}

private void InitializeClassCleanupManager(string source, UnitTestRunner testRunner, ICollection<TestCase> testsToRun, TestAssemblySettings sourceSettings)
{
try
{
var unitTestElements = testsToRun.Select(e => e.ToUnitTestElement(source)).ToArray();
testRunner.InitializeClassCleanupManager(unitTestElements, (int)sourceSettings.ClassCleanupLifecycle);
}
catch (Exception ex)
{
// source might not support this if it's legacy make sure it's supported by checking for the type
if (ex.GetType().FullName != "System.Runtime.Remoting.RemotingException")
{
throw;
}
}
}

Expand All @@ -372,6 +387,7 @@ private void ExecuteTestsWithTestRunner(
}

var unitTestElement = currentTest.ToUnitTestElement(source);

testExecutionRecorder.RecordStart(currentTest);

var startTime = DateTimeOffset.Now;
Expand Down
19 changes: 0 additions & 19 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,25 +113,6 @@ internal UnitTestResult[] Execute()

UnitTestResult[] result = null;

string ignoreMessage = null;
var isIgnoreAttributeOnClass = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.Parent.ClassType, typeof(UTF.IgnoreAttribute), false);
var isIgnoreAttributeOnMethod = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.TestMethod, typeof(UTF.IgnoreAttribute), false);

if (isIgnoreAttributeOnClass)
{
ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.Parent.ClassType.GetTypeInfo());
}

if (string.IsNullOrEmpty(ignoreMessage) && isIgnoreAttributeOnMethod)
{
ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.TestMethod);
}

if (isIgnoreAttributeOnClass || isIgnoreAttributeOnMethod)
{
return new[] { new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage) };
}

try
{
using (LogMessageListener logListener = new LogMessageListener(this.captureDebugTraces))
Expand Down
Loading

0 comments on commit 477ad52

Please sign in to comment.