Implement Class Cleanup Lifecycle selection

scripts/build/TestFx.Sign.targets

@@ -12,7 +12,7 @@
   </ItemGroup>
 
   <!-- Signing and Localization. -->
-  <ItemGroup Condition=" '$(IsTest)' == '' or '$(IsTest)' == 'false' ">
+  <ItemGroup>
     <FilesToSign Include="$(OutDir)\$(AssemblyName).dll" Condition=" '$(IsVsixProj)' == '' or '$(IsVsixProj)' != 'true' ">
       <Authenticode>Microsoft400</Authenticode>
       <StrongName>StrongName</StrongName>
@@ -24,7 +24,7 @@
     </SignFilesDependsOn>
   </ItemGroup>
 
-  <Target Name="GatherLocalizedOutputsForSigning" DependsOnTargets="TestFxLocalization">
+  <Target Name="GatherLocalizedOutputsForSigning" DependsOnTargets="TestFxLocalization" Condition=" '$(IsTest)' == 'false' and '$(IsLocalizationEnabled)' == 'true' ">
     <ItemGroup>
       <FilesToSign Include="$(OutDir)\**\$(AssemblyName).resources.dll">
         <Authenticode>Microsoft400</Authenticode>

scripts/build/TestFx.props

@@ -4,6 +4,8 @@
     <RepoRoot Condition=" '$(RepoRoot)' == '' ">$([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'LICENSE'))'))</RepoRoot>
     <ArtifactsBinDir Condition=" '$(ArtifactsBinDir)' == '' ">$(RepoRoot)artifacts\$(Configuration)\</ArtifactsBinDir>
     <IsLocalizationEnabled Condition=" '$(UpdateXlf)' == 'true' or '$(IsLocalizedBuild)' == 'true' ">true</IsLocalizationEnabled>
+    <IsTest Condition="$(MSBuildProjectDirectory.Contains('\test\'))">true</IsTest>
+    <IsTest Condition=" '$(IsTest)' == '' ">false</IsTest>
   </PropertyGroup>
 
   <Import Project="$(RepoRoot)eng\Versions.props" />
@@ -19,10 +21,6 @@
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
 
-  <PropertyGroup>
-    <IsTest Condition="$(MSBuildProjectDirectory.Contains('\test\'))">true</IsTest>
-  </PropertyGroup>
-
   <!-- Code analysis settings -->
   <PropertyGroup>
     <RunCodeAnalysis>false</RunCodeAnalysis>

scripts/build/TestFx.targets

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <!-- Import localization specific Targets if enabled. -->
-  <Import Project="$(MSBuildThisFileDirectory)TestFx.Loc.props" Condition=" ('$(IsTest)' == '' or '$(IsTest)' == 'false') and '$(IsLocalizationEnabled)' == 'true' "/>
+  <Import Project="$(MSBuildThisFileDirectory)TestFx.Loc.props" Condition=" '$(IsTest)' == 'false' and '$(IsLocalizationEnabled)' == 'true' "/>
   <Import Project="$(RepoRoot)scripts\build\TestFx.Sign.targets" Condition=" '$(TestFxSigningTargetsImported)' != 'true' " />
 
   <!-- StyleCop settings. -->

src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs

@@ -57,6 +57,12 @@ internal TestAssemblySettings GetSettings(string source)
 
             testAssemblySettings.CanParallelizeAssembly = !this.reflectHelper.IsDoNotParallelizeSet(testAssembly);
 
+            var classCleanupSequencingAttribute = this.reflectHelper.GetClassCleanupAttribute(testAssembly);
+            if (classCleanupSequencingAttribute != null)
+            {
+                testAssemblySettings.ClassCleanupLifecycle = classCleanupSequencingAttribute.CleanupBehavior;
+            }
+
             return testAssemblySettings;
         }
     }

src/Adapter/MSTest.CoreAdapter/Execution/TestClassInfo.cs

@@ -20,11 +20,11 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
     /// </summary>
     public class TestClassInfo
     {
+        private readonly object testClassExecuteSyncObject;
         private MethodInfo classCleanupMethod;
         private MethodInfo classInitializeMethod;
         private MethodInfo testCleanupMethod;
         private MethodInfo testInitializeMethod;
-        private object testClassExecuteSyncObject;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="TestClassInfo"/> class.
@@ -110,6 +110,11 @@ internal set
         /// </summary>
         public bool IsClassInitializeExecuted { get; internal set; }
 
+        /// <summary>
+        /// Gets a value indicating whether class cleanup has executed.
+        /// </summary>
+        public bool IsClassCleanupExecuted { get; internal set; }
+
         /// <summary>
         /// Gets a stack of class cleanup methods to be executed.
         /// </summary>
@@ -120,6 +125,11 @@ internal set
         /// </summary>
         public Exception ClassInitializationException { get; internal set; }
 
+        /// <summary>
+        /// Gets the exception thrown during <see cref="ClassCleanupAttribute"/> method invocation.
+        /// </summary>
+        public Exception ClassCleanupException { get; internal set; }
+
         /// <summary>
         /// Gets the class cleanup method.
         /// </summary>
@@ -332,58 +342,83 @@ public void RunClassInitialize(TestContext testContext)
         /// <summary>
         /// Run class cleanup methods
         /// </summary>
+        /// <param name="classCleanupLifecycle">The current lifecyle position that ClassCleanup is executing from</param>
         /// <returns>
         /// Any exception that can be thrown as part of a class cleanup as warning messages.
         /// </returns>
-        [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")]
-        public string RunClassCleanup()
+        public string RunClassCleanup(ClassCleanupBehavior classCleanupLifecycle = ClassCleanupBehavior.EndOfAssembly)
         {
-            if (this.ClassCleanupMethod is null && !this.BaseClassInitAndCleanupMethods.Any(p => p.Item2 != null))
+            if (this.ClassCleanupMethod is null && this.BaseClassInitAndCleanupMethods.All(p => p.Item2 == null))
             {
                 return null;
             }
 
-            if (this.IsClassInitializeExecuted || this.ClassInitializeMethod is null)
+            if (!this.IsClassCleanupExecuted)
             {
-                MethodInfo classCleanupMethod = null;
-
-                try
+                lock (this.testClassExecuteSyncObject)
                 {
-                    classCleanupMethod = this.ClassCleanupMethod;
-                    classCleanupMethod?.InvokeAsSynchronousTask(null);
-                    var baseClassCleanupQueue = new Queue<MethodInfo>(this.BaseClassCleanupMethodsStack);
-                    while (baseClassCleanupQueue.Count > 0)
+                    if (this.IsClassCleanupExecuted)
                     {
-                        classCleanupMethod = baseClassCleanupQueue.Dequeue();
-                        classCleanupMethod?.InvokeAsSynchronousTask(null);
+                        return null;
                     }
 
-                    return null;
-                }
-                catch (Exception exception)
-                {
-                    var realException = exception.InnerException ?? exception;
+                    if (this.IsClassInitializeExecuted || this.ClassInitializeMethod is null)
+                    {
+                        MethodInfo classCleanupMethod = null;
 
-                    string errorMessage;
+                        try
+                        {
+                            classCleanupMethod = this.ClassCleanupMethod;
+                            classCleanupMethod?.InvokeAsSynchronousTask(null);
+                            var baseClassCleanupQueue = new Queue<MethodInfo>(this.BaseClassCleanupMethodsStack);
+                            while (baseClassCleanupQueue.Count > 0)
+                            {
+                                classCleanupMethod = baseClassCleanupQueue.Dequeue();
+                                classCleanupMethod?.InvokeAsSynchronousTask(null);
+                            }
 
-                    // special case AssertFailedException to trim off part of the stack trace
-                    if (realException is AssertFailedException ||
-                        realException is AssertInconclusiveException)
-                    {
-                        errorMessage = realException.Message;
-                    }
-                    else
-                    {
-                        errorMessage = StackTraceHelper.GetExceptionMessage(realException);
-                    }
+                            this.IsClassCleanupExecuted = true;
+
+                            return null;
+                        }
+                        catch (Exception exception)
+                        {
+                            var realException = exception.InnerException ?? exception;
+                            this.ClassCleanupException = realException;
+
+                            string errorMessage;
+
+                            // special case AssertFailedException to trim off part of the stack trace
+                            if (realException is AssertFailedException ||
+                                realException is AssertInconclusiveException)
+                            {
+                                errorMessage = realException.Message;
+                            }
+                            else
+                            {
+                                errorMessage = StackTraceHelper.GetExceptionMessage(realException);
+                            }
+
+                            var exceptionStackTraceInfo = realException.TryGetStackTraceInformation();
 
-                    return string.Format(
-                        CultureInfo.CurrentCulture,
-                        Resource.UTA_ClassCleanupMethodWasUnsuccesful,
-                        classCleanupMethod.DeclaringType.Name,
-                        classCleanupMethod.Name,
-                        errorMessage,
-                        StackTraceHelper.GetStackTraceInformation(realException)?.ErrorStackTrace);
+                            errorMessage = string.Format(
+                                CultureInfo.CurrentCulture,
+                                Resource.UTA_ClassCleanupMethodWasUnsuccesful,
+                                classCleanupMethod.DeclaringType.Name,
+                                classCleanupMethod.Name,
+                                errorMessage,
+                                exceptionStackTraceInfo?.ErrorStackTrace);
+
+                            if (classCleanupLifecycle == ClassCleanupBehavior.EndOfClass)
+                            {
+                                var testFailedException = new TestFailedException(UnitTestOutcome.Failed, errorMessage, exceptionStackTraceInfo);
+                                this.ClassCleanupException = testFailedException;
+                                throw testFailedException;
+                            }
+
+                            return errorMessage;
+                        }
+                    }
                 }
             }
 

src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs

@@ -226,15 +226,15 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
                 PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source);
 
                 // Default test set is filtered tests based on user provided filter criteria
-                IEnumerable<TestCase> testsToRun = Enumerable.Empty<TestCase>();
+                ICollection<TestCase> testsToRun = new TestCase[0];
                 var filterExpression = this.TestMethodFilter.GetFilterExpression(runContext, frameworkHandle, out var filterHasError);
                 if (filterHasError)
                 {
                     // Bail out without processing everything else below.
                     return;
                 }
 
-                testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter));
+                testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter)).ToArray();
 
                 // this is done so that appropriate values of test context properties are set at source level
                 // and are merged with session level parameters
@@ -255,14 +255,13 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
                 }
                 catch (Exception ex)
                 {
-                    PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
-                        "Could not create TestAssemblySettingsProvider instance in child app-domain",
-                        ex);
+                    PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Could not create TestAssemblySettingsProvider instance in child app-domain", ex);
                 }
 
                 var sourceSettings = (sourceSettingsProvider != null) ? sourceSettingsProvider.GetSettings(source) : new TestAssemblySettings();
                 var parallelWorkers = sourceSettings.Workers;
                 var parallelScope = sourceSettings.Scope;
+                this.InitializeClassCleanupManager(source, testRunner, testsToRun, sourceSettings);
 
                 if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue)
                 {
@@ -303,6 +302,7 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
                             case ExecutionScope.MethodLevel:
                                 queue = new ConcurrentQueue<IEnumerable<TestCase>>(parallelizableTestSet.Select(t => new[] { t }));
                                 break;
+
                             case ExecutionScope.ClassLevel:
                                 queue = new ConcurrentQueue<IEnumerable<TestCase>>(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string));
                                 break;
@@ -350,9 +350,24 @@ private void ExecuteTestsInSource(IEnumerable<TestCase> tests, IRunContext runCo
 
                 this.RunCleanup(frameworkHandle, testRunner);
 
-                PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
-                    "Executed tests belonging to source {0}",
-                    source);
+                PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed tests belonging to source {0}", source);
+            }
+        }
+
+        private void InitializeClassCleanupManager(string source, UnitTestRunner testRunner, ICollection<TestCase> testsToRun, TestAssemblySettings sourceSettings)
+        {
+            try
+            {
+                var unitTestElements = testsToRun.Select(e => e.ToUnitTestElement(source)).ToArray();
+                testRunner.InitializeClassCleanupManager(unitTestElements, (int)sourceSettings.ClassCleanupLifecycle);
+            }
+            catch (Exception ex)
+            {
+                // source might not support this if it's legacy make sure it's supported by checking for the type
+                if (ex.GetType().FullName != "System.Runtime.Remoting.RemotingException")
+                {
+                    throw;
+                }
             }
         }
 
@@ -372,6 +387,7 @@ private void ExecuteTestsWithTestRunner(
                 }
 
                 var unitTestElement = currentTest.ToUnitTestElement(source);
+
                 testExecutionRecorder.RecordStart(currentTest);
 
                 var startTime = DateTimeOffset.Now;

src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs

@@ -113,25 +113,6 @@ internal UnitTestResult[] Execute()
 
             UnitTestResult[] result = null;
 
-            string ignoreMessage = null;
-            var isIgnoreAttributeOnClass = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.Parent.ClassType, typeof(UTF.IgnoreAttribute), false);
-            var isIgnoreAttributeOnMethod = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.TestMethod, typeof(UTF.IgnoreAttribute), false);
-
-            if (isIgnoreAttributeOnClass)
-            {
-                ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.Parent.ClassType.GetTypeInfo());
-            }
-
-            if (string.IsNullOrEmpty(ignoreMessage) && isIgnoreAttributeOnMethod)
-            {
-                ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.TestMethod);
-            }
-
-            if (isIgnoreAttributeOnClass || isIgnoreAttributeOnMethod)
-            {
-                return new[] { new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage) };
-            }
-
             try
             {
                 using (LogMessageListener logListener = new LogMessageListener(this.captureDebugTraces))