From 37d6b05ff9af323411177dbcd2983dcdbc7657d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 19 Dec 2022 11:09:10 +0100 Subject: [PATCH] Fix cleanup inheritance calls (#1475) --- .../Execution/TestClassInfo.cs | 13 +- .../Helpers/ReflectHelper.cs | 25 +- .../Smoke.E2E.Tests/SuiteLifeCycleTests.cs | 128 +++++++++- .../LifecycleInheritance.cs | 228 ++++++++++++++++++ 4 files changed, 379 insertions(+), 15 deletions(-) create mode 100644 test/E2ETests/TestAssets/SuiteLifeCycleTestProject/LifecycleInheritance.cs diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index 7fe9a419dc..f5f444bd8d 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -10,7 +10,6 @@ using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestTools.UnitTesting; using ObjectModelUnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; @@ -155,19 +154,19 @@ public bool HasExecutableCleanupMethod { get { - if (BaseClassCleanupMethodsStack.Any()) + // If class has a cleanup method, then it is executable. + if (ClassCleanupMethod is not null) { - // If any base cleanups were pushed to the stack we need to run them return true; } - // If no class cleanup, then continue with the next one. - if (ClassCleanupMethod == null) + // Otherwise, if any base cleanups were pushed to the stack we need to run them + if (BaseClassCleanupMethodsStack.Any()) { - return false; + return true; } - return true; + return false; } } diff --git a/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs b/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs index 930cee5853..db99990368 100644 --- a/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs +++ b/src/Adapter/MSTest.TestAdapter/Helpers/ReflectHelper.cs @@ -1,11 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#nullable enable - using System; using System.Collections.Generic; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; @@ -487,14 +484,30 @@ internal virtual TAttribute[] GetCustomAttributes(MemberInfo attribu /// Returns if provided, otherwise null. internal virtual ClassCleanupBehavior? GetClassCleanupBehavior(TestClassInfo classInfo) { - if (classInfo.ClassCleanupMethod == null) + if (!classInfo.HasExecutableCleanupMethod) { return null; } - var sequencingAttribute = GetCustomAttributes(classInfo.ClassCleanupMethod, true)?.FirstOrDefault(); + var cleanupBehaviors = + new HashSet( + classInfo.BaseClassCleanupMethodsStack + .Select(x => x.GetCustomAttribute(true)?.CleanupBehavior)) + { + classInfo.ClassCleanupMethod?.GetCustomAttribute(true)?.CleanupBehavior, + }; + + if (cleanupBehaviors.Contains(ClassCleanupBehavior.EndOfClass)) + { + return ClassCleanupBehavior.EndOfClass; + } - return sequencingAttribute?.CleanupBehavior; + if (cleanupBehaviors.Contains(ClassCleanupBehavior.EndOfAssembly)) + { + return ClassCleanupBehavior.EndOfAssembly; + } + + return null; } /// diff --git a/test/E2ETests/Smoke.E2E.Tests/SuiteLifeCycleTests.cs b/test/E2ETests/Smoke.E2E.Tests/SuiteLifeCycleTests.cs index e66180f368..b445cbece7 100644 --- a/test/E2ETests/Smoke.E2E.Tests/SuiteLifeCycleTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/SuiteLifeCycleTests.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.Linq; using FluentAssertions; @@ -24,9 +23,134 @@ public void ValidateTestRunLifecycle_net462() ValidateTestRunLifecycle("net462"); } + public void ValidateInheritanceBehavior() + { + InvokeVsTestForExecution( + new[] { "net462\\" + Assembly }, + testCaseFilter: "FullyQualifiedName~LifecycleInheritance", + targetFramework: "net462"); + + RunEventsHandler.PassedTests.Should().HaveCount(10); + + var testMethod1 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerived_EndOfClass.TestMethod")); + testMethod1.Messages[0].Text.Should().Be( + """ + Console: AssemblyInit was called + TestClassBaseEndOfClass: ClassInitialize + TestClassDerived_EndOfClass: TestMethod + TestClassBaseEndOfClass: ClassCleanup + + """); + + var testMethod2 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerived_EndOfAssembly.TestMethod")); + testMethod2.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfAssembly: ClassInitialize + TestClassDerived_EndOfAssembly: TestMethod + + """); + + var testMethod3 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerivedEndOfClass_EndOfClassEndOfClass.TestMethod")); + testMethod3.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfClass: ClassInitialize + TestClassIntermediateEndOfClassBaseEndOfClass: ClassInitialize + TestClassDerivedEndOfClass_EndOfClassEndOfClass: TestMethod + TestClassDerivedEndOfClass_EndOfClassEndOfClass: ClassCleanup + TestClassIntermediateEndOfClassBaseEndOfClass: ClassCleanup + TestClassBaseEndOfClass: ClassCleanup + + """); + + var testMethod4 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerived_EndOfClassEndOfClass.TestMethod")); + testMethod4.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfClass: ClassInitialize + TestClassIntermediateEndOfClassBaseEndOfClass: ClassInitialize + TestClassDerived_EndOfClassEndOfClass: TestMethod + TestClassIntermediateEndOfClassBaseEndOfClass: ClassCleanup + TestClassBaseEndOfClass: ClassCleanup + + """); + + var testMethod5 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerivedEndOfClass_EndOfClassEndOfAssembly.TestMethod")); + testMethod5.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfAssembly: ClassInitialize + TestClassIntermediateEndOfClassBaseEndOfAssembly: ClassInitialize + TestClassDerivedEndOfClass_EndOfClassEndOfAssembly: TestMethod + TestClassDerivedEndOfClass_EndOfClassEndOfAssembly: ClassCleanup + TestClassIntermediateEndOfClassBaseEndOfAssembly: ClassCleanup + TestClassBaseEndOfAssembly: ClassCleanup + + """); + + var testMethod6 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerived_EndOfClassEndOfAssembly.TestMethod")); + testMethod6.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfAssembly: ClassInitialize + TestClassIntermediateEndOfClassBaseEndOfAssembly: ClassInitialize + TestClassDerived_EndOfClassEndOfAssembly: TestMethod + TestClassIntermediateEndOfClassBaseEndOfAssembly: ClassCleanup + TestClassBaseEndOfAssembly: ClassCleanup + + """); + + var testMethod7 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerivedEndOfClass_EndOfAssemblyEndOfClass.TestMethod")); + testMethod7.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfClass: ClassInitialize + TestClassIntermediateEndOfAssemblyBaseEndOfClass: ClassInitialize + TestClassDerivedEndOfClass_EndOfAssemblyEndOfClass: TestMethod + TestClassDerivedEndOfClass_EndOfAssemblyEndOfClass: ClassCleanup + TestClassIntermediateEndOfAssemblyBaseEndOfClass: ClassCleanup + TestClassBaseEndOfClass: ClassCleanup + + """); + + var testMethod8 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerived_EndOfAssemblyEndOfClass.TestMethod")); + testMethod8.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfClass: ClassInitialize + TestClassIntermediateEndOfAssemblyBaseEndOfClass: ClassInitialize + TestClassDerived_EndOfAssemblyEndOfClass: TestMethod + TestClassIntermediateEndOfAssemblyBaseEndOfClass: ClassCleanup + TestClassBaseEndOfClass: ClassCleanup + + """); + + var testMethod9 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerivedEndOfClass_EndOfAssemblyEndOfAssembly.TestMethod")); + testMethod9.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfAssembly: ClassInitialize + TestClassIntermediateEndOfAssemblyBaseEndOfAssembly: ClassInitialize + TestClassDerivedEndOfClass_EndOfAssemblyEndOfAssembly: TestMethod + TestClassDerivedEndOfClass_EndOfAssemblyEndOfAssembly: ClassCleanup + TestClassIntermediateEndOfAssemblyBaseEndOfAssembly: ClassCleanup + TestClassBaseEndOfAssembly: ClassCleanup + + """); + + var testMethod10 = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.EndsWith("TestClassDerived_EndOfAssemblyEndOfAssembly.TestMethod")); + testMethod10.Messages[0].Text.Should().Be( + """ + TestClassBaseEndOfAssembly: ClassInitialize + TestClassIntermediateEndOfAssemblyBaseEndOfAssembly: ClassInitialize + TestClassDerived_EndOfAssemblyEndOfAssembly: TestMethod + TestClassIntermediateEndOfAssemblyBaseEndOfAssembly: ClassCleanup + TestClassBaseEndOfAssembly: ClassCleanup + TestClassBaseEndOfAssembly: ClassCleanup + Console: AssemblyCleanup was called + + """); + } + private void ValidateTestRunLifecycle(string targetFramework) { - InvokeVsTestForExecution(new[] { targetFramework + "\\" + Assembly }, targetFramework: targetFramework); + InvokeVsTestForExecution( + new[] { targetFramework + "\\" + Assembly }, + testCaseFilter: "FullyQualifiedName~SuiteLifeCycleTestProject", + targetFramework: targetFramework); RunEventsHandler.PassedTests.Should().HaveCount(27); // The inherit class tests are called twice. var caseClassCleanup = RunEventsHandler.PassedTests.Single(x => x.TestCase.FullyQualifiedName.Contains("LifeCycleClassCleanup.TestMethod")); diff --git a/test/E2ETests/TestAssets/SuiteLifeCycleTestProject/LifecycleInheritance.cs b/test/E2ETests/TestAssets/SuiteLifeCycleTestProject/LifecycleInheritance.cs new file mode 100644 index 0000000000..21e5a0c20b --- /dev/null +++ b/test/E2ETests/TestAssets/SuiteLifeCycleTestProject/LifecycleInheritance.cs @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LifecycleInheritance; + +[TestClass] +public class TestClassBaseEndOfClass +{ + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void BaseClassInit(TestContext testContext) + { + Console.WriteLine("TestClassBaseEndOfClass: ClassInitialize"); + } + + [ClassCleanup(InheritanceBehavior.BeforeEachDerivedClass, ClassCleanupBehavior.EndOfClass)] + public static void BaseClassCleanup() + { + Console.WriteLine("TestClassBaseEndOfClass: ClassCleanup"); + } +} + +[TestClass] +public class TestClassBaseEndOfAssembly +{ + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void BaseClassInit(TestContext testContext) + { + Console.WriteLine("TestClassBaseEndOfAssembly: ClassInitialize"); + } + + [ClassCleanup(InheritanceBehavior.BeforeEachDerivedClass, ClassCleanupBehavior.EndOfAssembly)] + public static void BaseClassCleanup() + { + Console.WriteLine("TestClassBaseEndOfAssembly: ClassCleanup"); + } +} + +[TestClass] +public class TestClassIntermediateEndOfClassBaseEndOfClass : TestClassBaseEndOfClass +{ + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void IntermediateClassInit(TestContext testContext) + { + Console.WriteLine("TestClassIntermediateEndOfClassBaseEndOfClass: ClassInitialize"); + } + + [ClassCleanup(InheritanceBehavior.BeforeEachDerivedClass, ClassCleanupBehavior.EndOfClass)] + public static void IntermediateClassCleanup() + { + Console.WriteLine("TestClassIntermediateEndOfClassBaseEndOfClass: ClassCleanup"); + } +} + +[TestClass] +public class TestClassIntermediateEndOfClassBaseEndOfAssembly : TestClassBaseEndOfAssembly +{ + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void IntermediateClassInit(TestContext testContext) + { + Console.WriteLine("TestClassIntermediateEndOfClassBaseEndOfAssembly: ClassInitialize"); + } + + [ClassCleanup(InheritanceBehavior.BeforeEachDerivedClass, ClassCleanupBehavior.EndOfClass)] + public static void IntermediateClassCleanup() + { + Console.WriteLine("TestClassIntermediateEndOfClassBaseEndOfAssembly: ClassCleanup"); + } +} + +[TestClass] +public class TestClassIntermediateEndOfAssemblyBaseEndOfAssembly : TestClassBaseEndOfAssembly +{ + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void IntermediateClassInit(TestContext testContext) + { + Console.WriteLine("TestClassIntermediateEndOfAssemblyBaseEndOfAssembly: ClassInitialize"); + } + + [ClassCleanup(InheritanceBehavior.BeforeEachDerivedClass, ClassCleanupBehavior.EndOfClass)] + public static void IntermediateClassCleanup() + { + Console.WriteLine("TestClassIntermediateEndOfAssemblyBaseEndOfAssembly: ClassCleanup"); + } +} + +[TestClass] +public class TestClassIntermediateEndOfAssemblyBaseEndOfClass : TestClassBaseEndOfClass +{ + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void IntermediateClassInit(TestContext testContext) + { + Console.WriteLine("TestClassIntermediateEndOfAssemblyBaseEndOfClass: ClassInitialize"); + } + + [ClassCleanup(InheritanceBehavior.BeforeEachDerivedClass, ClassCleanupBehavior.EndOfClass)] + public static void IntermediateClassCleanup() + { + Console.WriteLine("TestClassIntermediateEndOfAssemblyBaseEndOfClass: ClassCleanup"); + } +} + +[TestClass] +public class TestClassDerived_EndOfClass : TestClassBaseEndOfClass +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerived_EndOfClass: TestMethod"); + } +} + +[TestClass] +public class TestClassDerived_EndOfAssembly : TestClassBaseEndOfAssembly +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerived_EndOfAssembly: TestMethod"); + } +} + +[TestClass] +public class TestClassDerivedEndOfClass_EndOfClassEndOfClass : TestClassIntermediateEndOfClassBaseEndOfClass +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfClassEndOfClass: TestMethod"); + } + + [ClassCleanup(ClassCleanupBehavior.EndOfClass)] + public static void ClassCleanup() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfClassEndOfClass: ClassCleanup"); + } +} + +[TestClass] +public class TestClassDerived_EndOfClassEndOfClass : TestClassIntermediateEndOfClassBaseEndOfClass +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerived_EndOfClassEndOfClass: TestMethod"); + } +} + +[TestClass] +public class TestClassDerivedEndOfClass_EndOfClassEndOfAssembly : TestClassIntermediateEndOfClassBaseEndOfAssembly +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfClassEndOfAssembly: TestMethod"); + } + + [ClassCleanup(ClassCleanupBehavior.EndOfClass)] + public static void ClassCleanup() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfClassEndOfAssembly: ClassCleanup"); + } +} + +[TestClass] +public class TestClassDerived_EndOfClassEndOfAssembly : TestClassIntermediateEndOfClassBaseEndOfAssembly +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerived_EndOfClassEndOfAssembly: TestMethod"); + } +} + +[TestClass] +public class TestClassDerivedEndOfClass_EndOfAssemblyEndOfClass : TestClassIntermediateEndOfAssemblyBaseEndOfClass +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfAssemblyEndOfClass: TestMethod"); + } + + [ClassCleanup(ClassCleanupBehavior.EndOfClass)] + public static void ClassCleanup() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfAssemblyEndOfClass: ClassCleanup"); + } +} + +[TestClass] +public class TestClassDerived_EndOfAssemblyEndOfClass : TestClassIntermediateEndOfAssemblyBaseEndOfClass +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerived_EndOfAssemblyEndOfClass: TestMethod"); + } +} + +[TestClass] +public class TestClassDerivedEndOfClass_EndOfAssemblyEndOfAssembly : TestClassIntermediateEndOfAssemblyBaseEndOfAssembly +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfAssemblyEndOfAssembly: TestMethod"); + } + + [ClassCleanup(ClassCleanupBehavior.EndOfClass)] + public static void ClassCleanup() + { + Console.WriteLine("TestClassDerivedEndOfClass_EndOfAssemblyEndOfAssembly: ClassCleanup"); + } +} + +[TestClass] +public class TestClassDerived_EndOfAssemblyEndOfAssembly : TestClassIntermediateEndOfAssemblyBaseEndOfAssembly +{ + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestClassDerived_EndOfAssemblyEndOfAssembly: TestMethod"); + } +}