diff --git a/TestFx.sln b/TestFx.sln index 61f330377d..956ed1d772 100644 --- a/TestFx.sln +++ b/TestFx.sln @@ -178,6 +178,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CompatTestProject", "test\E EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestProjectForAssemblyResolution", "test\ComponentTests\TestAssets\TestProjectForAssemblyResolution\TestProjectForAssemblyResolution.csproj", "{0B057B99-DCDD-417A-BC19-3E63DDD86F24}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataRowTestProject", "test\E2ETests\TestAssets\DataRowTestProject\DataRowTestProject.csproj", "{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\TestFramework\Extension.Shared\Extension.Shared.projitems*{272ca5e1-8e81-4825-9e47-86cce02f700d}*SharedItemsImports = 13 @@ -1039,6 +1041,30 @@ Global {0B057B99-DCDD-417A-BC19-3E63DDD86F24}.Release|x64.Build.0 = Release|Any CPU {0B057B99-DCDD-417A-BC19-3E63DDD86F24}.Release|x86.ActiveCfg = Release|Any CPU {0B057B99-DCDD-417A-BC19-3E63DDD86F24}.Release|x86.Build.0 = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|ARM.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|ARM.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x64.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x64.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x86.ActiveCfg = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x86.Build.0 = Debug|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|Any CPU.Build.0 = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|ARM.ActiveCfg = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|ARM.Build.0 = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x64.ActiveCfg = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x64.Build.0 = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.ActiveCfg = Release|Any CPU + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1098,6 +1124,7 @@ Global {3FCE3987-7C8C-4E12-B54A-C6EC2D9FC7A6} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {2D2C5B73-F1F1-47C8-BC5C-A172E9BB3D16} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {0B057B99-DCDD-417A-BC19-3E63DDD86F24} = {1899187D-8B9C-40C2-9F04-9E9A76C9A919} + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9} diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs index 2fd1b39333..c44f9af324 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs @@ -161,7 +161,7 @@ internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, D continue; } - var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings.MapInconclusiveToFailed); + var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings); if (unitTestResult.DatarowIndex >= 0) { diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 4ac11553a2..fc7b8b57ed 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -303,7 +303,7 @@ internal UnitTestResult[] RunTestMethod() } else { - UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes(true)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); + UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes(false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); if (testDataSources != null && testDataSources.Length > 0) { diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs index 4a2b46639e..8c0ebd2a54 100644 --- a/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs @@ -14,9 +14,9 @@ internal static class UnitTestOutcomeHelper /// Converts the parameter unitTestOutcome to testOutcome /// /// The unit Test Outcome. - /// Should map inconclusive to failed. + /// Current MSTest settings /// The Test platforms outcome. - internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool mapInconclusiveToFailed) + internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, MSTestSettings currentSettings) { switch (unitTestOutcome) { @@ -29,14 +29,21 @@ internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool return TestOutcome.Failed; case UnitTestOutcome.NotRunnable: - return TestOutcome.None; + { + if (currentSettings.MapNotRunnableToFailed) + { + return TestOutcome.Failed; + } + + return TestOutcome.None; + } case UnitTestOutcome.Ignored: return TestOutcome.Skipped; case UnitTestOutcome.Inconclusive: { - if (mapInconclusiveToFailed) + if (currentSettings.MapInconclusiveToFailed) { return TestOutcome.Failed; } diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs index 4643581082..d6e679820a 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs @@ -51,6 +51,7 @@ public MSTestSettings() { this.CaptureDebugTraces = true; this.MapInconclusiveToFailed = false; + this.MapNotRunnableToFailed = false; this.EnableBaseClassTestMethodsFromOtherAssemblies = true; this.ForcedLegacyMode = false; this.TestSettingsFile = null; @@ -121,6 +122,11 @@ private set /// public bool MapInconclusiveToFailed { get; private set; } + /// + /// Gets a value indicating whether a not runnable result be mapped to failed test. + /// + public bool MapNotRunnableToFailed { get; private set; } + /// /// Gets a value indicating whether to enable discovery of test methods from base classes in a different assembly from the inheriting test class. /// @@ -159,6 +165,7 @@ public static void PopulateSettings(MSTestSettings settings) CurrentSettings.ForcedLegacyMode = settings.ForcedLegacyMode; CurrentSettings.TestSettingsFile = settings.TestSettingsFile; CurrentSettings.MapInconclusiveToFailed = settings.MapInconclusiveToFailed; + CurrentSettings.MapNotRunnableToFailed = settings.MapNotRunnableToFailed; CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies = settings.EnableBaseClassTestMethodsFromOtherAssemblies; CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers; CurrentSettings.ParallelizationScope = settings.ParallelizationScope; @@ -280,6 +287,7 @@ private static MSTestSettings ToSettings(XmlReader reader) // // true // false + // false // false // 5000 // @@ -350,6 +358,16 @@ private static MSTestSettings ToSettings(XmlReader reader) break; } + case "MAPNOTRUNNABLETOFAILED": + { + if (bool.TryParse(reader.ReadInnerXml(), out result)) + { + settings.MapNotRunnableToFailed = result; + } + + break; + } + case "SETTINGSFILE": { string fileName = reader.ReadInnerXml(); diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs index c8aae7fd21..43bee38e66 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs @@ -147,9 +147,9 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage) /// The test Case. /// The start Time. /// The end Time. - /// Indication to map inconclusive tests to failed. + /// Current MSTest settings. /// The . - internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, bool mapInconclusiveToFailed) + internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, MSTestSettings currentSettings) { Debug.Assert(testCase != null, "testCase"); @@ -159,7 +159,7 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da Duration = this.Duration, ErrorMessage = this.ErrorMessage, ErrorStackTrace = this.ErrorStackTrace, - Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, mapInconclusiveToFailed), + Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, currentSettings), StartTime = startTime, EndTime = endTime }; diff --git a/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs b/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs index 4bb97b58ce..46e7641eed 100644 --- a/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs +++ b/src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs @@ -118,11 +118,11 @@ public override DataRow DataRow } /// - public override IDictionary Properties + public override IDictionary Properties { get { - return this.properties as IDictionary; + return this.properties; } } diff --git a/src/TestFramework/Extension.Desktop/PrivateObject.cs b/src/TestFramework/Extension.Desktop/PrivateObject.cs index b4c633f929..6220b88e1c 100644 --- a/src/TestFramework/Extension.Desktop/PrivateObject.cs +++ b/src/TestFramework/Extension.Desktop/PrivateObject.cs @@ -127,6 +127,11 @@ public PrivateObject(Type type, Type[] parameterTypes, object[] args) catch (TargetInvocationException e) { Debug.Assert(e.InnerException != null, "Inner exception should not be null."); + if (e.InnerException != null) + { + throw e.InnerException; + } + throw; } } @@ -410,6 +415,11 @@ public object Invoke(string name, BindingFlags bindingFlags, Type[] parameterTyp catch (TargetInvocationException e) { Debug.Assert(e.InnerException != null, "Inner exception should not be null."); + if (e.InnerException != null) + { + throw e.InnerException; + } + throw; } } @@ -738,6 +748,11 @@ private object InvokeHelper(string name, BindingFlags bindingFlags, object[] arg catch (TargetInvocationException e) { Debug.Assert(e.InnerException != null, "Inner exception should not be null."); + if (e.InnerException != null) + { + throw e.InnerException; + } + throw; } } diff --git a/src/TestFramework/Extension.Desktop/PrivateType.cs b/src/TestFramework/Extension.Desktop/PrivateType.cs index 5300ebc908..65133bdbc8 100644 --- a/src/TestFramework/Extension.Desktop/PrivateType.cs +++ b/src/TestFramework/Extension.Desktop/PrivateType.cs @@ -208,6 +208,11 @@ public object InvokeStatic(string name, BindingFlags bindingFlags, Type[] parame catch (TargetInvocationException e) { Debug.Assert(e.InnerException != null, "Inner Exception should not be null."); + if (e.InnerException != null) + { + throw e.InnerException; + } + throw; } } @@ -508,6 +513,11 @@ private object InvokeHelperStatic(string name, BindingFlags bindingFlags, object catch (TargetInvocationException e) { Debug.Assert(e.InnerException != null, "Inner Exception should not be null."); + if (e.InnerException != null) + { + throw e.InnerException; + } + throw; } } diff --git a/src/TestFramework/Extension.Desktop/TestContext.cs b/src/TestFramework/Extension.Desktop/TestContext.cs index bd5d4b8fcd..2cefc01d8f 100644 --- a/src/TestFramework/Extension.Desktop/TestContext.cs +++ b/src/TestFramework/Extension.Desktop/TestContext.cs @@ -4,7 +4,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting { using System; - using System.Collections; + using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Diagnostics; @@ -19,7 +19,7 @@ public abstract class TestContext /// /// Gets test properties for a test. /// - public abstract IDictionary Properties { get; } + public abstract IDictionary Properties { get; } /// /// Gets the current data row when test is used for data driven testing. diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index 2dc761445b..42b9cad4a9 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -117,12 +117,16 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList) /// Provide the full test name similar to this format SampleTest.TestCode.TestMethodPass. public void ValidatePassedTests(params string[] passedTests) { - // Make sure only expected number of tests passed and not more. - Assert.AreEqual(passedTests.Length, this.runEventsHandler.PassedTests.Count); - + this.ValidatePassedTestsCount(passedTests.Length); this.ValidatePassedTestsContain(passedTests); } + public void ValidatePassedTestsCount(int expectedPassedTestsCount) + { + // Make sure only expected number of tests passed and not more. + Assert.AreEqual(expectedPassedTestsCount, this.runEventsHandler.PassedTests.Count); + } + /// /// Validates if the test results have the specified set of failed tests. /// diff --git a/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs b/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs new file mode 100644 index 0000000000..6967befb57 --- /dev/null +++ b/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace MSTestAdapter.Smoke.E2ETests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class DataRowTests : CLITestBase + { + private const string TestAssembly = "DataRowTestProject.dll"; + + [TestMethod] + public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerviedClassHasDataRows() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }); + + this.ValidatePassedTestsContain( + "DataRowTestMethod (BaseString1)", + "DataRowTestMethod (BaseString2)", + "DataRowTestMethod (BaseString3)", + "DataRowTestMethod (DerivedString1)", + "DataRowTestMethod (DerivedString2)"); + + // 4 tests of BaseClass - 3 datarow result and 1 parent result + // 3 tests of DerivedClass - 2 datarow result and 1 parent result + // Total 7 tests - Making sure that DerivedClass doesn't run BaseClass tests + this.ValidatePassedTestsCount(7); + } + + [TestMethod] + public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~DerivedClass"); + + this.ValidatePassedTestsContain( + "DataRowTestMethod (DerivedString1)", + "DataRowTestMethod (DerivedString2)"); + + // 3 tests of DerivedClass - 2 datarow result and 1 parent result + this.ValidatePassedTestsCount(3); + } + } +} diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj index 4078045120..ffc75ae641 100644 --- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj +++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj @@ -48,6 +48,7 @@ + diff --git a/test/E2ETests/TestAssets/DataRowTestProject/BaseClass.cs b/test/E2ETests/TestAssets/DataRowTestProject/BaseClass.cs new file mode 100644 index 0000000000..75183e05a5 --- /dev/null +++ b/test/E2ETests/TestAssets/DataRowTestProject/BaseClass.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DataRowTestProject +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class BaseClass + { + [TestMethod] + [DataRow("BaseString1")] + [DataRow("BaseString2")] + [DataRow("BaseString3")] + public virtual void DataRowTestMethod(string a) + { + Assert.IsTrue(true); + } + } +} diff --git a/test/E2ETests/TestAssets/DataRowTestProject/DataRowTestProject.csproj b/test/E2ETests/TestAssets/DataRowTestProject/DataRowTestProject.csproj new file mode 100644 index 0000000000..56d97e8d71 --- /dev/null +++ b/test/E2ETests/TestAssets/DataRowTestProject/DataRowTestProject.csproj @@ -0,0 +1,23 @@ + + + + ..\..\..\..\ + + + + net452 + false + false + $(TestFxRoot)artifacts\TestAssets\ + + + + + + + + + + + + diff --git a/test/E2ETests/TestAssets/DataRowTestProject/DerivedClass.cs b/test/E2ETests/TestAssets/DataRowTestProject/DerivedClass.cs new file mode 100644 index 0000000000..0ecc6ecf05 --- /dev/null +++ b/test/E2ETests/TestAssets/DataRowTestProject/DerivedClass.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DataRowTestProject +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class DerivedClass : BaseClass + { + [TestMethod] + [DataRow("DerivedString1")] + [DataRow("DerivedString2")] + public override void DataRowTestMethod(string a) + { + Assert.IsTrue(true); + } + } +} diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs index 8b0039f150..515a8f25f9 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/UnitTestResultTest.cs @@ -10,6 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution using System; using System.Linq; using System.Reflection; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -64,8 +65,16 @@ public void ToTestResultShouldReturnConvertedTestResultWithFieldsSet() var startTime = DateTimeOffset.Now; var endTime = DateTimeOffset.Now; + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + // Act - var testResult = result.ToTestResult(testCase, startTime, endTime, mapInconclusiveToFailed: false); + var testResult = result.ToTestResult(testCase, startTime, endTime, adapterSettings); // Validate Assert.AreEqual(testCase, testResult.TestCase); @@ -87,7 +96,15 @@ public void ToTestResultForUniTestResultWithStandardOutShouldReturnTestResultWit StandardOut = "DummyOutput" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, mapInconclusiveToFailed: false); + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyOutput") && m.Category.Equals("StdOutMsgs"))); } @@ -99,7 +116,14 @@ public void ToTestResultForUniTestResultWithDebugTraceShouldReturnTestResultWith DebugTrace = "DummyDebugTrace" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, mapInconclusiveToFailed: false); + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nDebug Trace:\nDummyDebugTrace") && m.Category.Equals("StdOutMsgs"))); } } diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/ReflectHelperTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/ReflectHelperTests.cs index 884e2d6c6b..c37d2d432a 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/ReflectHelperTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/ReflectHelperTests.cs @@ -69,12 +69,32 @@ public void GetTestCategoryAttributeShouldIncludeTestCategoriesAtClassLevel() [TestMethod] public void GetTestCategoryAttributeShouldIncludeTestCategoriesAtAllLevels() { - this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("AsmLevel") }, MemberTypes.All); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("AsmLevel1"), new UTF.TestCategoryAttribute("AsmLevel2") }, MemberTypes.All); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("AsmLevel3") }, MemberTypes.All); this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("ClassLevel") }, MemberTypes.TypeInfo); this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("MethodLevel") }, MemberTypes.Method); var actual = this.reflectHelper.GetCategories(this.method.Object).ToArray(); - string[] expected = new[] { "MethodLevel", "ClassLevel", "AsmLevel" }; + string[] expected = new[] { "MethodLevel", "ClassLevel", "AsmLevel1", "AsmLevel2", "AsmLevel3" }; + + CollectionAssert.AreEqual(expected, actual); + } + + /// + /// Testing test category attributes adorned at calss, assembly and method level are getting collected. + /// + [TestMethod] + public void GetTestCategoryAttributeShouldConcatCustomAttributeOfSameType() + { + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("AsmLevel1") }, MemberTypes.All); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("AsmLevel2") }, MemberTypes.All); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("ClassLevel1") }, MemberTypes.TypeInfo); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("ClassLevel2") }, MemberTypes.TypeInfo); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("MethodLevel1") }, MemberTypes.Method); + this.reflectHelper.SetCustomAttribute(typeof(UTF.TestCategoryBaseAttribute), new[] { new UTF.TestCategoryAttribute("MethodLevel2") }, MemberTypes.Method); + + var actual = this.reflectHelper.GetCategories(this.method.Object).ToArray(); + string[] expected = new[] { "MethodLevel1", "MethodLevel2", "ClassLevel1", "ClassLevel2", "AsmLevel1", "AsmLevel2" }; CollectionAssert.AreEqual(expected, actual); } diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs index 91b70b3609..8c7f4c89bd 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Helpers/UnitTestOutcomeHelperTests.cs @@ -6,79 +6,92 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Helpers extern alias FrameworkV1; extern alias FrameworkV2; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Assert = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.Assert; using TestClass = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute; + using TestInitialize = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestInitializeAttribute; using TestMethod = FrameworkV1::Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute; - using UTF = FrameworkV2::Microsoft.VisualStudio.TestTools.UnitTesting; - [TestClass] public class UnitTestOutcomeHelperTests { + private MSTestSettings adapterSettings; + + [TestInitialize] + public void TestInit() + { + string runSettingxml = + @" + + + "; + + this.adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + } + [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, this.adapterSettings); Assert.AreEqual(TestOutcome.Passed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeFailedShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, this.adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeErrorShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, this.adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNone() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, this.adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeTimeoutShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, this.adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeIgnoredShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, this.adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, this.adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotFoundShouldReturnTestOutcomeNotFound() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, this.adapterSettings); Assert.AreEqual(TestOutcome.NotFound, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInProgressShouldReturnTestOutcomeNone() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, mapInconclusiveToFailed: false); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, this.adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } } diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs index 3704926ba5..36f292f1c2 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs @@ -64,6 +64,20 @@ public void MapInconclusiveToFailedIsByDefaultFalseWhenNotSpecified() Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); } + [TestMethod] + public void MapNotRunnableToFailedIsByDefaultFalseWhenNotSpecified() + { + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); + } + [TestMethod] public void MapInconclusiveToFailedShouldBeConsumedFromRunSettingsWhenSpecified() { @@ -79,6 +93,21 @@ public void MapInconclusiveToFailedShouldBeConsumedFromRunSettingsWhenSpecified( Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); } + [TestMethod] + public void MapNotRunnableToFailedShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + True + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); + } + [TestMethod] public void ForcedLegacyModeIsByDefaultFalseWhenNotSpecified() { @@ -566,6 +595,7 @@ public void GetSettingsShouldBeAbleToReadSettingsAfterThePlatformServiceReadsIts @" True + True True DummyPath\\TestSettings1.testsettings @@ -608,6 +638,7 @@ public void GetSettingsShouldBeAbleToReadSettingsAfterThePlatformServiceReadsIts // Assert. Assert.IsTrue(dummyPlatformSpecificSetting); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual("DummyPath\\\\TestSettings1.testsettings", adapterSettings.TestSettingsFile); } @@ -618,6 +649,7 @@ public void GetSettingsShouldBeAbleToReadSettingsIfThePlatformServiceDoesNotUnde @" True + True foobar true true @@ -663,6 +695,7 @@ public void GetSettingsShouldBeAbleToReadSettingsIfThePlatformServiceDoesNotUnde // Assert. Assert.IsTrue(dummyPlatformSpecificSetting); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.AreEqual("DummyPath\\\\TestSettings1.testsettings", adapterSettings.TestSettingsFile); @@ -724,6 +757,7 @@ public void GetSettingsShouldWorkIfThereAreCommentsInTheXML() True + True True true @@ -770,6 +804,7 @@ public void GetSettingsShouldWorkIfThereAreCommentsInTheXML() // Assert. Assert.IsTrue(dummyPlatformSpecificSetting); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -825,6 +860,7 @@ public void PopulateSettingsShouldFillInSettingsFromSettingsObject() False True + True DummyPath\\TestSettings1.testsettings true true @@ -837,6 +873,7 @@ public void PopulateSettingsShouldFillInSettingsFromSettingsObject() Assert.AreEqual(MSTestSettings.CurrentSettings.CaptureDebugTraces, false); Assert.AreEqual(MSTestSettings.CurrentSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(MSTestSettings.CurrentSettings.MapNotRunnableToFailed, true); Assert.AreEqual(MSTestSettings.CurrentSettings.ForcedLegacyMode, true); Assert.AreEqual(MSTestSettings.CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.IsFalse(string.IsNullOrEmpty(MSTestSettings.CurrentSettings.TestSettingsFile)); @@ -850,6 +887,7 @@ public void PopulateSettingsShouldInitializeDefaultAdapterSettingsWhenDiscoveryC MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -861,6 +899,7 @@ public void PopulateSettingsShouldInitializeDefaultSettingsWhenRunSettingsIsNull MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -873,6 +912,7 @@ public void PopulateSettingsShouldInitializeDefaultSettingsWhenRunSettingsXmlIsE MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, false); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, false); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); } @@ -905,6 +945,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestSection() @" True + True DummyPath\\TestSettings1.testsettings true true @@ -920,6 +961,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestSection() Assert.IsNotNull(adapterSettings); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.IsFalse(string.IsNullOrEmpty(adapterSettings.TestSettingsFile)); @@ -932,6 +974,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2Section() @" True + True DummyPath\\TestSettings1.testsettings true true @@ -947,6 +990,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2Section() Assert.IsNotNull(adapterSettings); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.IsFalse(string.IsNullOrEmpty(adapterSettings.TestSettingsFile)); @@ -959,6 +1003,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2OverMSTestV1Sect @" True + True true @@ -977,6 +1022,7 @@ public void PopulateSettingsShouldInitializeSettingsFromMSTestV2OverMSTestV1Sect Assert.IsNotNull(adapterSettings); Assert.AreEqual(adapterSettings.MapInconclusiveToFailed, true); + Assert.AreEqual(adapterSettings.MapNotRunnableToFailed, true); Assert.AreEqual(adapterSettings.EnableBaseClassTestMethodsFromOtherAssemblies, true); Assert.AreEqual(adapterSettings.ForcedLegacyMode, false); Assert.AreEqual(adapterSettings.CaptureDebugTraces, true); diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs index f6ce578efd..ef6db77911 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/ObjectModel/UnitTestResultTests.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.ObjectMode using System.Collections.Generic; using System.Linq; using System.Reflection; - + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -64,8 +64,16 @@ public void ToTestResultShouldReturnConvertedTestResultWithFieldsSet() var startTime = DateTimeOffset.Now; var endTime = DateTimeOffset.Now; + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + // Act - var testResult = result.ToTestResult(testCase, startTime, endTime, false); + var testResult = result.ToTestResult(testCase, startTime, endTime, adapterSettings); // Validate Assert.AreEqual(testCase, testResult.TestCase); @@ -87,7 +95,16 @@ public void ToTestResultForUniTestResultWithStandardOutShouldReturnTestResultWit StandardOut = "DummyOutput" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyOutput") && m.Category.Equals("StdOutMsgs"))); } @@ -99,7 +116,16 @@ public void ToTestResultForUniTestResultWithStandardErrorShouldReturnTestResultW StandardError = "DummyError" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("DummyError") && m.Category.Equals("StdErrMsgs"))); } @@ -111,7 +137,16 @@ public void ToTestResultForUniTestResultWithDebugTraceShouldReturnTestResultWith DebugTrace = "DummyDebugTrace" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nDebug Trace:\nDummyDebugTrace") && m.Category.Equals("StdOutMsgs"))); } @@ -123,7 +158,16 @@ public void ToTestResultForUniTestResultWithTestContextMessagesShouldReturnTestR TestContextMessages = "KeepMovingForward" }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.IsTrue(testresult.Messages.All(m => m.Text.Contains("\n\nTestContext Messages:\nKeepMovingForward") && m.Category.Equals("StdOutMsgs"))); } @@ -135,7 +179,16 @@ public void ToTestResultForUniTestResultWithResultFilesShouldReturnTestResultWit ResultFiles = new List() { "dummy://DummyFile.txt" } }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.AreEqual(testresult.Attachments.Count, 1); Assert.AreEqual(testresult.Attachments[0].Attachments[0].Description, "dummy://DummyFile.txt"); @@ -149,7 +202,16 @@ public void ToTestResultForUniTestResultWithNoResultFilesShouldReturnTestResultW ResultFiles = null }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.AreEqual(testresult.Attachments.Count, 0); } @@ -168,7 +230,16 @@ public void ToTestResultForUniTestResultWithParentInfoShouldReturnTestResultWith InnerResultsCount = innerResultsCount }; TestCase testCase = new TestCase("Foo", new Uri("Uri", UriKind.Relative), Assembly.GetExecutingAssembly().FullName); - var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, false); + + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var testresult = result.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, adapterSettings); Assert.AreEqual(executionId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ExecutionIdProperty)); Assert.AreEqual(parentExecId, testresult.GetPropertyValue(MSTest.TestAdapter.Constants.ParentExecIdProperty)); @@ -178,71 +249,167 @@ public void ToTestResultForUniTestResultWithParentInfoShouldReturnTestResultWith [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomePassedShouldReturnTestOutcomePassed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Passed, adapterSettings); Assert.AreEqual(TestOutcome.Passed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeFailedShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Failed, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeErrorShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Error, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] - public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNone() + public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeNoneWhenNotSpecified() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeTimeoutShouldReturnTestOutcomeFailed() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Timeout, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeIgnoredShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Ignored, adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeSkipped() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, adapterSettings); Assert.AreEqual(TestOutcome.Skipped, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInconclusiveShouldReturnTestOutcomeFailedWhenSpecifiedSo() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, true); + string runSettingxml = + @" + + true + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.Inconclusive, adapterSettings); Assert.AreEqual(TestOutcome.Failed, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotFoundShouldReturnTestOutcomeNotFound() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotFound, adapterSettings); Assert.AreEqual(TestOutcome.NotFound, resultOutcome); } [TestMethod] public void UniTestHelperToTestOutcomeForUnitTestOutcomeInProgressShouldReturnTestOutcomeNone() { - var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, false); + string runSettingxml = + @" + + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.InProgress, adapterSettings); Assert.AreEqual(TestOutcome.None, resultOutcome); } + + [TestMethod] + public void UniTestHelperToTestOutcomeForUnitTestOutcomeNotRunnableShouldReturnTestOutcomeFailedWhenSpecifiedSo() + { + string runSettingxml = + @" + + true + + "; + + var adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + var resultOutcome = UnitTestOutcomeHelper.ToTestOutcome(UnitTestOutcome.NotRunnable, adapterSettings); + Assert.AreEqual(TestOutcome.Failed, resultOutcome); + } } } diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/TestableImplementations/TestableReflectHelper.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/TestableImplementations/TestableReflectHelper.cs index c09e9dba37..3f38b01e81 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/TestableImplementations/TestableReflectHelper.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/TestableImplementations/TestableReflectHelper.cs @@ -33,7 +33,7 @@ public void SetCustomAttribute(Type type, Attribute[] values, MemberTypes member var hashcode = type.FullName.GetHashCode() + memberTypes.GetHashCode(); if (this.customAttributes.ContainsKey(hashcode)) { - this.customAttributes[hashcode].Concat(values); + this.customAttributes[hashcode] = this.customAttributes[hashcode].Concat(values).ToArray(); } else { diff --git a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopTestContextImplTests.cs b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopTestContextImplTests.cs index beb84028a2..005b9c1c3d 100644 --- a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopTestContextImplTests.cs +++ b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/Services/DesktopTestContextImplTests.cs @@ -59,10 +59,10 @@ public void TestContextConstructorShouldInitializeDefaultProperties() Assert.IsNotNull(this.testContextImplementation.Properties); CollectionAssert.Contains( - this.testContextImplementation.Properties, + this.testContextImplementation.Properties.ToList(), new KeyValuePair("FullyQualifiedTestClassName", "A.C.M")); CollectionAssert.Contains( - this.testContextImplementation.Properties, + this.testContextImplementation.Properties.ToList(), new KeyValuePair("TestName", "M")); } @@ -115,8 +115,8 @@ public void PropertiesShouldReturnPropertiesPassedToTestContext() this.testContextImplementation = new TestContextImplementation(this.testMethod.Object, new System.IO.StringWriter(), this.properties); - CollectionAssert.Contains(this.testContextImplementation.Properties, property1); - CollectionAssert.Contains(this.testContextImplementation.Properties, property2); + CollectionAssert.Contains(this.testContextImplementation.Properties.ToList(), property1); + CollectionAssert.Contains(this.testContextImplementation.Properties.ToList(), property2); } [TestMethod] @@ -162,7 +162,7 @@ public void AddPropertyShouldAddPropertiesToThePropertyBag() this.testContextImplementation.AddProperty("SomeNewProperty", "SomeValue"); CollectionAssert.Contains( - this.testContextImplementation.Properties, + this.testContextImplementation.Properties.ToList(), new KeyValuePair("SomeNewProperty", "SomeValue")); }