diff --git a/TestFx.sln b/TestFx.sln index 12a5f7a6cf..6441825168 100644 --- a/TestFx.sln +++ b/TestFx.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26228.9 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29728.190 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{FF8B1B72-55A1-4FFE-809E-7B79323ED8D0}" EndProject @@ -185,6 +185,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeploymentTestProjectNetCor EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProjectNetCore", "test\E2ETests\TestAssets\TimeoutTestProjectNetCore\TimeoutTestProjectNetCore.csproj", "{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharpTestProject", "test\E2ETests\TestAssets\FSharpTestProject\FSharpTestProject.fsproj", "{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\TestFramework\Extension.Shared\Extension.Shared.projitems*{272ca5e1-8e81-4825-9e47-86cce02f700d}*SharedItemsImports = 13 @@ -1141,6 +1143,30 @@ Global {ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x64.Build.0 = Release|Any CPU {ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x86.ActiveCfg = Release|Any CPU {ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x86.Build.0 = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|ARM.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|ARM.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x64.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x64.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x86.ActiveCfg = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x86.Build.0 = Debug|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|Any CPU.Build.0 = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|ARM.ActiveCfg = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|ARM.Build.0 = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.ActiveCfg = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.Build.0 = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.ActiveCfg = Release|Any CPU + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1204,6 +1230,7 @@ Global {4F0B2ACF-1341-42AF-918C-669A6D5CEA2B} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {26F0B8EF-92D4-4A23-ACB4-D1B662F0EEBE} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {ED27A844-6870-4FE6-8FEF-3ABDD1ED6564} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9} diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs index 4f9099c265..8b7e7f994f 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs @@ -24,8 +24,13 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string var testClassName = testCase.GetPropertyValue(Constants.TestClassNameProperty) as string; var declaringClassName = testCase.GetPropertyValue(Constants.DeclaringClassNameProperty) as string; - var parts = testCase.FullyQualifiedName.Split('.'); - var name = parts[parts.Length - 1]; + var fullyQualifiedName = testCase.FullyQualifiedName; + + // Not using Replace because there can be multiple instances of that string. + var name = fullyQualifiedName.StartsWith($"{testClassName}.") + ? fullyQualifiedName.Remove(0, $"{testClassName}.".Length) + : fullyQualifiedName; + TestMethod testMethod = new TestMethod(name, testClassName, source, isAsync); if (declaringClassName != null && declaringClassName != testClassName) diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index fedd165ab4..206b76b60c 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -75,6 +75,10 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "", // this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll")); vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler); + if (this.runEventsHandler.Errors.Any()) + { + throw new Exception($"Run failed with {this.runEventsHandler.Errors.Count} errors:{Environment.NewLine}{string.Join(Environment.NewLine, this.runEventsHandler.Errors)}"); + } } /// @@ -101,7 +105,7 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList) { var flag = this.discoveryEventsHandler.Tests.Contains(test) || this.discoveryEventsHandler.Tests.Contains(GetTestMethodName(test)); - Assert.IsTrue(flag, "Test {0} does not appear in discovered tests list.", test); + Assert.IsTrue(flag, "Test '{0}' does not appear in discovered tests list.", test); } // Make sure only expected number of tests are discovered and not more. @@ -170,12 +174,15 @@ public void ValidateSkippedTests(params string[] skippedTests) /// Provide the full test name similar to this format SampleTest.TestCode.TestMethodPass. public void ValidatePassedTestsContain(params string[] passedTests) { + var passedTestResults = this.runEventsHandler.PassedTests.ToList(); foreach (var test in passedTests) { - var testFound = this.runEventsHandler.PassedTests.Any( + var testFound = passedTestResults.Any( p => test.Equals(p.TestCase?.FullyQualifiedName) - || test.Equals(p.DisplayName)); - Assert.IsTrue(testFound, "Test {0} does not appear in passed tests list.", test); + || test.Equals(p.DisplayName) + || test.Equals(p.TestCase.DisplayName)); + + Assert.IsTrue(testFound, "Test '{0}' does not appear in passed tests list.", test); } } @@ -195,7 +202,7 @@ public void ValidateFailedTestsContain(string source, bool validateStackTraceInf { var testFound = this.runEventsHandler.FailedTests.FirstOrDefault(f => test.Equals(f.TestCase?.FullyQualifiedName) || test.Equals(f.DisplayName)); - Assert.IsNotNull(testFound, "Test {0} does not appear in failed tests list.", test); + Assert.IsNotNull(testFound, "Test '{0}' does not appear in failed tests list.", test); // Skipping this check for x64 as of now. https://github.com/Microsoft/testfx/issues/60 should fix this. if (source.IndexOf("x64") == -1 && validateStackTraceInfo) @@ -222,7 +229,7 @@ public void ValidateSkippedTestsContain(params string[] skippedTests) { var testFound = this.runEventsHandler.SkippedTests.Any(s => test.Equals(s.TestCase.FullyQualifiedName) || test.Equals(s.DisplayName)); - Assert.IsTrue(testFound, "Test {0} does not appear in skipped tests list.", test); + Assert.IsTrue(testFound, "Test '{0}' does not appear in skipped tests list.", test); } } diff --git a/test/E2ETests/Automation.CLI/RunEventsHandler.cs b/test/E2ETests/Automation.CLI/RunEventsHandler.cs index 042dc97900..325789ae67 100644 --- a/test/E2ETests/Automation.CLI/RunEventsHandler.cs +++ b/test/E2ETests/Automation.CLI/RunEventsHandler.cs @@ -25,6 +25,8 @@ public class RunEventsHandler : ITestRunEventsHandler /// public IList SkippedTests { get; private set; } + public IList Errors { get; private set; } + public double ElapsedTimeInRunningTests { get; private set; } public RunEventsHandler() @@ -32,6 +34,7 @@ public RunEventsHandler() this.PassedTests = new List(); this.FailedTests = new List(); this.SkippedTests = new List(); + this.Errors = new List(); } public void HandleLogMessage(TestMessageLevel level, string message) @@ -45,6 +48,7 @@ public void HandleLogMessage(TestMessageLevel level, string message) EqtTrace.Warning(message); break; case TestMessageLevel.Error: + this.Errors.Add(message); EqtTrace.Error(message); break; default: diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj index cf9db9e8d6..bc5ac4b3c6 100644 --- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj +++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj @@ -41,6 +41,7 @@ + diff --git a/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs b/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs new file mode 100644 index 0000000000..48d5b906c9 --- /dev/null +++ b/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace MSTestAdapter.Smoke.E2ETests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class TestProjectFSharpTests : CLITestBase + { + private const string TestAssembly = "FSharpTestProject.dll"; + + [TestMethod] + public void ExecuteCustomTestExtensibilityTests() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }); + this.ValidateFailedTestsCount(0); + this.ValidatePassedTestsCount(1); + this.ValidatePassedTestsContain("Test method passing with a . in it"); + } + } +} \ No newline at end of file diff --git a/test/E2ETests/TestAssets/FSharpTestProject/FSharpTestProject.fsproj b/test/E2ETests/TestAssets/FSharpTestProject/FSharpTestProject.fsproj new file mode 100644 index 0000000000..f0bf4d7c61 --- /dev/null +++ b/test/E2ETests/TestAssets/FSharpTestProject/FSharpTestProject.fsproj @@ -0,0 +1,24 @@ + + + + ..\..\..\..\ + + + + net452 + false + false + false + $(TestFxRoot)artifacts\TestAssets\ + + + + + + + + + + + + diff --git a/test/E2ETests/TestAssets/FSharpTestProject/Tests.fs b/test/E2ETests/TestAssets/FSharpTestProject/Tests.fs new file mode 100644 index 0000000000..b50b2ac6fb --- /dev/null +++ b/test/E2ETests/TestAssets/FSharpTestProject/Tests.fs @@ -0,0 +1,11 @@ +namespace FSharpTestProject + +open System +open Microsoft.VisualStudio.TestTools.UnitTesting + +[] +type UnitTest1 () = + + [] + member this.``Test method passing with a . in it`` () = + Assert.IsTrue(true); diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestCaseExtensionsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestCaseExtensionsTests.cs index 2a7a7d9a40..01a864ad02 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestCaseExtensionsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestCaseExtensionsTests.cs @@ -20,7 +20,7 @@ public class TestCaseExtensionsTests [TestMethod] public void ToUnitTestElementShouldReturnUnitTestElementWithFieldsSet() { - TestCase testCase = new TestCase("DummyClass.DummyMethod", new Uri("DummyUri", UriKind.Relative), Assembly.GetCallingAssembly().FullName); + TestCase testCase = new TestCase("DummyClassName.DummyMethod", new Uri("DummyUri", UriKind.Relative), Assembly.GetCallingAssembly().FullName); testCase.DisplayName = "DummyDisplayName"; var testCategories = new[] { "DummyCategory" };