Merge branch 'master' into depItemNetCore

TestFx.sln

@@ -178,6 +178,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CompatTestProject", "test\E
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestProjectForAssemblyResolution", "test\ComponentTests\TestAssets\TestProjectForAssemblyResolution\TestProjectForAssemblyResolution.csproj", "{0B057B99-DCDD-417A-BC19-3E63DDD86F24}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataRowTestProject", "test\E2ETests\TestAssets\DataRowTestProject\DataRowTestProject.csproj", "{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}"
+EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
 		src\TestFramework\Extension.Shared\Extension.Shared.projitems*{272ca5e1-8e81-4825-9e47-86cce02f700d}*SharedItemsImports = 13
@@ -1039,6 +1041,30 @@ Global
 		{0B057B99-DCDD-417A-BC19-3E63DDD86F24}.Release|x64.Build.0 = Release|Any CPU
 		{0B057B99-DCDD-417A-BC19-3E63DDD86F24}.Release|x86.ActiveCfg = Release|Any CPU
 		{0B057B99-DCDD-417A-BC19-3E63DDD86F24}.Release|x86.Build.0 = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|ARM.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x64.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Debug|x86.Build.0 = Debug|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|ARM.ActiveCfg = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|ARM.Build.0 = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x64.ActiveCfg = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x64.Build.0 = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.ActiveCfg = Release|Any CPU
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1098,6 +1124,7 @@ Global
 		{3FCE3987-7C8C-4E12-B54A-C6EC2D9FC7A6} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
 		{2D2C5B73-F1F1-47C8-BC5C-A172E9BB3D16} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
 		{0B057B99-DCDD-417A-BC19-3E63DDD86F24} = {1899187D-8B9C-40C2-9F04-9E9A76C9A919}
+		{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}

src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs

@@ -161,7 +161,7 @@ internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, D
                     continue;
                 }
 
-                var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings.MapInconclusiveToFailed);
+                var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
 
                 if (unitTestResult.DatarowIndex >= 0)
                 {

src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs

@@ -303,7 +303,7 @@ internal UnitTestResult[] RunTestMethod()
                 }
                 else
                 {
-                    UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes<Attribute>(true)?.Where(a => a is UTF.ITestDataSource).OfType<UTF.ITestDataSource>().ToArray();
+                    UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes<Attribute>(false)?.Where(a => a is UTF.ITestDataSource).OfType<UTF.ITestDataSource>().ToArray();
 
                     if (testDataSources != null && testDataSources.Length > 0)
                     {

src/Adapter/MSTest.CoreAdapter/Helpers/UnitTestOutcomeHelper.cs

@@ -14,9 +14,9 @@ internal static class UnitTestOutcomeHelper
         /// Converts the parameter unitTestOutcome to testOutcome
         /// </summary>
         /// <param name="unitTestOutcome"> The unit Test Outcome. </param>
-        /// <param name="mapInconclusiveToFailed">Should map inconclusive to failed.</param>
+        /// <param name="currentSettings">Current MSTest settings</param>
         /// <returns>The Test platforms outcome.</returns>
-        internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool mapInconclusiveToFailed)
+        internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, MSTestSettings currentSettings)
         {
             switch (unitTestOutcome)
             {
@@ -29,14 +29,21 @@ internal static TestOutcome ToTestOutcome(UnitTestOutcome unitTestOutcome, bool
                     return TestOutcome.Failed;
 
                 case UnitTestOutcome.NotRunnable:
-                    return TestOutcome.None;
+                    {
+                        if (currentSettings.MapNotRunnableToFailed)
+                        {
+                            return TestOutcome.Failed;
+                        }
+
+                        return TestOutcome.None;
+                    }
 
                 case UnitTestOutcome.Ignored:
                     return TestOutcome.Skipped;
 
                 case UnitTestOutcome.Inconclusive:
                     {
-                        if (mapInconclusiveToFailed)
+                        if (currentSettings.MapInconclusiveToFailed)
                         {
                             return TestOutcome.Failed;
                         }

src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs

@@ -51,6 +51,7 @@ public MSTestSettings()
         {
             this.CaptureDebugTraces = true;
             this.MapInconclusiveToFailed = false;
+            this.MapNotRunnableToFailed = false;
             this.EnableBaseClassTestMethodsFromOtherAssemblies = true;
             this.ForcedLegacyMode = false;
             this.TestSettingsFile = null;
@@ -121,6 +122,11 @@ private set
         /// </summary>
         public bool MapInconclusiveToFailed { get; private set; }
 
+        /// <summary>
+        /// Gets a value indicating whether a not runnable result be mapped to failed test.
+        /// </summary>
+        public bool MapNotRunnableToFailed { get; private set; }
+
         /// <summary>
         /// Gets a value indicating whether to enable discovery of test methods from base classes in a different assembly from the inheriting test class.
         /// </summary>
@@ -159,6 +165,7 @@ public static void PopulateSettings(MSTestSettings settings)
             CurrentSettings.ForcedLegacyMode = settings.ForcedLegacyMode;
             CurrentSettings.TestSettingsFile = settings.TestSettingsFile;
             CurrentSettings.MapInconclusiveToFailed = settings.MapInconclusiveToFailed;
+            CurrentSettings.MapNotRunnableToFailed = settings.MapNotRunnableToFailed;
             CurrentSettings.EnableBaseClassTestMethodsFromOtherAssemblies = settings.EnableBaseClassTestMethodsFromOtherAssemblies;
             CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers;
             CurrentSettings.ParallelizationScope = settings.ParallelizationScope;
@@ -280,6 +287,7 @@ private static MSTestSettings ToSettings(XmlReader reader)
             // <MSTestV2>
             //     <CaptureTraceOutput>true</CaptureTraceOutput>
             //     <MapInconclusiveToFailed>false</MapInconclusiveToFailed>
+            //     <MapNotRunnableToFailed>false</MapNotRunnableToFailed>
             //     <EnableBaseClassTestMethodsFromOtherAssemblies>false</EnableBaseClassTestMethodsFromOtherAssemblies>
             //     <TestTimeout>5000</TestTimeout>
             //     <Parallelize>
@@ -350,6 +358,16 @@ private static MSTestSettings ToSettings(XmlReader reader)
                                 break;
                             }
 
+                        case "MAPNOTRUNNABLETOFAILED":
+                            {
+                                if (bool.TryParse(reader.ReadInnerXml(), out result))
+                                {
+                                    settings.MapNotRunnableToFailed = result;
+                                }
+
+                                break;
+                            }
+
                         case "SETTINGSFILE":
                             {
                                 string fileName = reader.ReadInnerXml();

src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs

@@ -147,9 +147,9 @@ internal UnitTestResult(UnitTestOutcome outcome, string errorMessage)
         /// <param name="testCase"> The test Case.  </param>
         /// <param name="startTime"> The start Time.  </param>
         /// <param name="endTime"> The end Time.  </param>
-        /// <param name="mapInconclusiveToFailed">Indication to map inconclusive tests to failed.</param>
+        /// <param name="currentSettings">Current MSTest settings.</param>
         /// <returns> The <see cref="TestResult"/>. </returns>
-        internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, bool mapInconclusiveToFailed)
+        internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, DateTimeOffset endTime, MSTestSettings currentSettings)
         {
             Debug.Assert(testCase != null, "testCase");
 
@@ -159,7 +159,7 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da
                                             Duration = this.Duration,
                                             ErrorMessage = this.ErrorMessage,
                                             ErrorStackTrace = this.ErrorStackTrace,
-                                            Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, mapInconclusiveToFailed),
+                                            Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, currentSettings),
                                             StartTime = startTime,
                                             EndTime = endTime
             };

src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs

@@ -118,11 +118,11 @@ public override DataRow DataRow
         }
 
         /// <inheritdoc/>
-        public override IDictionary Properties
+        public override IDictionary<string, object> Properties
         {
             get
             {
-                return this.properties as IDictionary;
+                return this.properties;
             }
         }
 

src/TestFramework/Extension.Desktop/PrivateObject.cs

@@ -127,6 +127,11 @@ public PrivateObject(Type type, Type[] parameterTypes, object[] args)
                 catch (TargetInvocationException e)
                 {
                     Debug.Assert(e.InnerException != null, "Inner exception should not be null.");
+                    if (e.InnerException != null)
+                    {
+                        throw e.InnerException;
+                    }
+
                     throw;
                 }
             }
@@ -410,6 +415,11 @@ public object Invoke(string name, BindingFlags bindingFlags, Type[] parameterTyp
                 catch (TargetInvocationException e)
                 {
                     Debug.Assert(e.InnerException != null, "Inner exception should not be null.");
+                    if (e.InnerException != null)
+                    {
+                        throw e.InnerException;
+                    }
+
                     throw;
                 }
             }
@@ -738,6 +748,11 @@ private object InvokeHelper(string name, BindingFlags bindingFlags, object[] arg
             catch (TargetInvocationException e)
             {
                 Debug.Assert(e.InnerException != null, "Inner exception should not be null.");
+                if (e.InnerException != null)
+                {
+                    throw e.InnerException;
+                }
+
                 throw;
             }
         }

src/TestFramework/Extension.Desktop/PrivateType.cs

@@ -208,6 +208,11 @@ public object InvokeStatic(string name, BindingFlags bindingFlags, Type[] parame
                 catch (TargetInvocationException e)
                 {
                     Debug.Assert(e.InnerException != null, "Inner Exception should not be null.");
+                    if (e.InnerException != null)
+                    {
+                        throw e.InnerException;
+                    }
+
                     throw;
                 }
             }
@@ -508,6 +513,11 @@ private object InvokeHelperStatic(string name, BindingFlags bindingFlags, object
             catch (TargetInvocationException e)
             {
                 Debug.Assert(e.InnerException != null, "Inner Exception should not be null.");
+                if (e.InnerException != null)
+                {
+                    throw e.InnerException;
+                }
+
                 throw;
             }
         }

src/TestFramework/Extension.Desktop/TestContext.cs

@@ -4,7 +4,7 @@
 namespace Microsoft.VisualStudio.TestTools.UnitTesting
 {
     using System;
-    using System.Collections;
+    using System.Collections.Generic;
     using System.Data;
     using System.Data.Common;
     using System.Diagnostics;
@@ -19,7 +19,7 @@ public abstract class TestContext
         /// <summary>
         /// Gets test properties for a test.
         /// </summary>
-        public abstract IDictionary Properties { get; }
+        public abstract IDictionary<string, object> Properties { get; }
 
         /// <summary>
         /// Gets the current data row when test is used for data driven testing.

test/E2ETests/Automation.CLI/CLITestBase.cs

@@ -117,12 +117,16 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList)
         /// <remarks>Provide the full test name similar to this format SampleTest.TestCode.TestMethodPass.</remarks>
         public void ValidatePassedTests(params string[] passedTests)
         {
-            // Make sure only expected number of tests passed and not more.
-            Assert.AreEqual(passedTests.Length, this.runEventsHandler.PassedTests.Count);
-
+            this.ValidatePassedTestsCount(passedTests.Length);
             this.ValidatePassedTestsContain(passedTests);
         }
 
+        public void ValidatePassedTestsCount(int expectedPassedTestsCount)
+        {
+            // Make sure only expected number of tests passed and not more.
+            Assert.AreEqual(expectedPassedTestsCount, this.runEventsHandler.PassedTests.Count);
+        }
+
         /// <summary>
         /// Validates if the test results have the specified set of failed tests.
         /// </summary>

test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace MSTestAdapter.Smoke.E2ETests
+{
+    using Microsoft.MSTestV2.CLIAutomation;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class DataRowTests : CLITestBase
+    {
+        private const string TestAssembly = "DataRowTestProject.dll";
+
+        [TestMethod]
+        public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerviedClassHasDataRows()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly });
+
+            this.ValidatePassedTestsContain(
+                "DataRowTestMethod (BaseString1)",
+                "DataRowTestMethod (BaseString2)",
+                "DataRowTestMethod (BaseString3)",
+                "DataRowTestMethod (DerivedString1)",
+                "DataRowTestMethod (DerivedString2)");
+
+            // 4 tests of BaseClass - 3 datarow result and 1 parent result
+            // 3 tests of DerivedClass - 2 datarow result and 1 parent result
+            // Total 7 tests - Making sure that DerivedClass doesn't run BaseClass tests
+            this.ValidatePassedTestsCount(7);
+        }
+
+        [TestMethod]
+        public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~DerivedClass");
+
+            this.ValidatePassedTestsContain(
+                "DataRowTestMethod (DerivedString1)",
+                "DataRowTestMethod (DerivedString2)");
+
+            // 3 tests of DerivedClass - 2 datarow result and 1 parent result
+            this.ValidatePassedTestsCount(3);
+        }
+    }
+}

test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj

@@ -48,6 +48,7 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="DynamicDataExtensibilityTests.cs" />
     <Compile Include="TestDataSourceExtensibilityTests.cs" />
+    <Compile Include="DataRowTests.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

test/E2ETests/TestAssets/DataRowTestProject/BaseClass.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information. 
+
+namespace DataRowTestProject
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class BaseClass
+    {
+        [TestMethod]
+        [DataRow("BaseString1")]
+        [DataRow("BaseString2")]
+        [DataRow("BaseString3")]
+        public virtual void DataRowTestMethod(string a)
+        {
+            Assert.IsTrue(true);
+        }
+    }
+}

test/E2ETests/TestAssets/DataRowTestProject/DataRowTestProject.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TestFxRoot Condition="$(TestFxRoot) == ''">..\..\..\..\</TestFxRoot>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TargetFramework>net452</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+    <OutputPath>$(TestFxRoot)artifacts\TestAssets\</OutputPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\..\src\TestFramework\Extension.Desktop\Extension.Desktop.csproj" />
+    <ProjectReference Include="..\..\..\..\src\TestFramework\MSTest.Core\MSTest.Core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+
+</Project>