Record test start/end events for data driven tests

src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblyInfo.cs

@@ -4,6 +4,7 @@
 namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
 {
     using System;
+    using System.Diagnostics;
     using System.Diagnostics.CodeAnalysis;
     using System.Globalization;
     using System.Reflection;
@@ -25,11 +26,20 @@ public class TestAssemblyInfo
         /// <summary>
         /// Initializes a new instance of the <see cref="TestAssemblyInfo"/> class.
         /// </summary>
-        internal TestAssemblyInfo()
+        /// <param name="assemblyLocation">The test assembly location</param>
+        internal TestAssemblyInfo(string assemblyLocation)
         {
+            Debug.Assert(!string.IsNullOrEmpty(assemblyLocation), "AssemblyLocation should not be null or empty");
+
             this.assemblyInfoExecuteSyncObject = new object();
+            this.AssemblyName = assemblyLocation;
         }
 
+        /// <summary>
+        /// Gets the assembly name.
+        /// </summary>
+        public string AssemblyName { get; }
+
         /// <summary>
         /// Gets <c>AssemblyInitialize</c> method for the assembly.
         /// </summary>

src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs

@@ -154,20 +154,27 @@ internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, D
                 return;
             }
 
+            Guid originalTestCaseId = test.Id;
             foreach (var unitTestResult in unitTestResults)
             {
                 if (test == null)
                 {
                     continue;
                 }
 
+                // If the result has a TestId set then update the TestCase to have that id
+                // this is done for data driven results so we can fire unique start/end events for each iteration
+                test.Id = unitTestResult.TestId == Guid.Empty ? originalTestCaseId : unitTestResult.TestId;
                 var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
 
                 if (unitTestResult.DatarowIndex >= 0)
                 {
                     testResult.DisplayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, unitTestResult.DatarowIndex);
                 }
 
+                // Fire a RecordEnd here even though we also fire RecordEnd inside TestMethodInfo
+                // this is to ensure we fire end events for error cases. This is acceptable because
+                // vstest ignores multiple end events.
                 testExecutionRecorder.RecordEnd(test, testResult.Outcome);
 
                 if (testResult.Outcome == TestOutcome.Failed)
@@ -185,6 +192,9 @@ internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, D
                     // Ignore this exception
                 }
             }
+
+            // Reset the test id back to the original id in case we changed it above
+            test.Id = originalTestCaseId;
         }
 
         private static bool MatchTestFilter(ITestCaseFilterExpression filterExpression, TestCase test, TestMethodFilter testMethodFilter)
@@ -355,6 +365,7 @@ private void ExecuteTestsWithTestRunner(
             IDictionary<string, object> sourceLevelParameters,
             UnitTestRunner testRunner)
         {
+            var testExecutionRecorderWrapper = new TestExecutionRecorderWrapper(testExecutionRecorder);
             foreach (var currentTest in tests)
             {
                 if (this.cancellationToken != null && this.cancellationToken.Canceled)
@@ -363,6 +374,10 @@ private void ExecuteTestsWithTestRunner(
                 }
 
                 var unitTestElement = currentTest.ToUnitTestElement(source);
+
+                // Fire a RecordStart here even though we also fire RecordStart inside TestMethodInfo
+                // this is to ensure we fire start events for error cases. This is acceptable because
+                // vstest ignores multiple start events.
                 testExecutionRecorder.RecordStart(currentTest);
 
                 var startTime = DateTimeOffset.Now;
@@ -374,7 +389,7 @@ private void ExecuteTestsWithTestRunner(
                 // Run single test passing test context properties to it.
                 var tcmProperties = TcmTestPropertiesProvider.GetTcmProperties(currentTest);
                 var testContextProperties = this.GetTestContextProperties(tcmProperties, sourceLevelParameters);
-                var unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testContextProperties);
+                var unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testExecutionRecorderWrapper, testContextProperties);
 
                 PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
                     "Executed test {0}",

src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionRecorderWrapper.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
+{
+    using System;
+    using System.Diagnostics;
+    using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
+    using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+
+    /// <summary>
+    /// Wraps calls to ITestExecutionRecorder using types that can be serialized across AppDomains.
+    /// </summary>
+    internal class TestExecutionRecorderWrapper : MarshalByRefObject
+    {
+        private readonly ITestExecutionRecorder recorder;
+
+        public TestExecutionRecorderWrapper(ITestExecutionRecorder testExecutionRecorder)
+        {
+            Debug.Assert(testExecutionRecorder != null, "TestExecutionRecorder should not be null");
+
+            this.recorder = testExecutionRecorder;
+        }
+
+        /// <summary>
+        /// Returns object to be used for controlling lifetime, null means infinite lifetime.
+        /// </summary>
+        /// <returns>
+        /// The <see cref="object"/>.
+        /// </returns>
+        public override object InitializeLifetimeService()
+        {
+            return null;
+        }
+
+        public void RecordEnd(UnitTestElement test, UnitTestOutcome outcome)
+        {
+            this.recorder.RecordEnd(test.ToTestCase(), UnitTestOutcomeHelper.ToTestOutcome(outcome, MSTestSettings.CurrentSettings));
+        }
+
+        public void RecordStart(UnitTestElement test)
+        {
+            this.recorder.RecordStart(test.ToTestCase());
+        }
+    }
+}

src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs

@@ -14,6 +14,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
     using UnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome;
     using UTF = Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -33,16 +34,24 @@ public class TestMethodInfo : ITestMethod
         internal TestMethodInfo(
             MethodInfo testMethod,
             TestClassInfo parent,
-            TestMethodOptions testmethodOptions)
+            TestMethodOptions testmethodOptions,
+            TestExecutionRecorderWrapper testExecutionRecorder)
         {
             Debug.Assert(testMethod != null, "TestMethod should not be null");
             Debug.Assert(parent != null, "Parent should not be null");
+            Debug.Assert(testExecutionRecorder != null, "TestExecutionRecorder should not be null");
 
             this.TestMethod = testMethod;
             this.Parent = parent;
             this.TestMethodOptions = testmethodOptions;
+            this.TestExecutionRecorder = testExecutionRecorder;
         }
 
+        /// <summary>
+        /// Gets or sets the id of the test.
+        /// </summary>
+        public Guid TestId { get; set; }
+
         /// <summary>
         /// Gets a value indicating whether timeout is set.
         /// </summary>
@@ -85,6 +94,11 @@ internal TestMethodInfo(
         /// </summary>
         internal TestMethodOptions TestMethodOptions { get; private set; }
 
+        /// <summary>
+        /// Gets the test execution recorder used to log test execution.
+        /// </summary>
+        internal TestExecutionRecorderWrapper TestExecutionRecorder { get; }
+
         public Attribute[] GetAllAttributes(bool inherit)
         {
             return ReflectHelper.GetCustomAttributes(this.TestMethod, inherit) as Attribute[];
@@ -137,9 +151,13 @@ public virtual TestResult Invoke(object[] arguments)
 
             using (LogMessageListener listener = new LogMessageListener(this.TestMethodOptions.CaptureDebugTraces))
             {
+                var testElement = new UnitTestElement(new TestMethod(this)) { TestId = this.TestId };
+
                 watch.Start();
                 try
                 {
+                    this.TestExecutionRecorder.RecordStart(testElement);
+
                     if (this.IsTimeoutSet)
                     {
                         result = this.ExecuteInternalWithTimeout(arguments);
@@ -162,7 +180,12 @@ public virtual TestResult Invoke(object[] arguments)
                         result.LogError = listener.StandardError;
                         result.TestContextMessages = this.TestMethodOptions.TestContext.GetAndClearDiagnosticMessages();
                         result.ResultFiles = this.TestMethodOptions.TestContext.GetResultFiles();
+                        result.TestId = this.TestId;
                     }
+
+                    this.TestExecutionRecorder.RecordEnd(
+                        testElement,
+                        result?.Outcome.ToUnitTestOutcome() ?? UnitTestOutcome.Error);
                 }
             }
 

src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs

@@ -258,6 +258,8 @@ internal UnitTestResult[] RunTestMethod()
                                     watch.Reset();
                                     watch.Start();
 
+                                    // Create a unique ID for each iteration so data collectors can attach results to specific iterations
+                                    this.testMethodInfo.TestId = Guid.NewGuid();
                                     this.testContext.SetDataRow(dataRow);
                                     UTF.TestResult[] testResults;
 
@@ -313,6 +315,8 @@ internal UnitTestResult[] RunTestMethod()
                         {
                             foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
                             {
+                                // Create a unique ID for each iteration so data collectors can attach results to specific iterations
+                                this.testMethodInfo.TestId = Guid.NewGuid();
                                 this.testMethodInfo.SetArguments(data);
                                 UTF.TestResult[] testResults;
                                 try

src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs

@@ -16,6 +16,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
     using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
     /// <summary>
@@ -121,9 +122,10 @@ public IEnumerable<TestClassInfo> ClassInfoCache
         /// </summary>
         /// <param name="testMethod"> The test Method. </param>
         /// <param name="testContext"> The test Context. </param>
+        /// <param name="testExecutionRecorder">A instance of TestExecutionRecorderWrapper used to log test execution.</param>
         /// <param name="captureDebugTraces"> Indicates whether the test method should capture debug traces.</param>
         /// <returns> The <see cref="TestMethodInfo"/>. </returns>
-        public TestMethodInfo GetTestMethodInfo(TestMethod testMethod, ITestContext testContext, bool captureDebugTraces)
+        public TestMethodInfo GetTestMethodInfo(TestMethod testMethod, ITestContext testContext, TestExecutionRecorderWrapper testExecutionRecorder, bool captureDebugTraces)
         {
             if (testMethod == null)
             {
@@ -146,7 +148,7 @@ public TestMethodInfo GetTestMethodInfo(TestMethod testMethod, ITestContext test
             }
 
             // Get the testMethod
-            return this.ResolveTestMethod(testMethod, testClassInfo, testContext, captureDebugTraces);
+            return this.ResolveTestMethod(testMethod, testClassInfo, testContext, testExecutionRecorder, captureDebugTraces);
         }
 
         /// <summary>
@@ -266,7 +268,7 @@ private TestClassInfo CreateClassInfo(Type classType, TestMethod testMethod)
 
             var testContextProperty = this.ResolveTestContext(classType);
 
-            var assemblyInfo = this.GetAssemblyInfo(classType);
+            var assemblyInfo = this.GetAssemblyInfo(classType, testMethod.AssemblyName);
 
             var classInfo = new TestClassInfo(classType, constructor, testContextProperty, this.reflectionHelper.GetDerivedAttribute<TestClassAttribute>(classType, false), assemblyInfo);
 
@@ -358,9 +360,10 @@ private PropertyInfo ResolveTestContext(Type classType)
         /// Get the assembly info for the parameter type
         /// </summary>
         /// <param name="type"> The type. </param>
+        /// <param name="assemblyLocation">The location of the assembly</param>
         /// <returns> The <see cref="TestAssemblyInfo"/> instance. </returns>
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Discoverer should continue with remaining sources.")]
-        private TestAssemblyInfo GetAssemblyInfo(Type type)
+        private TestAssemblyInfo GetAssemblyInfo(Type type, string assemblyLocation)
         {
             var assembly = type.GetTypeInfo().Assembly;
 
@@ -378,7 +381,7 @@ private TestAssemblyInfo GetAssemblyInfo(Type type)
                     var assemblyInitializeType = typeof(AssemblyInitializeAttribute);
                     var assemblyCleanupType = typeof(AssemblyCleanupAttribute);
 
-                    assemblyInfo = new TestAssemblyInfo();
+                    assemblyInfo = new TestAssemblyInfo(assemblyLocation);
 
                     var types = new AssemblyEnumerator().GetTypes(assembly, assembly.FullName, null);
 
@@ -631,11 +634,12 @@ private void UpdateInfoIfTestInitializeOrCleanupMethod(
         /// <param name="testMethod"> The test Method. </param>
         /// <param name="testClassInfo"> The test Class Info. </param>
         /// <param name="testContext"> The test Context. </param>
+        /// <param name="testExecutionRecorder">A instance of TestExecutionRecorderWrapper used to log test execution.</param>
         /// <param name="captureDebugTraces"> Indicates whether the test method should capture debug traces.</param>
         /// <returns>
         /// The TestMethodInfo for the given test method. Null if the test method could not be found.
         /// </returns>
-        private TestMethodInfo ResolveTestMethod(TestMethod testMethod, TestClassInfo testClassInfo, ITestContext testContext, bool captureDebugTraces)
+        private TestMethodInfo ResolveTestMethod(TestMethod testMethod, TestClassInfo testClassInfo, ITestContext testContext, TestExecutionRecorderWrapper testExecutionRecorder, bool captureDebugTraces)
         {
             Debug.Assert(testMethod != null, "testMethod is Null");
             Debug.Assert(testClassInfo != null, "testClassInfo is Null");
@@ -651,7 +655,7 @@ private TestMethodInfo ResolveTestMethod(TestMethod testMethod, TestClassInfo te
             var timeout = this.GetTestTimeout(methodInfo, testMethod);
 
             var testMethodOptions = new TestMethodOptions() { Timeout = timeout, Executor = this.GetTestMethodAttribute(methodInfo, testClassInfo), ExpectedException = expectedExceptionAttribute, TestContext = testContext, CaptureDebugTraces = captureDebugTraces };
-            var testMethodInfo = new TestMethodInfo(methodInfo, testClassInfo, testMethodOptions);
+            var testMethodInfo = new TestMethodInfo(methodInfo, testClassInfo, testMethodOptions, testExecutionRecorder);
 
             this.SetCustomProperties(testMethodInfo, testContext);
 

src/Adapter/MSTest.CoreAdapter/Execution/UnitTestRunner.cs

@@ -12,6 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
 
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
     using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
     /// <summary>
     /// The runner that runs a single unit test. Also manages the assembly and class cleanup methods at the end of the run.
@@ -62,15 +63,21 @@ public override object InitializeLifetimeService()
         /// Runs a single test.
         /// </summary>
         /// <param name="testMethod"> The test Method. </param>
+        /// <param name="testExecutionRecorder">A instance of TestExecutionRecorderWrapper used to log test execution.</param>
         /// <param name="testContextProperties"> The test context properties. </param>
         /// <returns> The <see cref="UnitTestResult"/>. </returns>
-        internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary<string, object> testContextProperties)
+        internal UnitTestResult[] RunSingleTest(TestMethod testMethod, TestExecutionRecorderWrapper testExecutionRecorder, IDictionary<string, object> testContextProperties)
         {
             if (testMethod == null)
             {
                 throw new ArgumentNullException("testMethod");
             }
 
+            if (testExecutionRecorder == null)
+            {
+                throw new ArgumentNullException(nameof(testExecutionRecorder));
+            }
+
             try
             {
                 using (var writer = new ThreadSafeStringWriter(CultureInfo.InvariantCulture))
@@ -80,7 +87,7 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary<strin
                     testContext.SetOutcome(TestTools.UnitTesting.UnitTestOutcome.InProgress);
 
                     // Get the testMethod
-                    var testMethodInfo = this.typeCache.GetTestMethodInfo(testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces);
+                    var testMethodInfo = this.typeCache.GetTestMethodInfo(testMethod, testContext, testExecutionRecorder, MSTestSettings.CurrentSettings.CaptureDebugTraces);
 
                     // If the specified TestMethod could not be found, return a NotFound result.
                     if (testMethodInfo == null)

src/Adapter/MSTest.CoreAdapter/Extensions/TestResultExtensions.cs

@@ -47,6 +47,7 @@ public static UnitTestResult[] ToUnitTestResults(this UTF.TestResult[] testResul
                 unitTestResult.DisplayName = testResults[i].DisplayName;
                 unitTestResult.DatarowIndex = testResults[i].DatarowIndex;
                 unitTestResult.ResultFiles = testResults[i].ResultFiles;
+                unitTestResult.TestId = testResults[i].TestId;
                 unitTestResult.ExecutionId = testResults[i].ExecutionId;
                 unitTestResult.ParentExecId = testResults[i].ParentExecId;
                 unitTestResult.InnerResultsCount = testResults[i].InnerResultsCount;

src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj

@@ -35,6 +35,7 @@
     <Compile Include="Discovery\TestMethodValidator.cs" />
     <Compile Include="Execution\RunCleanupResult.cs" />
     <Compile Include="Execution\TcmTestPropertiesProvider.cs" />
+    <Compile Include="Execution\TestExecutionRecorderWrapper.cs" />
     <Compile Include="Extensions\TestContextExtensions.cs" />
     <Compile Include="Extensions\TestResultExtensions.cs" />
     <Compile Include="Extensions\UnitTestOutcomeExtensions.cs" />