Dotnet test integration (first part)

Directory.Packages.props

@@ -65,6 +65,7 @@
     <PackageVersion Include="Microsoft.TemplateSearch.Common" Version="$(MicrosoftTemplateSearchCommonPackageVersion)" />
     <PackageVersion Include="Microsoft.TestPlatform.Build" Version="$(MicrosoftTestPlatformBuildPackageVersion)" />
     <PackageVersion Include="Microsoft.TestPlatform.CLI" Version="$(MicrosoftTestPlatformCLIPackageVersion)" />
+    <PackageVersion Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformPackageVersion)" />
     <PackageVersion Include="Microsoft.VisualStudio.Composition" Version="17.4.16" />
     <PackageVersion Include="Microsoft.VisualStudio.Setup.Configuration.Interop" Version="$(MicrosoftVisualStudioSetupConfigurationInteropVersion)" />
     <PackageVersion Include="Microsoft.Web.Deployment" Version="$(WebDeploymentPackageVersion)" />

eng/Version.Details.xml

@@ -228,6 +228,10 @@
       <Uri>https://github.com/microsoft/vstest</Uri>
       <Sha>bc9161306b23641b0364b8f93d546da4d48da1eb</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.Testing.Platform" Version="1.5.0-preview.24516.3">
+      <Uri>https://github.com/microsoft/testfx</Uri>
+      <Sha>bc9161306b23641b0364b8f93d546da4d48da1eb</Sha>
+    </Dependency>
     <!-- Intermediate is necessary for source build. -->
     <Dependency Name="Microsoft.SourceBuild.Intermediate.vstest" Version="17.12.0-release-24508-01">
       <Uri>https://github.com/microsoft/vstest</Uri>

eng/Versions.props

@@ -199,6 +199,10 @@
     <MicrosoftTestPlatformCLIPackageVersion>17.12.0-release-24508-01</MicrosoftTestPlatformCLIPackageVersion>
     <MicrosoftTestPlatformBuildPackageVersion>17.12.0-release-24508-01</MicrosoftTestPlatformBuildPackageVersion>
   </PropertyGroup>
+  <PropertyGroup>
+    <!-- Dependencies from https://github.com/Microsoft/testfx -->
+    <MicrosoftTestingPlatformPackageVersion>1.5.0-preview.24516.3</MicrosoftTestingPlatformPackageVersion>
+  </PropertyGroup>
   <PropertyGroup>
     <!-- Dependencies from https://github.com/dotnet/roslyn-analyzers -->
     <MicrosoftCodeAnalysisNetAnalyzersVersion>10.0.0-preview.24508.1</MicrosoftCodeAnalysisNetAnalyzersVersion>

src/Cli/dotnet/commands/dotnet-test/TestingPlatformCommand.cs

@@ -5,6 +5,9 @@
 using System.CommandLine;
 using Microsoft.DotNet.Tools.Test;
 using Microsoft.TemplateEngine.Cli.Commands;
+using Microsoft.Testing.Platform.Helpers;
+using Microsoft.Testing.Platform.OutputDevice;
+using Microsoft.Testing.Platform.OutputDevice.Terminal;
 
 namespace Microsoft.DotNet.Cli
 {
@@ -15,9 +18,12 @@ internal partial class TestingPlatformCommand : CliCommand, ICustomHelp
 
         private MSBuildConnectionHandler _msBuildConnectionHandler;
         private TestModulesFilterHandler _testModulesFilterHandler;
+        private TerminalTestReporter _output;
         private TestApplicationActionQueue _actionQueue;
         private Task _namedPipeConnectionLoop;
         private List<string> _args;
+        private Dictionary<TestApplication, (string ModulePath, string TargetFramework, string Architecture, string ExecutionId)> _executions = new();
+        private byte _cancelled;
 
         public TestingPlatformCommand(string name, string description = null) : base(name, description)
         {
@@ -26,6 +32,12 @@ public TestingPlatformCommand(string name, string description = null) : base(nam
 
         public int Run(ParseResult parseResult)
         {
+            Console.CancelKeyPress += (s, e) =>
+            {
+                _output?.StartCancelling();
+                CompleteRun();
+            };
+
             // User can decide what the degree of parallelism should be
             // If not specified, we will default to the number of processors
             if (!int.TryParse(parseResult.GetValue(TestingPlatformOptions.MaxParallelTestModulesOption), out int degreeOfParallelism))
@@ -36,6 +48,7 @@ public int Run(ParseResult parseResult)
             if (filterModeEnabled && parseResult.HasOption(TestingPlatformOptions.ArchitectureOption))
             {
                 VSTestTrace.SafeWriteTrace(() => $"The --arch option is not supported yet.");
+                return ExitCodes.GenericFailure;
             }
 
             BuiltInOptions builtInOptions = new(
@@ -44,6 +57,18 @@ public int Run(ParseResult parseResult)
                 parseResult.GetValue(TestingPlatformOptions.ConfigurationOption),
                 parseResult.GetValue(TestingPlatformOptions.ArchitectureOption));
 
+            var console = new SystemConsole();
+            var output = new TerminalTestReporter(console, new TerminalTestReporterOptions()
+            {
+                ShowPassedTests = Environment.GetEnvironmentVariable("SHOW_PASSED") == "1",
+                ShowProgress = () => Environment.GetEnvironmentVariable("NO_PROGRESS") != "1",
+                UseAnsi = Environment.GetEnvironmentVariable("NO_ANSI") != "1",
+                ShowAssembly = true,
+                ShowAssemblyStartAndComplete = true,
+            });
+            _output = output;
+            _output.TestExecutionStarted(DateTimeOffset.Now, degreeOfParallelism);
+
             if (ContainsHelpOption(parseResult.GetArguments()))
             {
                 _actionQueue = new(degreeOfParallelism, async (TestApplication testApp) =>
@@ -54,7 +79,9 @@ public int Run(ParseResult parseResult)
                     testApp.Run += OnTestApplicationRun;
                     testApp.ExecutionIdReceived += OnExecutionIdReceived;
 
-                    return await testApp.RunAsync(filterModeEnabled, enableHelp: true, builtInOptions);
+                    var result = await testApp.RunAsync(filterModeEnabled, enableHelp: true, builtInOptions);
+                    CompleteRun();
+                    return result;
                 });
             }
             else
@@ -84,6 +111,7 @@ public int Run(ParseResult parseResult)
             {
                 if (!_testModulesFilterHandler.RunWithTestModulesFilter(parseResult))
                 {
+                    CompleteRun();
                     return ExitCodes.GenericFailure;
                 }
             }
@@ -94,6 +122,7 @@ public int Run(ParseResult parseResult)
                 if (msbuildResult != 0)
                 {
                     VSTestTrace.SafeWriteTrace(() => $"MSBuild task _GetTestsProject didn't execute properly with exit code: {msbuildResult}.");
+                    CompleteRun();
                     return ExitCodes.GenericFailure;
                 }
             }
@@ -108,9 +137,18 @@ public int Run(ParseResult parseResult)
             // Clean up everything
             CleanUp();
 
+            CompleteRun();
             return hasFailed ? ExitCodes.GenericFailure : ExitCodes.Success;
         }
 
+        private void CompleteRun()
+        {
+            if (Interlocked.CompareExchange(ref _cancelled, 1, 0) == 0)
+            {
+                _output?.TestExecutionCompleted(DateTimeOffset.Now);
+            }
+        }
+
         private void CleanUp()
         {
             _msBuildConnectionHandler.Dispose();
@@ -122,6 +160,14 @@ private void CleanUp()
 
         private void OnHandshakeReceived(object sender, HandshakeArgs args)
         {
+            var testApplication = (TestApplication)sender;
+            var executionId = args.Handshake.Properties[HandshakeMessagePropertyNames.ExecutionId];
+            var arch = args.Handshake.Properties[HandshakeMessagePropertyNames.Architecture]?.ToLower();
+            var tfm = TargetFrameworkParser.GetShortTargetFramework(args.Handshake.Properties[HandshakeMessagePropertyNames.Framework]);
+            (string ModulePath, string TargetFramework, string Architecture, string ExecutionId) appInfo = new(testApplication.Module.DLLOrExe, tfm, arch, executionId);
+            _executions[testApplication] = appInfo;
+            _output.AssemblyRunStarted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId);
+
             if (!VSTestTrace.TraceEnabled)
             {
                 return;
@@ -153,6 +199,42 @@ private void OnDiscoveredTestsReceived(object sender, DiscoveredTestEventArgs ar
 
         private void OnTestResultsReceived(object sender, TestResultEventArgs args)
         {
+            foreach (var testResult in args.SuccessfulTestResults)
+            {
+                var testApp = (TestApplication)sender;
+                var appInfo = _executions[testApp];
+                // TODO: timespan for duration
+                _output.TestCompleted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId,
+                    testResult.DisplayName,
+                    ToOutcome(testResult.State),
+                    TimeSpan.FromSeconds(1),
+                    errorMessage: null,
+                    errorStackTrace: null,
+                    expected: null,
+                    actual: null,
+                    standardOutput: null,
+                    errorOutput: null);
+            }
+
+            foreach (var testResult in args.FailedTestResults)
+            {
+                var testApp = (TestApplication)sender;
+                // TODO: timespan for duration
+                // TODO: expected
+                // TODO: actual
+                var appInfo = _executions[testApp];
+                _output.TestCompleted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId,
+                    testResult.DisplayName,
+                    ToOutcome(testResult.State),
+                    TimeSpan.FromSeconds(1),
+                    errorMessage: testResult.ErrorMessage,
+                    errorStackTrace: testResult.ErrorStackTrace,
+                    expected: null,
+                    actual: null,
+                    standardOutput: null,
+                    errorOutput: null);
+            }
+
             if (!VSTestTrace.TraceEnabled)
             {
                 return;
@@ -174,6 +256,20 @@ private void OnTestResultsReceived(object sender, TestResultEventArgs args)
             }
         }
 
+        public static TestOutcome ToOutcome(byte? testState)
+        {
+            return testState switch
+            {
+                TestStates.Passed => TestOutcome.Passed,
+                TestStates.Skipped => TestOutcome.Skipped,
+                TestStates.Failed => TestOutcome.Fail,
+                TestStates.Error => TestOutcome.Error,
+                TestStates.Timeout => TestOutcome.Timeout,
+                TestStates.Cancelled => TestOutcome.Canceled,
+                _ => throw new ArgumentOutOfRangeException(nameof(testState), $"Invalid test state value {testState}")
+            };
+        }
+
         private void OnFileArtifactsReceived(object sender, FileArtifactEventArgs args)
         {
             if (!VSTestTrace.TraceEnabled)
@@ -185,7 +281,7 @@ private void OnFileArtifactsReceived(object sender, FileArtifactEventArgs args)
 
             foreach (FileArtifact fileArtifactMessage in args.FileArtifacts)
             {
-                VSTestTrace.SafeWriteTrace(() => $"FileArtifacr: {fileArtifactMessage.FullPath}, {fileArtifactMessage.DisplayName}, " +
+                VSTestTrace.SafeWriteTrace(() => $"FileArtifact: {fileArtifactMessage.FullPath}, {fileArtifactMessage.DisplayName}, " +
                 $"{fileArtifactMessage.Description}, {fileArtifactMessage.TestUid}, {fileArtifactMessage.TestDisplayName}, " +
                 $"{fileArtifactMessage.SessionUid}");
             }
@@ -214,6 +310,11 @@ private void OnErrorReceived(object sender, ErrorEventArgs args)
 
         private void OnTestProcessExited(object sender, TestProcessExitEventArgs args)
         {
+            var testApplication = (TestApplication)sender;
+
+            var appInfo = _executions[testApplication];
+            _output.AssemblyRunCompleted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId);
+
             if (!VSTestTrace.TraceEnabled)
             {
                 return;

src/Cli/dotnet/dotnet.csproj

@@ -96,6 +96,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
+    <PackageReference Include="Microsoft.Testing.Platform" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="Microsoft.ApplicationInsights" />
     <PackageReference Include="Microsoft.Build" />