Add some checks for merged test groups

src/tests/Common/CLRTest.Execute.Batch.targets

@@ -164,8 +164,8 @@ IF NOT "%CLRTestExitCode%"=="%CLRTestExpectedExitCode%" (
 :TakeLock
 md %lockFolder%
 IF NOT "!ERRORLEVEL!"=="0" (
-timeout /t 10 /nobreak
-goto :TakeLock
+  timeout /t 10 /nobreak
+  goto :TakeLock
 )
 Exit /b 2
 
@@ -403,15 +403,17 @@ IF /I [%1] == [-%(Identity)] set cond=1
 IF /I [%1] == [/%(Identity)] set cond=1
 IF %cond% EQU 1 (
 %(Command)
-shift
-IF /I [%(HasParam)] == [true] shift
-goto NextArg
+    shift
+    IF /I [%(HasParam)] == [true] shift
+    goto NextArg
 )','
 ')
 
+:ExtraArgs
 if NOT "%1" == "" (
-set CLRTestExecutionArguments=%*
-goto :ArgsDone
+    set CLRTestExecutionArguments=%CLRTestExecutionArguments% %1
+    shift
+    goto :ExtraArgs
 )
 
 goto ArgsDone

src/tests/Common/XUnitWrapperGenerator/Descriptors.cs

@@ -0,0 +1,21 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.CodeAnalysis;
+
+namespace XUnitWrapperGenerator;
+
+public static class Descriptors
+{
+    public static readonly DiagnosticDescriptor XUWG1001 =
+        new DiagnosticDescriptor(
+            "XUW1001",
+            "Projects in merged tests group should not have entry points",
+            "Projects in merged tests group should not have entry points. Convert to Facts or Theories.",
+            "XUnitWrapperGenerator",
+            DiagnosticSeverity.Warning,
+            isEnabledByDefault: true);
+}

src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs

@@ -415,22 +415,18 @@ public CodeBuilder WrapTestExecutionWithReporting(CodeBuilder testExecutionExpre
             builder.AppendLine("try");
             using (builder.NewBracesScope())
             {
-                builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
+                builder.AppendLine($"{_summaryLocalIdentifier}.ReportStartingTest({test.TestNameExpression}, System.Console.Out);");
                 builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();");
                 builder.Append(testExecutionExpression);
 
                 builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\","
-                                 + $" stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);");
-
-                builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
+                                 + $" stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), System.Console.Out, tempLogSw, statsCsvSw);");
             }
             builder.AppendLine("catch (System.Exception ex)");
             using (builder.NewBracesScope())
             {
                 builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\","
-                                 + $" stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw, statsCsvSw);");
-
-                builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
+                                 + $" stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), System.Console.Out, tempLogSw, statsCsvSw);");
             }
         }
         builder.AppendLine("else");

src/tests/Common/XUnitWrapperGenerator/OptionsHelper.cs

@@ -6,6 +6,7 @@ namespace XUnitWrapperGenerator;
 
 public static class OptionsHelper
 {
+    private const string InMergedTestDirectoryOption = "build_property.InMergedTestDirectory";
     private const string IsMergedTestRunnerAssemblyOption = "build_property.IsMergedTestRunnerAssembly";
     private const string PriorityOption = "build_property.Priority";
     private const string RuntimeFlavorOption = "build_property.RuntimeFlavor";
@@ -29,6 +30,8 @@ private static bool GetBoolOption(this AnalyzerConfigOptions options, string key
                 ? result : 0;
     }
 
+    internal static bool InMergedTestDirectory(this AnalyzerConfigOptions options) => options.GetBoolOption(InMergedTestDirectoryOption);
+
     internal static bool IsMergedTestRunnerAssembly(this AnalyzerConfigOptions options) => options.GetBoolOption(IsMergedTestRunnerAssemblyOption);
 
     internal static int? Priority(this AnalyzerConfigOptions options) => options.GetIntOption(PriorityOption);

src/tests/Common/XUnitWrapperGenerator/RoslynUtils.cs

@@ -0,0 +1,37 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
+
+namespace XUnitWrapperGenerator
+{
+    internal class RoslynUtils
+    {
+        /// <summary>
+        ///   Returns the Main method that would serve as the entry point of the assembly, ignoring
+        ///   whether the current target is an executable.
+        /// </summary>
+        /// <remarks>
+        ///   Replacement for CSharpCompilation.GetEntryPoint() which only works for executables.
+        ///   Replacement for its helpers that are internal.
+        ///
+        ///   Intended for the analyzer that is trying to find Main methods that won't be called in
+        ///   merged test groups. Ignores details such as SynthesizedSimpleProgramEntryPointSymbol.
+        ///   Ignores top-level statements as (1) in exes, they will generate an error for conflicting
+        ///   with the auto-generated main, and (2) in libs, they will generate an error for existing
+        ///   at all.
+        /// </remarks>
+        internal static IEnumerable<IMethodSymbol> GetPossibleEntryPoints(Compilation comp, CancellationToken cancellationToken)
+            => comp
+                .GetSymbolsWithName(WellKnownMemberNames.EntryPointMethodName, SymbolFilter.Member)
+                .OfType<IMethodSymbol>()
+                .Where(m => m.IsStatic && !m.IsAbstract && !m.IsVirtual);
+    }
+}

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -15,6 +16,22 @@
 
 namespace XUnitWrapperGenerator;
 
+internal struct CompData
+{
+    internal CompData(string assemblyName, IMethodSymbol? entryPoint, IEnumerable<IMethodSymbol> possibleEntryPoints, OutputKind outputKind)
+    {
+        AssemblyName = assemblyName;
+        EntryPoint = entryPoint;
+        PossibleEntryPoints = possibleEntryPoints;
+        OutputKind = outputKind;
+    }
+
+    public string AssemblyName { get; private set; }
+    public IMethodSymbol? EntryPoint { get; private set; }
+    public IEnumerable<IMethodSymbol> PossibleEntryPoints { get; private set; }
+    public OutputKind OutputKind { get; private set; }
+}
+
 [Generator]
 public sealed class XUnitWrapperGenerator : IIncrementalGenerator
 {
@@ -58,9 +75,11 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             return aliasMap.ToImmutable();
         }).WithComparer(new ImmutableDictionaryValueComparer<string, string>(EqualityComparer<string>.Default));
 
-        var assemblyName = context.CompilationProvider.Select((comp, ct) => comp.Assembly.MetadataName);
-
-        var alwaysWriteEntryPoint = context.CompilationProvider.Select((comp, ct) => comp.Options.OutputKind == OutputKind.ConsoleApplication && comp.GetEntryPoint(ct) is null);
+        var compData = context.CompilationProvider.Select((comp, ct) => new CompData(
+            assemblyName: comp.Assembly.MetadataName,
+            entryPoint: comp.GetEntryPoint(ct),
+            possibleEntryPoints: RoslynUtils.GetPossibleEntryPoints(comp, ct),
+            outputKind: comp.Options.OutputKind));
 
         var testsInSource =
             methodsInSource
@@ -112,40 +131,65 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             .Collect()
             .Combine(context.AnalyzerConfigOptionsProvider)
             .Combine(aliasMap)
-            .Combine(assemblyName)
-            .Combine(alwaysWriteEntryPoint),
+            .Combine(compData),
             static (context, data) =>
             {
-                var ((((methods, configOptions), aliasMap), assemblyName), alwaysWriteEntryPoint) = data;
+                var (((methods, configOptions), aliasMap), compData) = data;
 
-                if (methods.Length == 0 && !alwaysWriteEntryPoint)
+                bool inMergedTestDirectory = configOptions.GlobalOptions.InMergedTestDirectory();
+                if (inMergedTestDirectory)
                 {
-                    // If we have no test methods, assume that this project is not migrated to the new system yet
-                    // and that we shouldn't generate a no-op Main method.
-                    return;
+                    CheckNoEntryPoint(context, compData);
                 }
 
-                bool isMergedTestRunnerAssembly = configOptions.GlobalOptions.IsMergedTestRunnerAssembly();
-                configOptions.GlobalOptions.TryGetValue("build_property.TargetOS", out string? targetOS);
-
-                if (isMergedTestRunnerAssembly)
+                if (compData.OutputKind != OutputKind.ConsoleApplication)
                 {
-                    if (targetOS?.ToLowerInvariant() is "ios" or "iossimulator" or "tvos" or "tvossimulator" or "maccatalyst" or "android" or "browser")
-                    {
-                        context.AddSource("XHarnessRunner.g.cs", GenerateXHarnessTestRunner(methods, aliasMap, assemblyName));
-                    }
-                    else
-                    {
-                        context.AddSource("FullRunner.g.cs", GenerateFullTestRunner(methods, aliasMap, assemblyName));
-                    }
+                    return;
                 }
-                else
+
+                bool alwaysWriteEntryPoint = (compData.EntryPoint is null);
+                if (methods.IsEmpty && !alwaysWriteEntryPoint)
                 {
-                    context.AddSource("SimpleRunner.g.cs", GenerateStandaloneSimpleTestRunner(methods, aliasMap));
+                    // If we have no test methods, assume that this project is not migrated to the new system yet
+                    // and that we shouldn't generate a no-op Main method.
+                    return;
                 }
+
+                AddRunnerSource(context, methods, configOptions, aliasMap, compData);
             });
     }
 
+    private static void AddRunnerSource(SourceProductionContext context, ImmutableArray<ITestInfo> methods, AnalyzerConfigOptionsProvider configOptions, ImmutableDictionary<string, string> aliasMap, CompData compData)
+    {
+        bool isMergedTestRunnerAssembly = configOptions.GlobalOptions.IsMergedTestRunnerAssembly();
+        configOptions.GlobalOptions.TryGetValue("build_property.TargetOS", out string? targetOS);
+        string assemblyName = compData.AssemblyName;
+
+        if (isMergedTestRunnerAssembly)
+        {
+            if (targetOS?.ToLowerInvariant() is "ios" or "iossimulator" or "tvos" or "tvossimulator" or "maccatalyst" or "android" or "browser")
+            {
+                context.AddSource("XHarnessRunner.g.cs", GenerateXHarnessTestRunner(methods, aliasMap, assemblyName));
+            }
+            else
+            {
+                context.AddSource("FullRunner.g.cs", GenerateFullTestRunner(methods, aliasMap, assemblyName));
+            }
+        }
+        else
+        {
+            context.AddSource("SimpleRunner.g.cs", GenerateStandaloneSimpleTestRunner(methods, aliasMap));
+        }
+    }
+
+    private static void CheckNoEntryPoint(SourceProductionContext context, CompData compData)
+    {
+        foreach (IMethodSymbol entryPoint in compData.PossibleEntryPoints)
+        {
+            context.ReportDiagnostic(Diagnostic.Create(Descriptors.XUWG1001, entryPoint.Locations[0]));
+        }
+    }
+
     private static void AppendAliasMap(CodeBuilder builder, ImmutableDictionary<string, string> aliasMap)
     {
         bool didOutput = false;
@@ -312,7 +356,7 @@ private static string GenerateXHarnessTestRunner(ImmutableArray<ITestInfo> testI
             builder.AppendLine("System.Collections.Generic.HashSet<string> testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();");
             builder.AppendLine($@"return await XHarnessRunnerLibrary.RunnerEntryPoint.RunTests(RunTests, ""{assemblyName}"", args.Length != 0 ? args[0] : null, testExclusionList);");
         }
-        builder.AppendLine("catch(System.Exception ex)");
+        builder.AppendLine("catch (System.Exception ex)");
         using (builder.NewBracesScope())
         {
             builder.AppendLine("System.Console.WriteLine(ex.ToString());");
@@ -435,7 +479,7 @@ private static string GenerateStandaloneSimpleTestRunner(ImmutableArray<ITestInf
                         builder.Append(testInfo.GenerateTestExecution(reporter));
                     }
                 }
-                builder.AppendLine("catch(System.Exception ex)");
+                builder.AppendLine("catch (System.Exception ex)");
                 using (builder.NewBracesScope())
                 {
                     builder.AppendLine("System.Console.WriteLine(ex.ToString());");

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.props

@@ -6,6 +6,7 @@
     <CompilerVisibleProperty Include="TargetArchitecture" />
     <CompilerVisibleProperty Include="Priority" />
     <!-- Properties that influence test harness generation -->
+    <CompilerVisibleProperty Include="InMergedTestDirectory" />
     <CompilerVisibleProperty Include="IsMergedTestRunnerAssembly" />
     <CompilerVisibleProperty Include="TestFilter" />
     <CompilerVisibleItemMetadata Include="AdditionalFiles" MetadataName="IsOutOfProcessTestAssembly" />

src/tests/Common/XUnitWrapperLibrary/TestSummary.cs

@@ -120,11 +120,18 @@ public void WriteFooterToTempLog(StreamWriter tempLogSw)
         tempLogSw.WriteLine("</assembly>");
     }
 
+    public void ReportStartingTest(string name, TextWriter outTw)
+    {
+        outTw.WriteLine("{0:HH:mm:ss.fff} Running test: {1}", System.DateTime.Now, name);
+        outTw.Flush();
+    }
+
     public void ReportPassedTest(string name,
                                  string containingTypeName,
                                  string methodName,
                                  TimeSpan duration,
                                  string output,
+                                 TextWriter outTw,
                                  StreamWriter tempLogSw,
                                  StreamWriter statsCsvSw)
     {
@@ -133,8 +140,10 @@ public void ReportPassedTest(string name,
         var result = new TestResult(name, containingTypeName, methodName, duration, null, null, output);
         _testResults.Add(result);
 
+        outTw.WriteLine($"{0:HH:mm:ss.fff} Passed test: {1}", System.DateTime.Now, name);
         statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}");
         tempLogSw.WriteLine(result.ToXmlString());
+        outTw.Flush();
         statsCsvSw.Flush();
         tempLogSw.Flush();
     }
@@ -145,6 +154,7 @@ public void ReportFailedTest(string name,
                                  TimeSpan duration,
                                  Exception ex,
                                  string output,
+                                 TextWriter outTw,
                                  StreamWriter tempLogSw,
                                  StreamWriter statsCsvSw)
     {
@@ -153,8 +163,11 @@ public void ReportFailedTest(string name,
         var result = new TestResult(name, containingTypeName, methodName, duration, ex, null, output);
         _testResults.Add(result);
 
+        outTw.WriteLine(ex);
+        outTw.WriteLine("{0:HH:mm:ss.fff} Failed test: {1}", System.DateTime.Now, name);
         statsCsvSw.WriteLine($"{TotalTests},{PassedTests},{FailedTests},{SkippedTests}");
         tempLogSw.WriteLine(result.ToXmlString());
+        outTw.Flush();
         statsCsvSw.Flush();
         tempLogSw.Flush();
     }

src/tests/Directory.Build.targets

@@ -510,7 +510,7 @@
     </ItemGroup>
   </Target>
 
-  <PropertyGroup Condition="'$(Language)' == 'C#' and ('$(BuildAsStandalone)' == 'true' or '$(RequiresProcessIsolation)' == 'true' or '$(IsMergedTestRunnerAssembly)' == 'true')">
+  <PropertyGroup Condition="'$(Language)' == 'C#' and ('$(BuildAsStandalone)' == 'true' or '$(RequiresProcessIsolation)' == 'true' or '$(InMergedTestDirectory)' == 'true' or '$(IsMergedTestRunnerAssembly)' == 'true')">
     <ReferenceXUnitWrapperGenerator Condition="'$(ReferenceXUnitWrapperGenerator)' == ''">true</ReferenceXUnitWrapperGenerator>
   </PropertyGroup>