TestConsole supporting parallel execution (#17993)

tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj

@@ -11,6 +11,9 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="WriteCodeFragmentTests.fs" />
     <Compile Include="MapSourceRootsTests.fs" />
   </ItemGroup>

tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs

@@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
     | CompilationResult.Success ({OutputPath = Some dllFile} as s) -> 
        let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
        File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
-       let _exitCode, _stdout, stderr, _exn =  CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
+       let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
 
-       Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
+       Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")
 
     | _ -> failwith (sprintf "%A" compilationResult)
 

tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj

@@ -28,6 +28,9 @@
 </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="CompilerDirectives\Line.fs" />
     <Compile Include="CompilerDirectives\Ifdef.fs" />
     <Compile Include="CompilerDirectives\NonStringArgs.fs" />

tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs

@@ -31,12 +31,13 @@ module ScriptRunner =
         let cu  = cu |> withDefines defaultDefines
         match cu with 
         | FS fsSource ->
+            use capture = new TestConsole.ExecutionCapture()
             let engine = createEngine (fsSource.Options |> Array.ofList,version)
             let res = evalScriptFromDiskInSharedSession engine cu
             match res with
             | CompilationResult.Failure _ -> res
             | CompilationResult.Success s ->
-                if engine.GetOutput().Contains "TEST PASSED OK" then
+                if capture.OutText |> TestFramework.outputPassed then
                     res
                 else
                     failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}"       

tests/FSharp.Compiler.ComponentTests/xunit.runner.json

@@ -1,7 +1,6 @@
 {
-    "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
-    "appDomain": "ifAvailable",
-    "shadowCopy": false,
-    "parallelizeTestCollections": false,
-    "maxParallelThreads": 1
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeTestCollections": false,
+  "maxParallelThreads": 1
 }

tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs

@@ -13,6 +13,7 @@ open FSharp.Compiler.DependencyManager
 open FSharp.Compiler.Diagnostics
 open FSharp.DependencyManager.Nuget
 open FSharp.Test.ScriptHelpers
+open FSharp.Test
 open FSharp.Test.Utilities
 
 open Internal.Utilities
@@ -148,8 +149,7 @@ type DependencyManagerInteractiveTests() =
         Assert.Equal(0, result.Roots |> Seq.length)
         ()
 
-
-    [<Fact(Skip="failing on main")>]
+    [<Fact>]
     member _.``Multiple Instances of DependencyProvider should be isolated``() =
 
         let assemblyProbingPaths () = Seq.empty<string>
@@ -721,101 +721,65 @@ x |> Seq.iter(fun r ->
 
     [<Fact>]
     member _.``Verify that #help produces help text for fsi + dependency manager``() =
-        let expected = [|
-            """  F# Interactive directives:"""
-            """"""
-            """    #r "file.dll";;                               // Reference (dynamically load) the given DLL"""
-            """    #i "package source uri";;                     // Include package source uri when searching for packages"""
-            """    #I "path";;                                   // Add the given search path for referenced DLLs"""
-            """    #load "file.fs" ...;;                         // Load the given file(s) as if compiled and referenced"""
-            """    #time ["on"|"off"];;                          // Toggle timing on/off"""
-            """    #clear;;                                      // Clear screen"""
-            """    #help;;                                       // Display help"""
-            """    #help "idn";;                                 // Display documentation for an identifier, e.g. #help "List.map";;"""
-            """    #quit;;                                       // Exit"""
-            """"""
-            """  F# Interactive command line options:"""
-            """"""
-
-            // this is the end of the line each different platform has a different mechanism for starting fsi
-            // Actual output looks similar to: """      See 'testhost --help' for options"""
-            """--help' for options"""
-
-            """"""
-            """"""
-        |]
+        let expected = """
+  F# Interactive directives:
+
+    #r "file.dll";;                               // Reference (dynamically load) the given DLL
+    #i "package source uri";;                     // Include package source uri when searching for packages
+    #I "path";;                                   // Add the given search path for referenced DLLs
+    #load "file.fs" ...;;                         // Load the given file(s) as if compiled and referenced
+    #time ["on"|"off"];;                          // Toggle timing on/off
+    #help;;                                       // Display help
+    #help "idn";;                                 // Display documentation for an identifier, e.g. #help "List.map";;
+    #clear;;                                      // Clear screen
+    #quit;;                                       // Exit
+
+  F# Interactive command line options:"""
 
-        let mutable found = 0
-        let lines = System.Collections.Generic.List()
-        use sawExpectedOutput = new ManualResetEvent(false)
-        let verifyOutput (line: string) =
-            let compareLine (s: string) =
-                if s = "" then line = ""
-                else line.EndsWith(s)
-            lines.Add(line)
-            match expected |> Array.tryFind(compareLine) with
-            | None -> ()
-            | Some t ->
-                found <- found + 1
-                if found = expected.Length then sawExpectedOutput.Set() |> ignore
-
-        let text = "#help"
         use script = new FSharpScript(quiet = false, langVersion = LangVersion.V47)
-        let mutable found = 0
-        script.OutputProduced.Add (fun line -> verifyOutput line)
-        let opt = script.Eval(text) |> getValue
-        Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines)
 
+        use capture = new TestConsole.ExecutionCapture()
+        let opt = script.Eval("#help") |> getValue
+
+        let output = capture.OutText
+
+        Assert.Contains(expected, output)
+
+        // this is the end of the line each different platform has a different mechanism for starting fsi
+        // Actual output looks similar to: """      See 'testhost --help' for options"""
+        Assert.EndsWith("--help' for options", output.Trim())
 
     [<Fact>]
     member _.``Verify that #help produces help text for fsi + dependency manager language version preview``() =
-        let expected = [|
-            """  F# Interactive directives:"""
-            """"""
-            """    #r "file.dll";;                               // Reference (dynamically load) the given DLL"""
-            """    #i "package source uri";;                     // Include package source uri when searching for packages"""
-            """    #I "path";;                                   // Add the given search path for referenced DLLs"""
-            """    #load "file.fs" ...;;                         // Load the given file(s) as if compiled and referenced"""
-            """    #time ["on"|"off"];;                          // Toggle timing on/off"""
-            """    #help;;                                       // Display help"""
-            """    #help "idn";;                                 // Display documentation for an identifier, e.g. #help "List.map";;"""
-            """    #r "nuget:FSharp.Data, 3.1.2";;               // Load Nuget Package 'FSharp.Data' version '3.1.2'"""
-            """    #r "nuget:FSharp.Data";;                      // Load Nuget Package 'FSharp.Data' with the highest version"""
-            """    #clear;;                                      // Clear screen"""
-            """    #quit;;                                       // Exit"""
-            """"""
-            """  F# Interactive command line options:"""
-            """"""
-
-            // this is the end of the line each different platform has a different mechanism for starting fsi
-            // Actual output looks similar to: """      See 'testhost --help' for options"""
-            """--help' for options"""
-
-            """"""
-            """"""
-        |]
+        let expected = """
+  F# Interactive directives:
+
+    #r "file.dll";;                               // Reference (dynamically load) the given DLL
+    #i "package source uri";;                     // Include package source uri when searching for packages
+    #I "path";;                                   // Add the given search path for referenced DLLs
+    #load "file.fs" ...;;                         // Load the given file(s) as if compiled and referenced
+    #time ["on"|"off"];;                          // Toggle timing on/off
+    #help;;                                       // Display help
+    #help "idn";;                                 // Display documentation for an identifier, e.g. #help "List.map";;
+    #r "nuget:FSharp.Data, 3.1.2";;               // Load Nuget Package 'FSharp.Data' version '3.1.2'
+    #r "nuget:FSharp.Data";;                      // Load Nuget Package 'FSharp.Data' with the highest version
+    #clear;;                                      // Clear screen
+    #quit;;                                       // Exit
+
+  F# Interactive command line options:"""
 
-        let mutable found = 0
-        let lines = System.Collections.Generic.List()
-        use sawExpectedOutput = new ManualResetEvent(false)
-        let verifyOutput (line: string) =
-            let compareLine (s: string) =
-                if s = "" then line = ""
-                else line.EndsWith(s)
-            lines.Add(line)
-            match expected |> Array.tryFind(compareLine) with
-            | None -> ()
-            | Some t ->
-                found <- found + 1
-                if found = expected.Length then sawExpectedOutput.Set() |> ignore
-
-        let text = "#help"
         use script = new FSharpScript(quiet = false, langVersion = LangVersion.Preview)
-        let mutable found = 0
-        script.OutputProduced.Add (fun line -> verifyOutput line)
-        let opt = script.Eval(text) |> getValue
-        Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines)
 
+        use capture = new TestConsole.ExecutionCapture()
+        let opt = script.Eval("#help") |> getValue
+
+        let output = capture.OutText
+
+        Assert.Contains(expected, output)
+
+        // this is the end of the line each different platform has a different mechanism for starting fsi
+        // Actual output looks similar to: """      See 'testhost --help' for options"""
+        Assert.EndsWith("--help' for options", output.Trim())
 
     [<Fact>]
     member _.``Verify that timeout --- times out and fails``() =

tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj

@@ -12,6 +12,9 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="$(FSharpSourcesRoot)\Compiler\Interactive\ControlledExecution.fs" />
     <Compile Include="$(FSharpSourcesRoot)\Compiler\Utilities\RidHelpers.fs" />
     <Compile Include="DependencyManagerInteractiveTests.fs" />

tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs

@@ -3,6 +3,7 @@
 namespace FSharp.Compiler.Scripting.UnitTests
 
 open System
+open System.Text
 open System.Diagnostics
 open System.IO
 open System.Reflection
@@ -11,6 +12,7 @@ open System.Threading
 open System.Threading.Tasks
 open FSharp.Compiler.Interactive
 open FSharp.Compiler.Interactive.Shell
+open FSharp.Test
 open FSharp.Test.ScriptHelpers
 
 open Xunit
@@ -87,22 +89,20 @@ x
 
     [<Fact>]
     member _.``Capture console input``() =
-        use script = new FSharpScript(input = "stdin:1234\r\n")
+        use _ = new TestConsole.ProvideInput("stdin:1234\r\n")
+        use script = new FSharpScript()
         let opt = script.Eval("System.Console.ReadLine()") |> getValue
         let value = opt.Value
         Assert.Equal(typeof<string>, value.ReflectionType)
         Assert.Equal("stdin:1234", downcast value.ReflectionValue)
 
     [<Fact>]
     member _.``Capture console output/error``() =
+        use capture = new TestConsole.ExecutionCapture()
         use script = new FSharpScript()
-        use sawOutputSentinel = new ManualResetEvent(false)
-        use sawErrorSentinel = new ManualResetEvent(false)
-        script.OutputProduced.Add (fun line -> if line = "stdout:1234" then sawOutputSentinel.Set() |> ignore)
-        script.ErrorProduced.Add (fun line -> if line = "stderr:5678" then sawErrorSentinel.Set() |> ignore)
         script.Eval("printfn \"stdout:1234\"; eprintfn \"stderr:5678\"") |> ignoreValue
-        Assert.True(sawOutputSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see output sentinel value written")
-        Assert.True(sawErrorSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see error sentinel value written")
+        Assert.Contains("stdout:1234", capture.OutText)
+        Assert.Contains("stderr:5678", capture.ErrorText)
 
     [<Fact>]
     member _.``Maintain state between submissions``() =
@@ -306,30 +306,26 @@ printfn "{@"%A"}" result
 
     [<Fact>]
     member _.``Eval script with invalid PackageName should fail immediately``() =
+        use capture = new TestConsole.ExecutionCapture()
         use script = new FSharpScript(additionalArgs=[| |])
-        let mutable found = 0
-        let outp = System.Collections.Generic.List<string>()
-        script.OutputProduced.Add(
-            fun line ->
-                if line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package") then
-                    found <- found + 1
-                outp.Add(line))
         let result, errors = script.Eval("""#r "nuget:FSharp.Really.Not.A.Package" """)
-        Assert.True( (found = 0), "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'")
+
+        let lines = capture.OutText.Split([| Environment.NewLine |], StringSplitOptions.None)
+        let found = lines |> Seq.exists (fun line -> line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package"))
+        Assert.False(found, "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'")
         Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'")
         Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'")
 
     [<Fact>]
     member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() =
+        use capture = new TestConsole.ExecutionCapture()
         use script = new FSharpScript(additionalArgs=[| |])
-        let mutable foundResolve = 0
-        script.OutputProduced.Add (fun line -> if line.Contains("error NU1101:") then foundResolve <- foundResolve + 1)
         let result, errors =
             script.Eval("""
 #r "nuget:FSharp.Really.Not.A.Package"
 #r "nuget:FSharp.Really.Not.Another.Package"
                 """)
-        Assert.True( (foundResolve = 0), (sprintf "Did not expected to see 'error NU1101:' in output" ))
+        Assert.DoesNotContain("error NU1101:", capture.OutText)
         Assert.Equal(2, (errors |> Seq.filter (fun error -> error.Message.Contains("error NU1101:")) |> Seq.length))
         Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length))
         Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length))
@@ -479,6 +475,7 @@ let x =
         script.Eval(code) |> ignoreValue
         Assert.False(foundInner)
 
+
     [<Fact>]
     member _.``Script with nuget package that yields out of order dependencies works correctly``() =
         // regression test for: https://github.com/dotnet/fsharp/issues/9217

tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json

@@ -1,7 +1,6 @@
 {
     "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
-    "appDomain": "ifAvailable",
-    "shadowCopy": false,
+    "appDomain": "denied",
     "parallelizeTestCollections": false,
     "maxParallelThreads": 1
 }

tests/FSharp.Compiler.Service.Tests/Common.fs

@@ -476,7 +476,7 @@ let assertRange
 [<AutoOpen>]
 module TempDirUtils =
     let getTempPath dir =
-        Path.Combine(TestFramework.tempDirectoryOfThisTestRun, dir)
+        Path.Combine(tempDirectoryOfThisTestRun.Value.FullName, dir)
 
     /// Returns the file name part of a temp file name created with tryCreateTemporaryFileName ()
     /// and an added process id and thread id to ensure uniqueness between threads.

tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj

@@ -21,6 +21,9 @@
     </Content>
     <Compile Include="AssemblyInfo.fs" />
     <Compile Include="SurfaceArea.fs" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="Common.fs" />
     <Compile Include="GeneratedCodeSymbolsTests.fs" />
     <Compile Include="AssemblyReaderShim.fs" />

tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs

@@ -4401,7 +4401,7 @@ let ``Test Project33 extension methods`` () =
              ("GetValue", ["member"; "extmem"])]
 
 module internal Project34 =
-    let directoryPath = createTemporaryDirectory "Project34"
+    let directoryPath = createTemporaryDirectory().FullName
     let sourceFileName = Path.Combine(directoryPath, "Program.fs")
     let dllName = Path.ChangeExtension(sourceFileName, ".dll")
     let projFileName = Path.ChangeExtension(sourceFileName, ".fsproj")

tests/FSharp.Compiler.Service.Tests/xunit.runner.json

@@ -1,5 +1,4 @@
 {
-    "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
-    "appDomain": "ifAvailable",
-    "shadowCopy": false
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied"
 }

tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj

@@ -26,6 +26,9 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
+      <Link>XunitSetup.fs</Link>
+    </Compile>
     <Compile Include="TestFrameworkHelpers.fs" />
     <Compile Include="LibraryTestFx.fs" />
     <Compile Include="FSharp.Core\PrimTypes.fs" />

tests/FSharp.Core.UnitTests/xunit.runner.json

@@ -1,7 +1,6 @@
 {
-    "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
-    "appDomain": "ifAvailable",
-    "shadowCopy": false,
-    "parallelizeTestCollections": false,
-    "maxParallelThreads": 1
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeTestCollections": false,
+  "maxParallelThreads": 1
 }