From 0329320d58616f8c5bc765da5f3dafa0477b1271 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 16 Dec 2019 21:06:28 +0100 Subject: [PATCH 01/14] Improve idle message --- src/app/Fake.Core.Target/Target.fs | 38 +++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/app/Fake.Core.Target/Target.fs b/src/app/Fake.Core.Target/Target.fs index a568c93ac2a..16cffd0f14c 100644 --- a/src/app/Fake.Core.Target/Target.fs +++ b/src/app/Fake.Core.Target/Target.fs @@ -748,14 +748,18 @@ module Target = waitList <- [] reply.Reply (async.Return(ctx, None)) else - let isRunnable (t:Target) = - not (known.ContainsKey (String.toLower t.Name)) && // not already finised - not (runningTasks |> Seq.exists (fun r -> String.toLower r.Name = String.toLower t.Name)) && // not already running - t.Dependencies @ List.filter inResolution t.SoftDependencies // all dependencies finished - |> Seq.forall (String.toLower >> known.ContainsKey) - let runnable = + let calculateOpenTargets() = + let isOpen (t:Target) = + not (known.ContainsKey (String.toLower t.Name)) && // not already finised + not (runningTasks |> Seq.exists (fun r -> String.toLower r.Name = String.toLower t.Name)) // not already running order |> Seq.concat + |> Seq.filter isOpen + let runnable = + let isRunnable (t:Target) = + t.Dependencies @ List.filter inResolution t.SoftDependencies // all dependencies finished + |> Seq.forall (String.toLower >> known.ContainsKey) + calculateOpenTargets() |> Seq.filter isRunnable |> Seq.toList @@ -784,9 +788,25 @@ module Target = // queue work let tcs = new TaskCompletionSource() let running = System.String.Join(", ", runningTasks |> Seq.map (fun t -> sprintf "'%s'" t.Name)) - let openList = System.String.Join(", ", runnable |> Seq.map (fun t -> sprintf "'%s'" t.Name)) - Trace.tracefn "FAKE worker idle because %d Targets (%s) are still running and all open targets (%s) depend on those. You might improve performance by splitting targets or removing dependencies." - runningTasks.Length running openList + // recalculate openTargets as getNextFreeRunableTarget could change runningTasks + let openTargets = calculateOpenTargets() |> Seq.toList + let orderedOpen = + let isDependencyResolved = + String.toLower >> known.ContainsKey + let isDependencyRunning t = + runningTasks + |> Seq.exists (fun running -> String.toLower running.Name = t) + let isDependencyResolvedOrRunning t = isDependencyResolved t || isDependencyRunning t + openTargets + |> List.sortBy (fun t -> + t.Dependencies @ List.filter inResolution t.SoftDependencies // Order by unresolved dependencies + |> Seq.filter (isDependencyResolvedOrRunning >> not) + |> Seq.length) + let openList = + System.String.Join(", ", orderedOpen :> seq<_> |> (if orderedOpen.Length > 3 then Seq.take 3 else id) |> Seq.map (fun t -> sprintf "'%s'" t.Name)) + + (if orderedOpen.Length > 3 then ", ..." else "") + Trace.tracefn "FAKE worker idle because '%d' targets (%s) are still running and all ('%d') open targets (%s) depend on those. You might improve performance by splitting targets or removing dependencies." + runningTasks.Length running openTargets.Length openList waitList <- waitList @ [ tcs ] reply.Reply (tcs.Task |> Async.AwaitTask) with e -> From 8ecf77a61cb6838ca69eafbe3321ec0fa2ca9008 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 16 Dec 2019 21:13:43 +0100 Subject: [PATCH 02/14] add debug flag --- build.fsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.fsx b/build.fsx index 612cb66a22a..52e4d8a8737 100644 --- a/build.fsx +++ b/build.fsx @@ -645,7 +645,7 @@ Target.create "HostDocs" (fun _ -> let runExpecto workDir dllPath resultsXml = let processResult = - DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary" + DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary --debug" if processResult.ExitCode <> 0 then failwithf "Tests in %s failed." (Path.GetFileName dllPath) Trace.publish (ImportData.Nunit NunitDataVersion.Nunit) (workDir resultsXml) From d6abeed56ba12bd134225c5d1791e92769212008 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 16 Dec 2019 21:24:26 +0100 Subject: [PATCH 03/14] enable sequenced as well --- build.fsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.fsx b/build.fsx index 52e4d8a8737..3d0ddeea25d 100644 --- a/build.fsx +++ b/build.fsx @@ -645,7 +645,7 @@ Target.create "HostDocs" (fun _ -> let runExpecto workDir dllPath resultsXml = let processResult = - DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary --debug" + DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary --debug --sequenced" if processResult.ExitCode <> 0 then failwithf "Tests in %s failed." (Path.GetFileName dllPath) Trace.publish (ImportData.Nunit NunitDataVersion.Nunit) (workDir resultsXml) From 54425d9131d414100f1904ca77a8141d2662c7cc Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 16 Dec 2019 22:19:03 +0100 Subject: [PATCH 04/14] see if it is thread pool starvation --- src/test/Fake.Core.CommandLine.UnitTests/Main.fs | 1 + src/test/Fake.Core.IntegrationTests/Main.fs | 1 + src/test/Fake.Core.UnitTests/Main.fs | 1 + src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs | 1 + src/test/Fake.ExpectoSupport/ExpectoHelpers.fs | 2 ++ 5 files changed, 6 insertions(+) diff --git a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs index 09381ecc116..d711fd31eaa 100644 --- a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs +++ b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs @@ -5,6 +5,7 @@ open System [] let main argv = + ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_Core_CommandLine_UnitTests.TestResults.xml", "Fake.Core.CommandLine.UnitTests") let config = defaultConfig.appendSummaryHandler writeResults diff --git a/src/test/Fake.Core.IntegrationTests/Main.fs b/src/test/Fake.Core.IntegrationTests/Main.fs index 881031e3e19..7bfd1767563 100644 --- a/src/test/Fake.Core.IntegrationTests/Main.fs +++ b/src/test/Fake.Core.IntegrationTests/Main.fs @@ -5,6 +5,7 @@ open System [] let main argv = + ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_Core_IntegrationTests.TestResults.xml", "Fake.Core.IntegrationTests") let config = defaultConfig.appendSummaryHandler writeResults diff --git a/src/test/Fake.Core.UnitTests/Main.fs b/src/test/Fake.Core.UnitTests/Main.fs index 926acd264e6..c076d531957 100644 --- a/src/test/Fake.Core.UnitTests/Main.fs +++ b/src/test/Fake.Core.UnitTests/Main.fs @@ -6,6 +6,7 @@ open Fake.ExpectoSupport [] let main argv = + ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_Core_UnitTests.TestResults.xml", "Fake.Core.UnitTests") let config = defaultConfig.appendSummaryHandler writeResults diff --git a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs index 2d10eb0d259..f7b326e30f4 100644 --- a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs +++ b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs @@ -5,6 +5,7 @@ open System [] let main argv = + ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_DotNet_Cli_IntegrationTests.TestResults.xml", "Fake.DotNet.Cli.IntegrationTests") let config = defaultConfig.appendSummaryHandler writeResults diff --git a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs index 8307c0a5273..8c25687961a 100644 --- a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs +++ b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs @@ -5,6 +5,8 @@ open System.Threading open System.Threading.Tasks module ExpectoHelpers = + let setThreadPool () = + ThreadPool.SetMinThreads(100, 100) let addFilter f (config:Impl.ExpectoConfig) = { config with filter = fun test -> From 241118304ab79321921ab8d7e4c7cfb85f232c94 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Mon, 16 Dec 2019 23:12:47 +0100 Subject: [PATCH 05/14] remove spinner --- build.fsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.fsx b/build.fsx index 3d0ddeea25d..7e48a6403b7 100644 --- a/build.fsx +++ b/build.fsx @@ -645,7 +645,7 @@ Target.create "HostDocs" (fun _ -> let runExpecto workDir dllPath resultsXml = let processResult = - DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary --debug --sequenced" + DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary --debug --sequenced --no-spinner" if processResult.ExitCode <> 0 then failwithf "Tests in %s failed." (Path.GetFileName dllPath) Trace.publish (ImportData.Nunit NunitDataVersion.Nunit) (workDir resultsXml) From d92525aed2cb55809b8110a430b901edab1cb46e Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Tue, 17 Dec 2019 00:12:45 +0100 Subject: [PATCH 06/14] inline expecto and disable printer --- .../Fake.Core.CommandLine.UnitTests/Main.fs | 3 +- src/test/Fake.Core.IntegrationTests/Main.fs | 3 +- src/test/Fake.Core.UnitTests/Main.fs | 3 +- .../Fake.DotNet.Cli.IntegrationTests/Main.fs | 3 +- .../Fake.ExpectoSupport/ExpectoHelpers.fs | 2 - src/test/Fake.ExpectoSupport/FakeExpecto.fs | 1396 +++++++++++++++++ 6 files changed, 1400 insertions(+), 10 deletions(-) create mode 100644 src/test/Fake.ExpectoSupport/FakeExpecto.fs diff --git a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs index d711fd31eaa..f472c2617de 100644 --- a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs +++ b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs @@ -5,10 +5,9 @@ open System [] let main argv = - ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_Core_CommandLine_UnitTests.TestResults.xml", "Fake.Core.CommandLine.UnitTests") let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - Tests.runTestsInAssembly { config with parallel = false } argv + FakeExpecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.Core.IntegrationTests/Main.fs b/src/test/Fake.Core.IntegrationTests/Main.fs index 7bfd1767563..b7bea83a086 100644 --- a/src/test/Fake.Core.IntegrationTests/Main.fs +++ b/src/test/Fake.Core.IntegrationTests/Main.fs @@ -5,9 +5,8 @@ open System [] let main argv = - ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_Core_IntegrationTests.TestResults.xml", "Fake.Core.IntegrationTests") let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - Tests.runTestsInAssembly { config with parallel = false } argv + FakeExpecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.Core.UnitTests/Main.fs b/src/test/Fake.Core.UnitTests/Main.fs index c076d531957..4051a509135 100644 --- a/src/test/Fake.Core.UnitTests/Main.fs +++ b/src/test/Fake.Core.UnitTests/Main.fs @@ -6,9 +6,8 @@ open Fake.ExpectoSupport [] let main argv = - ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_Core_UnitTests.TestResults.xml", "Fake.Core.UnitTests") let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - Tests.runTestsInAssembly config argv + FakeExpecto.Tests.runTestsInAssembly config argv diff --git a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs index f7b326e30f4..b357e22012f 100644 --- a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs +++ b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs @@ -5,9 +5,8 @@ open System [] let main argv = - ExpectoHelpers.setThreadPool() let writeResults = TestResults.writeNUnitSummary ("Fake_DotNet_Cli_IntegrationTests.TestResults.xml", "Fake.DotNet.Cli.IntegrationTests") let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - Tests.runTestsInAssembly { config with parallel = false } argv + FakeExpecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs index 8c25687961a..8307c0a5273 100644 --- a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs +++ b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs @@ -5,8 +5,6 @@ open System.Threading open System.Threading.Tasks module ExpectoHelpers = - let setThreadPool () = - ThreadPool.SetMinThreads(100, 100) let addFilter f (config:Impl.ExpectoConfig) = { config with filter = fun test -> diff --git a/src/test/Fake.ExpectoSupport/FakeExpecto.fs b/src/test/Fake.ExpectoSupport/FakeExpecto.fs new file mode 100644 index 00000000000..d1410e71ef2 --- /dev/null +++ b/src/test/Fake.ExpectoSupport/FakeExpecto.fs @@ -0,0 +1,1396 @@ +namespace FakeExpecto + +open System.Globalization + +#nowarn "46" +open Expecto +open System +open System.Diagnostics +open System.Reflection +open System.Runtime.CompilerServices +open System.Threading +open System.Threading.Tasks +module internal Helpers = + let inline dispose (d:IDisposable) = d.Dispose() + let inline addFst a b = a,b + let inline addSnd b a = a,b + let inline fst3 (a,_,_) = a + let inline commaString (i:int) = i.ToString("#,##0") + let inline tryParse (s: string) = + let mutable r = Unchecked.defaultof<_> + if (^a : (static member TryParse: string * ^a byref -> bool) (s, &r)) + then Some r else None + let inline tryParseNumber (s: string) = + let mutable r = Unchecked.defaultof<_> + if (^a : (static member TryParse: string * NumberStyles * IFormatProvider * ^a byref -> bool) (s, NumberStyles.Any, CultureInfo.InvariantCulture, &r)) + then Some r else None + + type ResizeMap<'k,'v> = Collections.Generic.Dictionary<'k,'v> + + module Option = + let orDefault def = + function | Some a -> a | None -> def + let orFun fn = + function | Some a -> a | None -> fn() + + module Result = + let traverse f list = + List.fold (fun s i -> + match s,f i with + | Ok l, Ok h -> Ok (h::l) + | Error l, Ok _ -> Error l + | Ok _, Error e -> Error [e] + | Error l, Error h -> Error (h::l) + ) (Ok []) list + let sequence list = traverse id list + + module Async = + open System + open System.Threading + open System.Threading.Tasks + + let map fn a = + async { + let! v = a + return fn v + } + + let bind fn a = + async.Bind(a, fn) + + let foldSequentiallyWithCancel (ct: CancellationToken) folder state (s:_ seq) = + async { + let mutable state = state + let tcs = TaskCompletionSource() + use _ = Action tcs.SetResult |> ct.Register + let tasks: Task[] = [| null; tcs.Task |] + use e = s.GetEnumerator() + while not ct.IsCancellationRequested && e.MoveNext() do + let task = Async.StartAsTask e.Current + tasks.[0] <- task :> Task + if Task.WaitAny tasks = 0 then + state <- folder state task.Result + return state + } + + let foldSequentially folder state (s: _ seq) = + foldSequentiallyWithCancel CancellationToken.None folder state s + + let foldParallelWithCancel maxParallelism (ct: CancellationToken) folder state (s: _ seq) = + async { + let mutable state = state + use e = s.GetEnumerator() + if e.MoveNext() then + let mutable tasks = [Async.StartAsTask e.Current] + while not(ct.IsCancellationRequested || List.isEmpty tasks) do + if List.length tasks = maxParallelism || not(e.MoveNext()) then + while not( tasks |> List.exists (fun t -> t.IsCompleted) + || ct.IsCancellationRequested) do + do! Async.Sleep 10 + tasks |> List.tryFindIndex (fun t -> t.IsCompleted) + |> Option.iter (fun i -> + let a,b = List.splitAt i tasks + state <- (List.head b).Result |> folder state + tasks <- a @ List.tail b + ) + else tasks <- Async.StartAsTask e.Current :: tasks + return state + } + + let matchFocusAttributes = function + | "Expecto.FTestsAttribute" -> Some (1, Focused) + | "Expecto.TestsAttribute" -> Some (2, Normal) + | "Expecto.PTestsAttribute" -> Some (3, Pending) + | _ -> None + + let allTestAttributes = + Set [ + typeof.FullName + typeof.FullName + typeof.FullName + ] + + type MemberInfo with + member m.HasAttributePred (pred: Type -> bool) = + m.GetCustomAttributes true + |> Seq.filter (fun a -> pred(a.GetType())) + |> Seq.length |> (<) 0 + + member m.HasAttributeType (attr: Type) = + m.HasAttributePred ((=) attr) + + member m.HasAttribute (attr: string) = + m.HasAttributePred (fun (t: Type) -> t.FullName = attr) + + member m.GetAttributes (attr: string) : Attribute seq = + m.GetCustomAttributes true + |> Seq.filter (fun a -> a.GetType().FullName = attr) + |> Seq.cast + + member m.MatchTestsAttributes () = + m.GetCustomAttributes true + |> Seq.map (fun t -> t.GetType().FullName) + |> Set.ofSeq + |> Set.intersect allTestAttributes + |> Set.toList + |> List.choose matchFocusAttributes + |> List.sortBy fst + |> List.map snd + |> List.tryFind (fun _ -> true) + + + +open Helpers + +type private TestNameHolder() = + [] + static val mutable private name : string + static member Name + with get () = TestNameHolder.name + and set name = TestNameHolder.name <- name + +// When exposing Extension Methods, you should declare an assembly-level attribute (in addition to class and method) +[] +do () + +type FlatTest = Expecto.FlatTest + +[] +module Test = + /// Compute the child test state based on parent test state + let computeChildFocusState parentState childState = + match parentState, childState with + | Focused, Pending -> Pending + | Pending, _ -> Pending + | Focused, _ -> Focused + | Normal, _ -> childState + + /// Is focused set on at least one test + let rec isFocused test = + match test with + | TestLabel (_,_,Focused) + | TestCase (_,Focused) + | TestList (_,Focused) -> true + | TestLabel (_,_,Pending) + | TestList (_,Pending) + | TestCase _ -> false + | TestLabel (_,test,Normal) + | Test.Sequenced (_,test) -> isFocused test + | TestList (tests,Normal) -> List.exists isFocused tests + + /// Flattens a tree of tests + let toTestCodeList test = + let isFocused = isFocused test + let rec loop parentName testList parentState sequenced = + function + | TestLabel (name, test, state) -> + let fullName = + if String.IsNullOrEmpty parentName + then name + else parentName + "/" + name + loop fullName testList (computeChildFocusState parentState state) sequenced test + | TestCase (test, state) -> + { name=parentName + test=test + state=computeChildFocusState parentState state + focusOn = isFocused + sequenced=sequenced } :: testList + | TestList (tests, state) -> List.collect (loop parentName testList (computeChildFocusState parentState state) sequenced) tests + | Test.Sequenced (sequenced,test) -> loop parentName testList parentState sequenced test + loop null [] Normal InParallel test + + let fromFlatTests (tests:FlatTest list) = + TestList( + List.map (fun t -> + TestLabel(t.name, Test.Sequenced(t.sequenced, TestCase (t.test, t.state)), t.state) + ) tests + , Normal) + + let shuffle (test:Test) = Expecto.Test.shuffle test + + /// Change the FocusState by applying the old state to a new state + /// Note: this is not state replacement!!! + /// + /// Used in replaceTestCode and the order is intended for scenario: + /// 1. User wants to automate some tests and his intent is not to change + /// the test state (use Normal), so this way the current state will be preserved + /// + /// Don't see the use case: the user wants to automate some tests and wishes + /// to change the test states + let rec translateFocusState newFocusState = + function + | TestCase (test, oldFocusState) -> TestCase(test, computeChildFocusState oldFocusState newFocusState) + | TestList (testList, oldFocusState) -> TestList(testList, computeChildFocusState oldFocusState newFocusState) + | TestLabel (label, test, oldFocusState) -> TestLabel(label, test, computeChildFocusState oldFocusState newFocusState) + | Test.Sequenced (sequenced,test) -> Test.Sequenced (sequenced,translateFocusState newFocusState test) + + /// Recursively replaces TestCodes in a Test. + /// Check translateFocusState for focus state behaviour description. + let rec replaceTestCode (f:string -> TestCode -> Test) = + function + | TestLabel (label, TestCase (test, childState), parentState) -> + f label test + |> translateFocusState (computeChildFocusState parentState childState) + | TestCase (test, state) -> + f null test + |> translateFocusState state + | TestList (testList, state) -> TestList (List.map (replaceTestCode f) testList, state) + | TestLabel (label, test, state) -> TestLabel (label, replaceTestCode f test, state) + | Test.Sequenced (sequenced,test) -> Test.Sequenced (sequenced,replaceTestCode f test) + + /// Filter tests by name. + let filter pred = + toTestCodeList + >> List.filter (fun t -> pred t.name) + >> List.map (fun t -> + let test = TestLabel (t.name, TestCase (t.test, t.state), t.state) + match t.sequenced with + | InParallel -> + test + | s -> + Test.Sequenced (s,test) + ) + >> (fun x -> TestList (x,Normal)) + + /// Applies a timeout to a test. + let timeout timeout (test: TestCode) : TestCode = + let timeoutAsync testAsync = + async { + try + let! async = Async.StartChild(testAsync, timeout) + do! async + with :? TimeoutException -> + let ts = TimeSpan.FromMilliseconds (float timeout) + raise <| AssertException(sprintf "Timeout (%A)" ts) + } + + match test with + | Sync test -> async { test() } |> timeoutAsync |> Async + | SyncWithCancel test -> + SyncWithCancel (fun ct -> + Async.StartImmediate(async { test ct } |> timeoutAsync) + ) + | Async test -> timeoutAsync test |> Async + | AsyncFsCheck (testConfig, stressConfig, test) -> + AsyncFsCheck (testConfig, stressConfig, test >> timeoutAsync) + + + +// TODO: make internal? +module Impl = + open Expecto.Logging + open Expecto.Logging.Message + open Mono.Cecil + + let mutable logger = Log.create "Expecto" + let setLogName name = logger <- Log.create name + + let rec private exnWithInnerMsg (ex: exn) msg = + let currentMsg = + msg + (sprintf "%s%s" Environment.NewLine (ex.ToString())) + if isNull ex.InnerException then + currentMsg + else + exnWithInnerMsg ex.InnerException currentMsg + + type TestResult = Expecto.Impl.TestResult + + type TestSummary = Expecto.Impl.TestSummary + + type TestRunSummary = Expecto.Impl.TestRunSummary + + let createSummaryMessage (summary: TestRunSummary) = Expecto.Impl.createSummaryMessage summary + + let createSummaryText (summary: TestRunSummary) = Expecto.Impl.createSummaryText summary + + let logSummary (summary: TestRunSummary) = + createSummaryMessage summary + |> logger.logWithAck Info + + let logSummaryWithLocation locate (summary: TestRunSummary) = Expecto.Impl.logSummaryWithLocation locate summary + + /// Hooks to print report through test run + type TestPrinters = Expecto.Impl.TestPrinters + + // Runner options + type ExpectoConfig = Expecto.Impl.ExpectoConfig + + let execTestAsync (ct:CancellationToken) (config:ExpectoConfig) (test:FlatTest) : Async = + async { + let w = Stopwatch.StartNew() + try + match test.shouldSkipEvaluation with + | Some ignoredMessage -> + return TestSummary.single (TestResult.Ignored ignoredMessage) 0.0 + | None -> + TestNameHolder.Name <- test.name + match test.test with + | Sync test -> + test() + | SyncWithCancel test -> + test ct + | Async test -> + do! test + | AsyncFsCheck (testConfig, stressConfig, test) -> + let fsConfig = + match config.stress with + | None -> testConfig + | Some _ -> stressConfig + |> Option.orFun (fun () -> + { FsCheckConfig.defaultConfig with + maxTest = config.fsCheckMaxTests + startSize = config.fsCheckStartSize + endSize = + match config.fsCheckEndSize, config.stress with + | Some i, _ -> i + | None, None -> 100 + | None, Some _ -> 10000 + } + ) + do! test fsConfig + w.Stop() + return TestSummary.single TestResult.Passed (float w.ElapsedMilliseconds) + with + | :? AssertException as e -> + w.Stop() + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + return TestSummary.single (TestResult.Failed msg) (float w.ElapsedMilliseconds) + | :? FailedException as e -> + w.Stop() + return TestSummary.single (TestResult.Failed ("\n"+e.Message)) (float w.ElapsedMilliseconds) + | :? IgnoreException as e -> + w.Stop() + return TestSummary.single (TestResult.Ignored e.Message) (float w.ElapsedMilliseconds) + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + w.Stop() + if e.InnerException :? IgnoreException then + return TestSummary.single (TestResult.Ignored e.InnerException.Message) (float w.ElapsedMilliseconds) + else + return TestSummary.single (TestResult.Error e.InnerException) (float w.ElapsedMilliseconds) + | e -> + w.Stop() + return TestSummary.single (TestResult.Error e) (float w.ElapsedMilliseconds) + } + + let private numberOfWorkers limit (config:ExpectoConfig) = + if config.parallelWorkers < 0 then + -config.parallelWorkers * Environment.ProcessorCount + elif config.parallelWorkers = 0 then + if limit then + Environment.ProcessorCount + else + Int32.MaxValue + else + config.parallelWorkers + + /// Evaluates tests. + let evalTestsWithCancel (ct:CancellationToken) (config:ExpectoConfig) test progressStarted = + async { + + let tests = Test.toTestCodeList test + let testLength = + tests + |> Seq.where (fun t -> Option.isNone t.shouldSkipEvaluation) + |> Seq.length + + let testsCompleted = ref 0 + + let evalTestAsync (test:FlatTest) = + + let beforeEach (test:FlatTest) = + config.printer.beforeEach test.name + + async { + let! beforeAsync = beforeEach test |> Async.StartChild + let! result = execTestAsync ct config test + do! beforeAsync + do! TestPrinters.printResult config test result + + //if progressStarted && Option.isNone test.shouldSkipEvaluation then + // Fraction (Interlocked.Increment testsCompleted, testLength) + // |> ProgressIndicator.update + + return test,result + } + + let inline cons xs x = x::xs + + if not config.``parallel`` || + config.parallelWorkers = 1 || + List.forall (fun t -> t.sequenced=Synchronous) tests then + return! + List.map evalTestAsync tests + |> Async.foldSequentiallyWithCancel ct cons [] + else + let sequenced = + List.filter (fun t -> t.sequenced=Synchronous) tests + |> List.map evalTestAsync + + let parallel = + List.filter (fun t -> t.sequenced<>Synchronous) tests + |> Seq.groupBy (fun t -> t.sequenced) + |> Seq.collect(fun (group,tests) -> + match group with + | InParallel -> + Seq.map (evalTestAsync >> List.singleton) tests + | _ -> + Seq.map evalTestAsync tests + |> Seq.toList + |> Seq.singleton + ) + |> Seq.toList + |> List.sortBy (List.length >> (~-)) + |> List.map ( + function + | [test] -> Async.map List.singleton test + | l -> Async.foldSequentiallyWithCancel ct cons [] l + ) + + let! parallelResults = + let noWorkers = numberOfWorkers false config + Async.foldParallelWithCancel noWorkers ct (@) [] parallel + + if List.isEmpty sequenced |> not && List.isEmpty parallel |> not then + do! config.printer.info "Starting sequenced tests..." + + let! results = Async.foldSequentiallyWithCancel ct cons parallelResults sequenced + return List.sortBy (fun (t,_) -> + List.tryFindIndex (LanguagePrimitives.PhysicalEquality t) tests + ) results + } + + /// Evaluates tests. + let evalTests config test = + evalTestsWithCancel CancellationToken.None config test false + + let evalTestsSilent test = + let config = + { ExpectoConfig.defaultConfig with + parallel = false + verbosity = LogLevel.Fatal + printer = TestPrinters.silent + } + evalTests config test + + /// Runs tests, returns error code + let runEvalWithCancel (ct:CancellationToken) (config:ExpectoConfig) test = + async { + do! config.printer.beforeRun test + + let progressStarted = false + //if config.noSpinner then false + //else + // ProgressIndicator.text "Expecto Running... " + // ProgressIndicator.start() + + + let w = Stopwatch.StartNew() + let! results = evalTestsWithCancel ct config test progressStarted + w.Stop() + let testSummary : TestRunSummary = { + results = results + duration = w.Elapsed + maxMemory = 0L + memoryLimit = 0L + timedOut = [] + } + do! config.printer.summary config testSummary + + //if progressStarted then + // ProgressIndicator.stop () + + //ANSIOutputWriter.close () + + return testSummary.errorCode + } + + /// Runs tests, returns error code + let runEval config test = + runEvalWithCancel CancellationToken.None config test + + let runStressWithCancel (ct: CancellationToken) (config:ExpectoConfig) test = + async { + do! config.printer.beforeRun test + + //let progressStarted = + // if config.noSpinner then false + // else + // ProgressIndicator.text "Expecto Running... " + // ProgressIndicator.start() + + let tests = + Test.toTestCodeList test + |> List.filter (fun t -> Option.isNone t.shouldSkipEvaluation) + + let memoryLimit = + config.stressMemoryLimit * 1024.0 * 1024.0 |> int64 + + let evalTestAsync test = + execTestAsync ct config test |> Async.map (addFst test) + + let rand = Random() + + let randNext tests = + let next = List.length tests |> rand.Next + List.item next tests + + let totalTicks = + config.stress.Value.TotalSeconds * float Stopwatch.Frequency + |> int64 + + let finishTime = + lazy + totalTicks |> (+) (Stopwatch.GetTimestamp()) + + let asyncRun foldRunner (runningTests: ResizeArray<_>, + results, + maxMemory) = + let cancel = new CancellationTokenSource() + + let folder (runningTests: ResizeArray<_>, results: ResizeMap<_,_>, maxMemory) + (test, result : TestSummary) = + + runningTests.Remove test |> ignore + + results.[test] <- + match results.TryGetValue test with + | true, existing -> + existing + (result.result, result.meanDuration) + | false, _ -> + result + + let maxMemory = GC.GetTotalMemory false |> max maxMemory + + if maxMemory > memoryLimit then + cancel.Cancel() + + runningTests, results, maxMemory + + Async.Start(async { + let finishMilliseconds = + max (finishTime.Value - Stopwatch.GetTimestamp()) 0L + * 1000L / Stopwatch.Frequency + let timeout = + int finishMilliseconds + int config.stressTimeout.TotalMilliseconds + do! Async.Sleep timeout + cancel.Cancel() + }, cancel.Token) + + Seq.takeWhile (fun test -> + let now = Stopwatch.GetTimestamp() + + //if progressStarted then + // 100 - int((finishTime.Value - now) * 100L / totalTicks) + // |> Percent + // |> ProgressIndicator.update + + if now < finishTime.Value + && not ct.IsCancellationRequested then + runningTests.Add test + true + else + false ) + >> Seq.map evalTestAsync + >> foldRunner cancel.Token folder (runningTests,results,maxMemory) + + let initial = ResizeArray(), ResizeMap(), GC.GetTotalMemory false + + let w = Stopwatch.StartNew() + + let! runningTests,results,maxMemory = + if not config.``parallel`` || + config.parallelWorkers = 1 || + List.forall (fun t -> t.sequenced=Synchronous) tests then + + Seq.initInfinite (fun _ -> randNext tests) + |> Seq.append tests + |> asyncRun Async.foldSequentiallyWithCancel initial + else + List.filter (fun t -> t.sequenced=Synchronous) tests + |> asyncRun Async.foldSequentiallyWithCancel initial + |> Async.bind (fun (runningTests,results,maxMemory) -> + if maxMemory > memoryLimit || + Stopwatch.GetTimestamp() > finishTime.Value then + async.Return (runningTests,results,maxMemory) + else + let parallel = + List.filter (fun t -> t.sequenced<>Synchronous) tests + Seq.initInfinite (fun _ -> randNext parallel) + |> Seq.append parallel + |> Seq.filter (fun test -> + let s = test.sequenced + s=InParallel || + not(Seq.exists (fun t -> t.sequenced=s) runningTests) + ) + |> asyncRun + (Async.foldParallelWithCancel (numberOfWorkers true config)) + (runningTests,results,maxMemory) + ) + + w.Stop() + + let testSummary : TestRunSummary = { + results = + results + |> Seq.map (fun kv -> kv.Key,kv.Value) + |> List.ofSeq + duration = w.Elapsed + maxMemory = maxMemory + memoryLimit = memoryLimit + timedOut = List.ofSeq runningTests } + + do! config.printer.summary config testSummary + + //if progressStarted then + // ProgressIndicator.stop() + + //ANSIOutputWriter.close() + + return testSummary.errorCode + } + + let runStress config test = + runStressWithCancel CancellationToken.None config test + + let testFromMember (mi: MemberInfo) : Test option = + let inline unboxTest v = + if isNull v then + "Test is null. Assembly may not be initialized. Consider adding an [] or making it a library/classlib." + |> NullTestDiscoveryException |> raise + else unbox v + let getTestFromMemberInfo focusedState = + match box mi with + | :? FieldInfo as m -> + if m.FieldType = typeof then Some(focusedState, m.GetValue(null) |> unboxTest) + else None + | :? MethodInfo as m -> + if m.ReturnType = typeof then Some(focusedState, m.Invoke(null, null) |> unboxTest) + else None + | :? PropertyInfo as m -> + if m.PropertyType = typeof then Some(focusedState, m.GetValue(null, null) |> unboxTest) + else None + | _ -> None + mi.MatchTestsAttributes () + |> Option.map getTestFromMemberInfo + |> function + | Some (Some (focusedState, test)) -> Some (Test.translateFocusState focusedState test) + | _ -> None + + let listToTestListOption = + function + | [] -> None + | x -> Some (TestList (x, Normal)) + + let testFromType = + let asMembers x = Seq.map (fun m -> m :> MemberInfo) x + let bindingFlags = BindingFlags.Public ||| BindingFlags.Static + fun (t: Type) -> + [ t.GetTypeInfo().GetMethods bindingFlags |> asMembers + t.GetTypeInfo().GetProperties bindingFlags |> asMembers + t.GetTypeInfo().GetFields bindingFlags |> asMembers ] + |> Seq.collect id + |> Seq.choose testFromMember + |> Seq.toList + |> listToTestListOption + + // If the test function we've found doesn't seem to be in the test assembly, it's + // possible we're looking at an FsCheck 'testProperty' style check. In that case, + // the function of interest (i.e., the one in the test assembly, and for which we + // might be able to find corresponding source code) is referred to in a field + // of the function object. + let isFsharpFuncType t = + let baseType = + let rec findBase (t:Type) = + if t.GetTypeInfo().BaseType |> isNull || t.GetTypeInfo().BaseType = typeof then + t + else + findBase (t.GetTypeInfo().BaseType) + findBase t + baseType.GetTypeInfo().IsGenericType && baseType.GetTypeInfo().GetGenericTypeDefinition() = typedefof> + + let getFuncTypeToUse (testFunc:unit->unit) (asm:Assembly) = + let t = testFunc.GetType() + if t.GetTypeInfo().Assembly.FullName = asm.FullName then + t + else + let nestedFunc = + t.GetTypeInfo().GetFields() + |> Seq.tryFind (fun f -> isFsharpFuncType f.FieldType) + match nestedFunc with + | Some f -> f.GetValue(testFunc).GetType() + | None -> t + + let getMethodName asm testCode = + match testCode with + | Sync test -> + let t = getFuncTypeToUse test asm + let m = t.GetTypeInfo().GetMethods () |> Seq.find (fun m -> (m.Name = "Invoke") && (m.DeclaringType = t)) + (t.FullName, m.Name) + | SyncWithCancel _ -> + ("Unknown SyncWithCancel", "Unknown SyncWithCancel") + | Async _ | AsyncFsCheck _ -> + ("Unknown Async", "Unknown Async") + + // Ported from + // https://github.com/adamchester/expecto-adapter/blob/885fc9fff0/src/Expecto.VisualStudio.TestAdapter/SourceLocation.fs + let getSourceLocation (asm: Assembly) className methodName = + let lineNumberIndicatingHiddenLine = 0xfeefee + let getEcma335TypeName (clrTypeName:string) = clrTypeName.Replace("+", "/") + + let types = + let readerParams = new ReaderParameters( ReadSymbols = true ) + let moduleDefinition = ModuleDefinition.ReadModule(asm.Location, readerParams) + + seq { for t in moduleDefinition.GetTypes() -> (t.FullName, t) } + |> Map.ofSeq + + let getMethods typeName = + match types.TryFind (getEcma335TypeName typeName) with + | Some t -> Some (t.Methods) + | _ -> None + + let getFirstOrDefaultSequencePoint (m:MethodDefinition) = + m.Body.Instructions + |> Seq.tryPick (fun i -> + let sp = m.DebugInformation.GetSequencePoint i + if isNull sp |> not && sp.StartLine <> lineNumberIndicatingHiddenLine then + Some sp else None) + + match getMethods className with + | None -> SourceLocation.empty + | Some methods -> + let candidateSequencePoints = + methods + |> Seq.where (fun m -> m.Name = methodName) + |> Seq.choose getFirstOrDefaultSequencePoint + |> Seq.sortBy (fun sp -> sp.StartLine) + |> Seq.toList + match candidateSequencePoints with + | [] -> SourceLocation.empty + | xs -> {sourcePath = xs.Head.Document.Url ; lineNumber = xs.Head.StartLine} + + //val apply : f:(TestCode * FocusState * SourceLocation -> TestCode * FocusState * SourceLocation) -> _arg1:Test -> Test + let getLocation (asm:Assembly) code = + let typeName, methodName = getMethodName asm code + try + getSourceLocation asm typeName methodName + with :? IO.FileNotFoundException -> + SourceLocation.empty + + /// Scan filtered tests marked with TestsAttribute from an assembly + let testFromAssemblyWithFilter typeFilter (a: Assembly) = + a.GetExportedTypes() + |> Seq.filter typeFilter + |> Seq.choose testFromType + |> Seq.toList + |> listToTestListOption + + /// Scan tests marked with TestsAttribute from an assembly + let testFromAssembly = testFromAssemblyWithFilter (fun _ -> true) + + /// Scan tests marked with TestsAttribute from entry assembly + let testFromThisAssembly () = testFromAssembly (Assembly.GetEntryAssembly()) + + /// When the failOnFocusedTests switch is activated this function that no + /// focused tests exist. + /// + /// Returns true if the check passes, otherwise false. + let passesFocusTestCheck (config:ExpectoConfig) tests = + let isFocused : FlatTest -> _ = function t when t.state = Focused -> true | _ -> false + let focused = Test.toTestCodeList tests |> List.filter isFocused + if focused.Length = 0 then true + else + if config.verbosity <> LogLevel.Fatal then + logger.logWithAck LogLevel.Error ( + eventX "It was requested that no focused tests exist, but yet there are {count} focused tests found." + >> setField "count" focused.Length) + |> Async.StartImmediate + //ANSIOutputWriter.flush () + false + +[] +module Tests = + open Impl + open Helpers + open Expecto.Logging + open FSharp.Control.Tasks.CopiedDoNotReference.V2 + + let mutable private afterRunTestsList = [] + let private afterRunTestsListLock = obj() + /// Add a function that will be called after all testing has finished. + let afterRunTests f = + lock afterRunTestsListLock (fun () -> + afterRunTestsList <- f :: afterRunTestsList + ) + let internal afterRunTestsInvoke() = + lock afterRunTestsListLock (fun () -> + let failures = + List.rev afterRunTestsList + |> List.choose (fun f -> + try + f() + None + with e -> Some e + ) + match failures with + | [] -> () + | l -> List.toArray l |> AggregateException |> raise + ) + Console.CancelKeyPress |> Event.add (fun _ -> afterRunTestsInvoke()) + + /// Expecto atomic printfn shadow function + let printfn format = + Printf.ksprintf (fun s -> + Console.Write(s.PadRight 40 + "\n") + ) format + + /// Expecto atomic eprintfn shadow function + let eprintfn format = + Printf.ksprintf (fun s -> + Console.Error.Write(s.PadRight 40 + "\n") + ) format + + /// The full name of the currently running test + let testName() = TestNameHolder.Name + + /// Fail this test + let inline failtest msg = raise <| AssertException msg + /// Fail this test + let inline failtestf fmt = Printf.ksprintf (AssertException >> raise) fmt + /// Fail this test + let inline failtestNoStack msg = raise <| FailedException msg + /// Fail this test + let inline failtestNoStackf fmt = Printf.ksprintf (FailedException >> raise) fmt + + /// Skip this test + let inline skiptest msg = raise <| IgnoreException msg + /// Skip this test + let inline skiptestf fmt = Printf.ksprintf (IgnoreException >> raise) fmt + + /// Builds a list/group of tests that will be ignored by Expecto if exists + /// focused tests and none of the parents is focused + let inline testList name tests = TestLabel(name, TestList (tests, Normal), Normal) + + /// Builds a list/group of tests that will make Expecto to ignore other unfocused tests + let inline ftestList name tests = TestLabel(name, TestList (tests, Focused), Focused) + /// Builds a list/group of tests that will be ignored by Expecto + let inline ptestList name tests = TestLabel(name, TestList (tests, Pending), Pending) + + /// Labels the passed test with a text segment. In Expecto, tests are slash-separated (`/`), so this wraps the passed + /// tests in such a label. Useful when you don't want lots of indentation in your tests (the code would become hard to + /// modify and read, due to all the whitespace), and you want to do `testList "..." [ ] |> testLabel "api"`. + let inline testLabel name test = TestLabel(name, test, Normal) + + /// Builds a test case that will be ignored by Expecto if exists focused + /// tests and none of the parents is focused + let inline testCase name test = TestLabel(name, TestCase (Sync test,Normal), Normal) + /// Builds a test case with a CancellationToken that can be check for cancel + let inline testCaseWithCancel name test = TestLabel(name, TestCase (SyncWithCancel test,Normal), Normal) + /// Builds an async test case + let inline testCaseAsync name test = TestLabel(name, TestCase (Async test,Normal), Normal) + /// Builds a test case that will make Expecto to ignore other unfocused tests + let inline ftestCase name test = TestLabel(name, TestCase (Sync test, Focused), Focused) + /// Builds a test case with cancel that will make Expecto to ignore other unfocused tests + let inline ftestCaseWithCancel name test = TestLabel(name, TestCase (SyncWithCancel test, Focused), Focused) + /// Builds an async test case that will make Expecto to ignore other unfocused tests + let inline ftestCaseAsync name test = TestLabel(name, TestCase (Async test, Focused), Focused) + /// Builds a test case that will be ignored by Expecto + let inline ptestCase name test = TestLabel(name, TestCase (Sync test, Pending), Pending) + /// Builds a test case with cancel that will be ignored by Expecto + let inline ptestCaseWithCancel name test = TestLabel(name, TestCase (SyncWithCancel test, Pending), Pending) + /// Builds an async test case that will be ignored by Expecto + let inline ptestCaseAsync name test = TestLabel(name, TestCase (Async test, Pending), Pending) + /// Test case or list needs to run sequenced. Use for any benchmark code or + /// for tests using `Expect.isFasterThan` + let inline testSequenced test = Test.Sequenced (Synchronous,test) + /// Test case or list needs to run sequenced with other tests in this group. + let inline testSequencedGroup name test = Test.Sequenced (SynchronousGroup name,test) + + /// Applies a function to a list of values to build test cases + let inline testFixture setup = + Seq.map (fun (name, partialTest) -> + testCase name (setup partialTest)) + + /// Applies a value to a list of partial tests + let inline testParam param = + Seq.map (fun (name, partialTest) -> + testCase name (partialTest param)) + + /// Test case computation expression builder + type TestCaseBuilder(name, focusState) = + member __.TryFinally(f, compensation) = + try + f() + finally + compensation() + member __.TryWith(f, catchHandler) = + try + f() + with e -> catchHandler e + member __.Using(disposable: #IDisposable, f) = + try + f disposable + finally + match box disposable with + | :? IDisposable as d when not (isNull d) -> d.Dispose() + | _ -> () + member __.For(sequence, f) = + for i in sequence do f i + member __.Combine(f1, f2) = f2(); f1 + member __.Zero() = () + member __.Delay f = f + member __.Run f = + match focusState with + | Normal -> testCase name f + | Focused -> ftestCase name f + | Pending -> ptestCase name f + + /// Builds a test case + let inline test name = + TestCaseBuilder (name, Normal) + /// Builds a test case that will make Expecto to ignore other unfocused tests + let inline ftest name = + TestCaseBuilder (name, Focused) + /// Builds a test case that will be ignored by Expecto + let inline ptest name = + TestCaseBuilder (name, Pending) + + /// Async test case computation expression builder + type TestAsyncBuilder(name, focusState) = + member __.Zero() = async.Zero() + member __.Delay(f) = async.Delay(f) + member __.Return(x) = async.Return(x) + member __.ReturnFrom(x) = async.ReturnFrom(x) + member __.Bind(p1, p2) = async.Bind(p1, p2) + member __.Using(g, p) = async.Using(g, p) + member __.While(gd, prog) = async.While(gd, prog) + member __.For(e, prog) = async.For(e, prog) + member __.Combine(p1, p2) = async.Combine(p1, p2) + member __.TryFinally(p, cf) = async.TryFinally(p, cf) + member __.TryWith(p, cf) = async.TryWith(p, cf) + member __.Run f = + match focusState with + | Normal -> testCaseAsync name f + | Focused -> ftestCaseAsync name f + | Pending -> ptestCaseAsync name f + + /// Builds an async test case + let inline testAsync name = + TestAsyncBuilder (name, Normal) + /// Builds an async test case that will make Expecto to ignore other unfocused tests + let inline ftestAsync name = + TestAsyncBuilder (name, Focused) + /// Builds an async test case that will be ignored by Expecto + let inline ptestAsync name = + TestAsyncBuilder (name, Pending) + + type TestTaskBuilder(name, focusState) = + member __.Zero() = task.Zero() + member __.Delay(f) = task.Delay(f) + member __.Return(x) = task.Return(x) + member __.ReturnFrom(x) = task.ReturnFrom(x) + member __.Bind(p1:Task<'a>, p2:'a->_) = task.Bind(p1, p2) + member __.Bind(p1:Task, p2:unit->_) = task.Bind(p1, p2) + member __.Using(g, p) = task.Using(g, p) + member __.While(gd, prog) = task.While(gd, prog) + member __.For(e, prog) = task.For(e, prog) + member __.Combine(p1, p2) = task.Combine(p1, p2) + member __.TryFinally(p, cf) = task.TryFinally(p, cf) + member __.TryWith(p, cf) = task.TryWith(p, cf) + member __.Run f = + let a = async { + do! task.Run f |> Async.AwaitTask + } + match focusState with + | Normal -> testCaseAsync name a + | Focused -> ftestCaseAsync name a + | Pending -> ptestCaseAsync name a + + /// Builds a task test case + let inline testTask name = + TestTaskBuilder (name, Normal) + /// Builds a task test case that will make Expecto to ignore other unfocused tests + let inline ftestTask name = + TestTaskBuilder (name, Focused) + /// Builds a task test case that will be ignored by Expecto + let inline ptestTask name = + TestTaskBuilder (name, Pending) + + /// The default configuration for Expecto. + let defaultConfig = ExpectoConfig.defaultConfig + + module Args = + open FSharp.Core + + type Parser<'a> = (string[] * int * int) -> Result<'a,string> * int + + let parseOptions (options:(string * string * Parser<_>) list) (strings:string[]) = + let rec updateUnknown unknown last length = + if length = 0 then unknown + else updateUnknown (strings.[last]::unknown) (last-1) (length-1) + let rec collect isHelp unknown args paramCount i = + if i>=0 then + let currentArg = strings.[i] + if currentArg = "--help" || currentArg = "-h" || currentArg = "-?" || currentArg = "/?" then + collect true (updateUnknown unknown (i+paramCount) paramCount) args 0 (i-1) + else + match List.tryFind (fst3 >> (=) currentArg) options with + | Some (option, _, parser) -> + let arg, unknownCount = parser (strings, i+1, paramCount) + collect isHelp + (updateUnknown unknown (i+paramCount) unknownCount) + (Result.mapError (fun i -> option + " " + i) arg::args) 0 (i-1) + | None -> collect isHelp unknown args (paramCount+1) (i-1) + else + let unknown = + match updateUnknown unknown (paramCount-1) paramCount with + | [] -> None + | l -> String.Join(" ","unknown options:" :: l) |> Some + match isHelp, Result.sequence args, unknown with + | false, Ok os, None -> Ok(List.rev os) + | true, Ok _, None -> Error [] + | _, Ok _, Some u -> Error [u] + | _, r, None -> r + | _, Error es, Some u -> List.rev (u::es) |> Error + collect false [] [] 0 (strings.Length-1) + + let [] depricated = "Deprecated" + let deprecated = "Deprecated" + + let usage commandName (options: (string * string * Parser<_>) list) = + let sb = Text.StringBuilder("Usage: ") + let add (text:string) = sb.Append(text) |> ignore + add commandName + add " [options]\n\nOptions:\n" + let maxLength = + options |> Seq.map (fun (s,_,_) -> s.Length) |> Seq.max + ["--help","Show this help message."] + |> Seq.append (Seq.map (fun (s,d,_) -> s,d) options) + |> Seq.where (snd >> (<>)deprecated) + |> Seq.iter (fun (s,d) -> + add " " + add (s.PadRight maxLength) + add " " + add d + add "\n" + ) + sb.ToString() + + let none case : Parser<_> = + fun (_,_,l) -> Ok case, l + + let string case : Parser<_> = + fun (ss,i,l) -> + if l>0 then Ok(case ss.[i]), l-1 + else Error "requires a parameter", 0 + + let list (parser:_->Parser<_>) case : Parser<_> = + fun (ss,i,l) -> + [i..i+l-1] + |> Result.traverse (fun j -> parser id (ss,j,1) |> fst) + |> Result.map (fun l -> case(List.rev l)) + |> Result.mapError (fun i -> String.Join(", ", i)) + , 0 + + let inline private parseWith tryParseFn case: Parser<'a> = + fun (args, i, l) -> + if l = 0 then Error "requires a parameter", 0 + else + match tryParseFn args.[i] with + | Some i -> Ok(case i), l-1 + | None -> Error("Cannot parse parameter '" + args.[i] + "'"), l-1 + + + let inline parse case: Parser<'a> = parseWith tryParse case + let inline number case: Parser<'a> = parseWith tryParseNumber case + + + [] + type SummaryHandler = + | SummaryHandler of (TestRunSummary -> unit) + + /// The CLI arguments are the parameters that are possible to send to Expecto + /// and change the runner's behaviour. + type CLIArguments = + /// Don't run the tests in parallel. + | Sequenced + /// Run all tests in parallel (default). + | Parallel + /// Set the number of parallel workers (defaults to the number of logical processors). + | Parallel_Workers of int + /// Set FsCheck maximum number of tests (default: 100). + | FsCheck_Max_Tests of int + /// Set FsCheck start size (default: 1). + | FsCheck_Start_Size of int + /// Set FsCheck end size (default: 100 for testing and 10,000 for stress testing). + | FsCheck_End_Size of int + /// Run the tests randomly for the given number of minutes. + | Stress of float + /// Set the time to wait in minutes after the stress test before reporting as a deadlock (default 5 mins). + | Stress_Timeout of float + /// Set the Stress test memory limit in MB to stop the test and report as a memory leak (default 100 MB). + | Stress_Memory_Limit of float + /// This will make the test runner fail if focused tests exist. + | Fail_On_Focused_Tests + /// Extra verbose printing. Useful to combine with --sequenced. + | Debug + /// Set the process name to log under (default: "Expecto"). + | Log_Name of name:string + /// Filters the list of tests by a hierarchy that's slash (/) separated. + | Filter of hiera:string + /// Filters the list of test lists by a given substring. + | Filter_Test_List of substring:string + /// Filters the list of test cases by a given substring. + | Filter_Test_Case of substring:string + /// Runs only provided list of tests. + | Run of tests:string list + /// Don't run tests, but prints out list of tests instead. + | List_Tests + /// Print out a summary after all tests are finished. + | Summary + /// Print out a summary after all tests are finished including their source code location. + | Summary_Location + /// Print out version information. + | Version + /// Depricated + | My_Spirit_Is_Weak + /// Allow duplicate test names. + | Allow_Duplicate_Names + /// Disable the spinner progress update. + | No_Spinner + // Set the level of colours to use. Can be 0, 8 (default) or 256. + | Colours of int + /// Adds a test printer. + | Printer of TestPrinters + /// Sets the verbosity level. + | Verbosity of LogLevel + /// Append a summary handler. + | Append_Summary_Handler of SummaryHandler + + let options = [ + "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced + "--parallel", "Run all tests in parallel (default).", Args.none Parallel + "--parallel-workers", "Set the number of parallel workers (defaults to the number of logical processors).", Args.number Parallel_Workers + "--stress", "Run the tests randomly for the given number of minutes.", Args.number Stress + "--stress-timeout", "Set the time to wait in minutes after the stress test before reporting as a deadlock (default 5 mins).", Args.number Stress_Timeout + "--stress-memory-limit", "Set the Stress test memory limit in MB to stop the test and report as a memory leak (default 100 MB).", Args.number Stress_Memory_Limit + "--fail-on-focused-tests", "This will make the test runner fail if focused tests exist.", Args.none Fail_On_Focused_Tests + "--debug", "Extra verbose printing. Useful to combine with --sequenced.", Args.none Debug + "--log-name", "Set the process name to log under (default: \"Expecto\").", Args.string Log_Name + "--filter", "Filters the list of tests by a hierarchy that's slash (/) separated.", Args.string Filter + "--filter-test-list", "Filters the list of test lists by a given substring.", Args.string Filter_Test_List + "--filter-test-case", "Filters the list of test cases by a given substring.", Args.string Filter_Test_Case + "--run", "Runs only provided list of tests.", Args.list Args.string Run + "--list-tests", "Don't run tests, but prints out list of tests instead.", Args.none List_Tests + "--summary", "Print out a summary after all tests are finished.", Args.none Summary + "--version", "Print out version information.", Args.none Version + "--summary-location", "Print out a summary after all tests are finished including their source code location.", Args.none Summary_Location + "--fscheck-max-tests", "Set FsCheck maximum number of tests (default: 100).", Args.number FsCheck_Max_Tests + "--fscheck-start-size", "Set FsCheck start size (default: 1).", Args.number FsCheck_Start_Size + "--fscheck-end-size", "Set FsCheck end size (default: 100 for testing and 10,000 for stress testing).", Args.number FsCheck_End_Size + "--my-spirit-is-weak", Args.deprecated, Args.none My_Spirit_Is_Weak + "--allow-duplicate-names", "Allow duplicate test names.", Args.none Allow_Duplicate_Names + "--colours", "Set the level of colours to use. Can be 0, 8 (default) or 256.", Args.number Colours + "--no-spinner", "Disable the spinner progress update.", Args.none No_Spinner + ] + + type FillFromArgsResult = + | ArgsRun of ExpectoConfig + | ArgsList of ExpectoConfig + | ArgsVersion of ExpectoConfig + | ArgsUsage of usage:string * errors:string list + + let private getTestList (s:string) = + let all = s.Split('/') + match all with + | [||] | [|_|] -> [||] + | xs -> xs.[0..all.Length-2] + + let private getTestCase (s:string) = + let i = s.LastIndexOf('/') + if i= -1 then s else s.Substring(i+1) + + let private foldCLIArgumentToConfig = function + | Sequenced -> fun o -> { o with ExpectoConfig.parallel = false } + | Parallel -> fun o -> { o with parallel = true } + | Parallel_Workers n -> fun o -> { o with parallelWorkers = n } + | Stress n -> fun o -> {o with + stress = TimeSpan.FromMinutes n |> Some + printer = TestPrinters.stressPrinter } + | Stress_Timeout n -> fun o -> { o with stressTimeout = TimeSpan.FromMinutes n } + | Stress_Memory_Limit n -> fun o -> { o with stressMemoryLimit = n } + | Fail_On_Focused_Tests -> fun o -> { o with failOnFocusedTests = true } + | Debug -> fun o -> { o with verbosity = LogLevel.Debug } + | Log_Name name -> fun o -> { o with logName = Some name } + | Filter hiera -> fun o -> {o with filter = Test.filter (fun s -> s.StartsWith hiera )} + | Filter_Test_List name -> fun o -> {o with filter = Test.filter (fun s -> s |> getTestList |> Array.exists(fun s -> s.Contains name )) } + | Filter_Test_Case name -> fun o -> {o with filter = Test.filter (fun s -> s |> getTestCase |> fun s -> s.Contains name )} + | Run tests -> fun o -> {o with filter = Test.filter (fun s -> tests |> List.exists ((=) s) )} + | List_Tests -> id + | Summary -> fun o -> {o with printer = TestPrinters.summaryPrinter o.printer} + | Version -> id + | Summary_Location -> fun o -> {o with printer = TestPrinters.summaryWithLocationPrinter o.printer} + | FsCheck_Max_Tests n -> fun o -> {o with fsCheckMaxTests = n } + | FsCheck_Start_Size n -> fun o -> {o with fsCheckStartSize = n } + | FsCheck_End_Size n -> fun o -> {o with fsCheckEndSize = Some n } + | My_Spirit_Is_Weak -> id + | Allow_Duplicate_Names -> fun o -> { o with allowDuplicateNames = true } + | No_Spinner -> fun o -> { o with noSpinner = true } + | Colours i -> fun o -> { o with colour = + if i >= 256 then Colour256 + elif i >= 8 then Colour8 + else Colour0 + } + | Printer p -> fun o -> { o with printer = p } + | Verbosity l -> fun o -> { o with verbosity = l } + | Append_Summary_Handler (SummaryHandler h) -> fun o -> o.appendSummaryHandler h + + [] + module ExpectoConfig = + + let expectoVersion = "8.13.1" + + /// Parses command-line arguments into a config. This allows you to + /// override the config from the command line, rather than having + /// to go into the compiled code to change how they are being run. + /// Also checks if tests should be run or only listed + let fillFromArgs baseConfig args = + match Args.parseOptions options args with + | Ok cliArguments -> + let config = + Seq.fold (fun s a -> foldCLIArgumentToConfig a s) baseConfig cliArguments + if List.contains List_Tests cliArguments then + ArgsList config + elif List.contains Version cliArguments then + ArgsVersion config + else + ArgsRun config + | Result.Error errors -> + let commandName = + Environment.GetCommandLineArgs().[0] + |> IO.Path.GetFileName + |> fun f -> if f.EndsWith(".dll") then "dotnet " + f else f + ArgsUsage (Args.usage commandName options, errors) + + /// Prints out names of all tests for given test suite. + let listTests test = + Test.toTestCodeList test + |> Seq.iter (fun t -> printfn "%s" t.name) + + /// Prints out names of all tests for given test suite. + let duplicatedNames test = + Test.toTestCodeList test + |> Seq.toList + |> List.groupBy (fun t -> t.name) + |> List.choose (function + | _, x :: _ :: _ -> Some x.name + | _ -> None + ) + /// Runs tests with the supplied config. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsWithCancel (ct:CancellationToken) config (tests:Test) = + printfn "Running Expecto with FakeExpecto!!!" + //ANSIOutputWriter.setColourLevel config.colour + Global.initialiseIfDefault + { Global.defaultConfig with + getLogger = fun name -> + LiterateConsoleTarget( + name, config.verbosity, + consoleSemaphore = obj()) :> Logger + } + + let config = { config with + printer = TestPrinters.silent } + + config.logName |> Option.iter setLogName + if config.failOnFocusedTests && passesFocusTestCheck config tests |> not then + 1 + else + let tests = config.filter tests + let duplicates = lazy duplicatedNames tests + if config.allowDuplicateNames || List.isEmpty duplicates.Value then + let retCode = + match config.stress with + | None -> runEvalWithCancel ct config tests |> Async.RunSynchronously + | Some _ -> runStressWithCancel ct config tests |> Async.RunSynchronously + afterRunTestsInvoke() + retCode + else + sprintf "Found duplicated test names, these names are: %A" duplicates.Value + |> config.printer.info + |> Async.RunSynchronously + //ANSIOutputWriter.close() + 1 + /// Runs tests with the supplied config. + /// Returns 0 if all tests passed, otherwise 1 + let runTests config tests = + runTestsWithCancel CancellationToken.None config tests + + /// Runs all given tests with the supplied command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsWithArgsAndCancel (ct:CancellationToken) config args tests = + match ExpectoConfig.fillFromArgs config args with + | ArgsUsage (usage, errors) -> + if not (List.isEmpty errors) then + printfn "ERROR: %s\n" (String.Join(" ",errors)) + printfn "EXPECTO! v%s\n\n%s" ExpectoConfig.expectoVersion usage + if List.isEmpty errors then 0 else 1 + | ArgsList config -> + config.filter tests + |> listTests + 0 + | ArgsRun config -> + runTestsWithCancel ct config tests + | ArgsVersion config -> + printfn "EXPECTO! v%s\n" ExpectoConfig.expectoVersion + runTestsWithCancel ct config tests + + /// Runs all given tests with the supplied typed command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsWithCLIArgsAndCancel (ct:CancellationToken) cliArgs args tests = + let config = + Seq.fold (fun s a -> foldCLIArgumentToConfig a s) + ExpectoConfig.defaultConfig cliArgs + runTestsWithArgsAndCancel ct config args tests + + /// Runs all given tests with the supplied command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsWithArgs config args tests = + runTestsWithArgsAndCancel CancellationToken.None config args tests + + /// Runs all given tests with the supplied typed command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsWithCLIArgs cliArgs args tests = + runTestsWithCLIArgsAndCancel CancellationToken.None cliArgs args tests + + /// Runs tests in this assembly with the supplied command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsInAssemblyWithCancel (ct:CancellationToken) config args = + let config = { config with locate = getLocation (Assembly.GetEntryAssembly()) } + testFromThisAssembly () + |> Option.orDefault (TestList ([], Normal)) + |> runTestsWithArgsAndCancel ct config args + + /// Runs tests in this assembly with the supplied command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsInAssemblyWithCLIArgsAndCancel (ct:CancellationToken) cliArgs args = + let config = { ExpectoConfig.defaultConfig + with locate = getLocation (Assembly.GetEntryAssembly()) } + let config = Seq.fold (fun s a -> foldCLIArgumentToConfig a s) config cliArgs + let tests = testFromThisAssembly() |> Option.orDefault (TestList ([], Normal)) + runTestsWithArgsAndCancel ct config args tests + + /// Runs tests in this assembly with the supplied command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsInAssembly config args = + runTestsInAssemblyWithCancel CancellationToken.None config args + + /// Runs tests in this assembly with the supplied command-line options. + /// Returns 0 if all tests passed, otherwise 1 + let runTestsInAssemblyWithCLIArgs cliArgs args = + runTestsInAssemblyWithCLIArgsAndCancel CancellationToken.None cliArgs args From f82ca6cc2acc8668a28b1777619e1cba4131fc18 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Tue, 17 Dec 2019 00:59:44 +0100 Subject: [PATCH 07/14] Update Fake.ExpectoSupport.fsproj --- src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj b/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj index d70cd1e599c..ce85a38e122 100644 --- a/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj +++ b/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj @@ -7,6 +7,7 @@ + From e14dbfc5dba1bef9d83e4d36dc3ade0bef1d421a Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Tue, 17 Dec 2019 09:43:41 +0100 Subject: [PATCH 08/14] remove silent printer --- src/test/Fake.ExpectoSupport/FakeExpecto.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/Fake.ExpectoSupport/FakeExpecto.fs b/src/test/Fake.ExpectoSupport/FakeExpecto.fs index d1410e71ef2..44a4f0bae8f 100644 --- a/src/test/Fake.ExpectoSupport/FakeExpecto.fs +++ b/src/test/Fake.ExpectoSupport/FakeExpecto.fs @@ -1304,8 +1304,8 @@ module Tests = consoleSemaphore = obj()) :> Logger } - let config = { config with - printer = TestPrinters.silent } + //let config = { config with + // printer = TestPrinters.silent } config.logName |> Option.iter setLogName if config.failOnFocusedTests && passesFocusTestCheck config tests |> not then From 532bb65826c8a80ecc3ab7a4a3eff6bd4e29f073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Hromad=C3=ADk?= Date: Tue, 17 Dec 2019 14:57:59 +0100 Subject: [PATCH 09/14] fixes #2440 - incorrect indentation in the right menu when the difference between the last heading of the previous section and the first heading of the current section is more than 1 --- help/content/assets/js/bulma.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/help/content/assets/js/bulma.js b/help/content/assets/js/bulma.js index 58bc150f0dd..cc36f0df0cd 100644 --- a/help/content/assets/js/bulma.js +++ b/help/content/assets/js/bulma.js @@ -36,9 +36,11 @@ function generateTableOfContent() { menuHtml += `
  • ${heading.innerText}`; levels++; } else if (currentHeadingCount < headingNumber) { - // decrease level - menuHtml += `
  • ${heading.innerText}`; - levels--; + // decrease level, could go down more than one level (say a previous section ends with h4 and a new section starts with h2) + let headingNumDif = headingNumber - currentHeadingCount; + for (let d = 0; d < headingNumDif; d++) { menuHtml += "
    • " } + menuHtml += `
  • ${heading.innerText}`; + levels-=headingNumDif; } else if (currentHeadingCount == headingNumber) { // same level menuHtml += `
  • ${heading.innerText}`; From 58c472295d4f8ad7c33755dcfaa4ac57fa0b6da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Hromad=C3=ADk?= Date: Tue, 17 Dec 2019 15:53:53 +0100 Subject: [PATCH 10/14] decrease footer z-index so that main menu is not hidden behind it --- help/content/assets/css/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/help/content/assets/css/style.css b/help/content/assets/css/style.css index 687da685c8d..4dd52b89d2c 100644 --- a/help/content/assets/css/style.css +++ b/help/content/assets/css/style.css @@ -27,7 +27,7 @@ a i.fa { footer { position: relative; - z-index: 99; + z-index: 98; } footer ul li { From 288888c41544dad0aa1ad53c62fc0fba2536df37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Hromad=C3=ADk?= Date: Tue, 17 Dec 2019 16:09:21 +0100 Subject: [PATCH 11/14] fixed "Visual Studio for Windows" download link --- help/markdown/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/help/markdown/contributing.md b/help/markdown/contributing.md index fc022385045..05559671a23 100644 --- a/help/markdown/contributing.md +++ b/help/markdown/contributing.md @@ -34,7 +34,7 @@ For FAKE development, [Visual Studio Code](https://code.visualstudio.com/Downloa - JetBrains Rider [[download](https://www.jetbrains.com/rider/download)] - Visual Studio for Mac [[download](https://visualstudio.microsoft.com/vs/mac/)] -- Visual Studio for Windows [[download](https://www.jetbrains.com/rider/download)] +- Visual Studio for Windows [[download](https://visualstudio.microsoft.com/vs/)] ### Install FAKE From 46411ce713ea9224288bbf072c7c63f7fd41db0f Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Tue, 17 Dec 2019 19:56:37 +0100 Subject: [PATCH 12/14] test proper workaround for https://github.com/haf/expecto/issues/367 --- build.fsx | 2 +- .../Fake.Core.CommandLine.UnitTests/Main.fs | 3 +- src/test/Fake.Core.IntegrationTests/Main.fs | 3 +- src/test/Fake.Core.UnitTests/Main.fs | 3 +- .../Fake.DotNet.Cli.IntegrationTests/Main.fs | 3 +- .../Fake.ExpectoSupport/ExpectoHelpers.fs | 113 +++++++++++++++--- .../Fake.ExpectoSupport.fsproj | 3 +- 7 files changed, 105 insertions(+), 25 deletions(-) diff --git a/build.fsx b/build.fsx index 7e48a6403b7..612cb66a22a 100644 --- a/build.fsx +++ b/build.fsx @@ -645,7 +645,7 @@ Target.create "HostDocs" (fun _ -> let runExpecto workDir dllPath resultsXml = let processResult = - DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary --debug --sequenced --no-spinner" + DotNet.exec (dtntWorkDir workDir) (sprintf "%s" dllPath) "--summary" if processResult.ExitCode <> 0 then failwithf "Tests in %s failed." (Path.GetFileName dllPath) Trace.publish (ImportData.Nunit NunitDataVersion.Nunit) (workDir resultsXml) diff --git a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs index f472c2617de..e8c5bac95b3 100644 --- a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs +++ b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs @@ -9,5 +9,6 @@ let main argv = let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) + |> ExpectoHelpers.setFakePrinter - FakeExpecto.Tests.runTestsInAssembly { config with parallel = false } argv + Expecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.Core.IntegrationTests/Main.fs b/src/test/Fake.Core.IntegrationTests/Main.fs index b7bea83a086..39a0b2f99e4 100644 --- a/src/test/Fake.Core.IntegrationTests/Main.fs +++ b/src/test/Fake.Core.IntegrationTests/Main.fs @@ -9,4 +9,5 @@ let main argv = let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - FakeExpecto.Tests.runTestsInAssembly { config with parallel = false } argv + |> ExpectoHelpers.setFakePrinter + Expecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.Core.UnitTests/Main.fs b/src/test/Fake.Core.UnitTests/Main.fs index 4051a509135..0699341620e 100644 --- a/src/test/Fake.Core.UnitTests/Main.fs +++ b/src/test/Fake.Core.UnitTests/Main.fs @@ -10,4 +10,5 @@ let main argv = let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - FakeExpecto.Tests.runTestsInAssembly config argv + |> ExpectoHelpers.setFakePrinter + Expecto.Tests.runTestsInAssembly config argv diff --git a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs index b357e22012f..4612f3cc2d4 100644 --- a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs +++ b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs @@ -9,4 +9,5 @@ let main argv = let config = defaultConfig.appendSummaryHandler writeResults |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) - FakeExpecto.Tests.runTestsInAssembly { config with parallel = false } argv + |> ExpectoHelpers.setFakePrinter + Expecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs index 8307c0a5273..7ee53e9e165 100644 --- a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs +++ b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs @@ -1,47 +1,124 @@ namespace Fake.ExpectoSupport -open Expecto + open System open System.Threading open System.Threading.Tasks module ExpectoHelpers = - let addFilter f (config:Impl.ExpectoConfig) = + + let inline internal commaString (i:int) = i.ToString("#,##0") + // because of https://github.com/haf/expecto/issues/367 + let fakeDefaultPrinter : Expecto.Impl.TestPrinters = + { beforeRun = fun _tests -> + printfn "EXPECTO? Running tests..." + async.Zero() + + beforeEach = fun n -> + printfn "EXPECTO? %s starting..." n + async.Zero() + + info = fun s -> + printfn "EXPECTO? %s" s + async.Zero() + + passed = fun n d -> + printfn "EXPECTO? %s passed in %O." n d + async.Zero() + + ignored = fun n m -> + printfn "EXPECTO? %s was ignored. %s" n m + async.Zero() + + failed = fun n m d -> + printfn "EXPECTO? %s failed in %O. %s" n d m + async.Zero() + + exn = fun n e d -> + printfn "EXPECTO? %s errored in %O: %O" n d e + async.Zero() + + summary = fun _config summary -> + let spirit = + if summary.successful then "Success!" else String.Empty + let commonAncestor = + let rec loop ancestor (descendants : string list) = + match descendants with + | [] -> ancestor + | hd::tl when hd.StartsWith(ancestor)-> + loop ancestor tl + | _ -> + if ancestor.Contains("/") then + loop (ancestor.Substring(0, ancestor.LastIndexOf "/")) descendants + else + "miscellaneous" + + let parentNames = + summary.results + |> List.map (fun (flatTest, _) -> + if flatTest.name.Contains("/") then + flatTest.name.Substring(0, flatTest.name.LastIndexOf "/") + else + flatTest.name ) + + match parentNames with + | [x] -> x + | hd::tl -> + loop hd tl + | _ -> "miscellaneous" //we can't get here + printfn "EXPECTO! %s tests run in %O for %s – %s passed, %s ignored, %s failed, %s errored. %s" + (summary.results |> List.sumBy (fun (_,r) -> if r.result.isIgnored then 0 else r.count) |> commaString) + summary.duration + commonAncestor + (summary.passed |> List.sumBy (fun (_,r) -> r.count) |> commaString) + (summary.ignored |> List.sumBy (fun (_,r) -> r.count) |> commaString) + (summary.failed |> List.sumBy (fun (_,r) -> r.count) |> commaString) + (summary.errored |> List.sumBy (fun (_,r) -> r.count) |> commaString) + spirit + async.Zero() + } + + let setPrinter printer (config:Expecto.Impl.ExpectoConfig) = + { config with printer = printer } + let setFakePrinter (config:Expecto.Impl.ExpectoConfig) = + setPrinter fakeDefaultPrinter config + + let addFilter f (config:Expecto.Impl.ExpectoConfig) = { config with filter = fun test -> let filteredTests = config.filter test f filteredTests } - let withTimeout (timeout:TimeSpan) (labelPath:string) (test: TestCode) : TestCode = + let withTimeout (timeout:TimeSpan) (labelPath:string) (test: Expecto.TestCode) : Expecto.TestCode = let timeoutAsync testAsync = async { let t = Async.StartAsTask(testAsync) let delay = Task.Delay(timeout) let! result = Task.WhenAny(t, delay) |> Async.AwaitTask if result = delay then - Tests.failtestf "Test '%s' timed out" labelPath + Expecto.Tests.failtestf "Test '%s' timed out" labelPath } match test with - | Sync test -> async { test() } |> timeoutAsync |> Async - | SyncWithCancel test -> - SyncWithCancel (fun ct -> + | Expecto.Sync test -> async { test() } |> timeoutAsync |> Expecto.Async + | Expecto.SyncWithCancel test -> + Expecto.SyncWithCancel (fun ct -> Async.StartImmediate(async { test ct } |> timeoutAsync) ) - | Async test -> timeoutAsync test |> Async - | AsyncFsCheck (testConfig, stressConfig, test) -> - AsyncFsCheck (testConfig, stressConfig, test >> timeoutAsync) + | Expecto.Async test -> timeoutAsync test |> Expecto.Async + | Expecto.AsyncFsCheck (testConfig, stressConfig, test) -> + Expecto.AsyncFsCheck (testConfig, stressConfig, test >> timeoutAsync) let mapTest f test = let rec recMapping labelPath test = match test with - | TestCase (code:TestCode, state:FocusState) -> - TestCase(f labelPath code, state) - | TestList (tests:Test list, state:FocusState) -> - TestList (tests |> List.map (recMapping labelPath), state) - | TestLabel (label:string, test:Test, state:FocusState) -> - TestLabel(label, recMapping (labelPath + "/" + label) test, state) - | Test.Sequenced (sequenceMethod, test) -> - Test.Sequenced(sequenceMethod, recMapping labelPath test) + | Expecto.TestCase (code:Expecto.TestCode, state:Expecto.FocusState) -> + Expecto.TestCase(f labelPath code, state) + | Expecto.TestList (tests:Expecto.Test list, state:Expecto.FocusState) -> + Expecto.TestList (tests |> List.map (recMapping labelPath), state) + | Expecto.TestLabel (label:string, test:Expecto.Test, state:Expecto.FocusState) -> + Expecto.TestLabel(label, recMapping (labelPath + "/" + label) test, state) + | Expecto.Test.Sequenced (sequenceMethod, test) -> + Expecto.Test.Sequenced(sequenceMethod, recMapping labelPath test) recMapping "" test diff --git a/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj b/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj index d70cd1e599c..a09442ad45c 100644 --- a/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj +++ b/src/test/Fake.ExpectoSupport/Fake.ExpectoSupport.fsproj @@ -5,11 +5,10 @@ + - - From ecc71c9c5c7c3c3c4ab0f894ea9540bb68a7d420 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Tue, 17 Dec 2019 20:00:22 +0100 Subject: [PATCH 13/14] make travis parallel again --- .travis.yml | 2 +- Fake.sln | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2bfb816657..4ba57e1cac3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ mono: script: - dotnet tool restore - - dotnet fake build + - dotnet fake build --parallel 3 branches: except: diff --git a/Fake.sln b/Fake.sln index b86939e0d5d..8f2546645b5 100644 --- a/Fake.sln +++ b/Fake.sln @@ -132,6 +132,7 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Files", "Solution Files", "{03CB61B6-EBB8-4C4A-B6A3-0D84D1F78A92}" ProjectSection(SolutionItems) = preProject .gitlab-ci.yml = .gitlab-ci.yml + .travis.yml = .travis.yml build.fsx = build.fsx legacy-build.fsx = legacy-build.fsx paket.dependencies = paket.dependencies @@ -181,7 +182,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fake.DotNet.Xdt", "src\app\ EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fake.DotNet.Testing.Coverlet", "src\app\Fake.DotNet.Testing.Coverlet\Fake.DotNet.Testing.Coverlet.fsproj", "{664A121E-17A2-453E-BC2E-1C59A67875D2}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Fake.ExpectoSupport", "src\test\Fake.ExpectoSupport\Fake.ExpectoSupport.fsproj", "{D063FC91-8F84-406D-AA48-9E946B7E4323}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fake.ExpectoSupport", "src\test\Fake.ExpectoSupport\Fake.ExpectoSupport.fsproj", "{D063FC91-8F84-406D-AA48-9E946B7E4323}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From b5622f759b04d372b5420cb789e00701955b1d44 Mon Sep 17 00:00:00 2001 From: Matthias Dittrich Date: Tue, 17 Dec 2019 21:11:52 +0100 Subject: [PATCH 14/14] fix gitlab by adding summary printers again --- Fake.sln | 2 ++ src/test/Fake.Core.CommandLine.UnitTests/Main.fs | 3 ++- src/test/Fake.Core.IntegrationTests/Main.fs | 4 +++- src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj | 1 + src/test/Fake.Core.UnitTests/Main.fs | 3 ++- src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs | 4 +++- src/test/Fake.ExpectoSupport/ExpectoHelpers.fs | 3 +++ 7 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Fake.sln b/Fake.sln index 8f2546645b5..20d0268ff1a 100644 --- a/Fake.sln +++ b/Fake.sln @@ -133,7 +133,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Files", "Solution ProjectSection(SolutionItems) = preProject .gitlab-ci.yml = .gitlab-ci.yml .travis.yml = .travis.yml + appveyor.yml = appveyor.yml build.fsx = build.fsx + global.json = global.json legacy-build.fsx = legacy-build.fsx paket.dependencies = paket.dependencies paket.lock = paket.lock diff --git a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs index e8c5bac95b3..d06b53b94a4 100644 --- a/src/test/Fake.Core.CommandLine.UnitTests/Main.fs +++ b/src/test/Fake.Core.CommandLine.UnitTests/Main.fs @@ -7,8 +7,9 @@ open System let main argv = let writeResults = TestResults.writeNUnitSummary ("Fake_Core_CommandLine_UnitTests.TestResults.xml", "Fake.Core.CommandLine.UnitTests") let config = - defaultConfig.appendSummaryHandler writeResults + defaultConfig |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) |> ExpectoHelpers.setFakePrinter + |> ExpectoHelpers.appendSummaryHandler writeResults Expecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.Core.IntegrationTests/Main.fs b/src/test/Fake.Core.IntegrationTests/Main.fs index 39a0b2f99e4..dfd6e238ffd 100644 --- a/src/test/Fake.Core.IntegrationTests/Main.fs +++ b/src/test/Fake.Core.IntegrationTests/Main.fs @@ -7,7 +7,9 @@ open System let main argv = let writeResults = TestResults.writeNUnitSummary ("Fake_Core_IntegrationTests.TestResults.xml", "Fake.Core.IntegrationTests") let config = - defaultConfig.appendSummaryHandler writeResults + defaultConfig |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) |> ExpectoHelpers.setFakePrinter + |> ExpectoHelpers.appendSummaryHandler writeResults + Expecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj b/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj index 0914abf936e..cce628d6ff3 100644 --- a/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj +++ b/src/test/Fake.Core.UnitTests/Fake.Core.UnitTests.fsproj @@ -36,6 +36,7 @@ + diff --git a/src/test/Fake.Core.UnitTests/Main.fs b/src/test/Fake.Core.UnitTests/Main.fs index 0699341620e..e7f0f3e3ff2 100644 --- a/src/test/Fake.Core.UnitTests/Main.fs +++ b/src/test/Fake.Core.UnitTests/Main.fs @@ -8,7 +8,8 @@ open Fake.ExpectoSupport let main argv = let writeResults = TestResults.writeNUnitSummary ("Fake_Core_UnitTests.TestResults.xml", "Fake.Core.UnitTests") let config = - defaultConfig.appendSummaryHandler writeResults + defaultConfig |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) |> ExpectoHelpers.setFakePrinter + |> ExpectoHelpers.appendSummaryHandler writeResults Expecto.Tests.runTestsInAssembly config argv diff --git a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs index 4612f3cc2d4..83c56101335 100644 --- a/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs +++ b/src/test/Fake.DotNet.Cli.IntegrationTests/Main.fs @@ -7,7 +7,9 @@ open System let main argv = let writeResults = TestResults.writeNUnitSummary ("Fake_DotNet_Cli_IntegrationTests.TestResults.xml", "Fake.DotNet.Cli.IntegrationTests") let config = - defaultConfig.appendSummaryHandler writeResults + defaultConfig |> ExpectoHelpers.addTimeout (TimeSpan.FromMinutes(20.)) |> ExpectoHelpers.setFakePrinter + |> ExpectoHelpers.appendSummaryHandler writeResults + Expecto.Tests.runTestsInAssembly { config with parallel = false } argv diff --git a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs index 7ee53e9e165..e80fee0e0fe 100644 --- a/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs +++ b/src/test/Fake.ExpectoSupport/ExpectoHelpers.fs @@ -82,6 +82,9 @@ module ExpectoHelpers = let setFakePrinter (config:Expecto.Impl.ExpectoConfig) = setPrinter fakeDefaultPrinter config + let appendSummaryHandler summaryPrinter (config:Expecto.Impl.ExpectoConfig) = + config.appendSummaryHandler summaryPrinter + let addFilter f (config:Expecto.Impl.ExpectoConfig) = { config with filter = fun test ->