From c7eec8aebbcb95ecf7260057007613a8a33bbacb Mon Sep 17 00:00:00 2001
From: Davide Fissore <fisso9900@gmail.com>
Date: Tue, 5 Nov 2024 14:17:51 +0100
Subject: [PATCH] [test-runner] add stop-on-first-error flag to makefile

---
 Makefile           |  3 +++
 tests/test.real.ml | 19 +++++++++++++++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 1fb8b29a2..a981e8d39 100644
--- a/Makefile
+++ b/Makefile
@@ -12,6 +12,7 @@ help:
 	@echo '  tests ONLY=rex         runs only tests matching rex'
 	@echo '  tests PROMOTE=true     runs and promote tests if different'
 	@echo '                           (can be combined with ONLY)'
+	@echo '  tests STOP_ON_FST_ERROR=true stops the test suite after first error'
 	@echo
 	@echo '  git/treeish            checkout treeish and build elpi.git.treeish'
 	@echo
@@ -28,6 +29,7 @@ BUILD=_build/default
 SHELL:=/usr/bin/env bash
 TIMEOUT=90.0
 PROMOTE=false
+STOP_ON_FST_ERROR=false
 PWD=$(shell pwd)
 RUNNERS=\
   dune \
@@ -83,6 +85,7 @@ tests:
 		--seed $$RANDOM \
 		--promote $(PROMOTE) \
 		--timeout $(TIMEOUT) \
+		--stop-on-first-error=$(STOP_ON_FST_ERROR) \
 		$(TIME) \
 		--sources=$(PWD)/tests/sources/ \
 		--plot=$(PWD)/tests/plot \
diff --git a/tests/test.real.ml b/tests/test.real.ml
index abe330fe2..67a3643af 100644
--- a/tests/test.real.ml
+++ b/tests/test.real.ml
@@ -87,7 +87,7 @@ let aNSITerminal_move_bol () =
   if Sys.win32 then ANSITerminal.printf [] "\n%!"
   else ANSITerminal.move_bol ()
 
-let run timeout _seed sources promote env { Runner.run; test; executable }  =
+let run stop_on_first_error timeout _seed sources promote env { Runner.run; test; executable }  =
 
   let { Test.name; description; _ } = test in
   let print = Printer.print ~executable:(Filename.basename executable) ~name ~description ~promote in
@@ -143,7 +143,14 @@ let rec find_map f = function
       | Some y -> y
       | None -> find_map f xs
 
-let main sources plot timeout promote executables namef catskip timetool seed =
+let rec map_stop_on_error stop_on_first_error f = function
+  | [] -> []
+  | x :: xs -> 
+    match f x with
+    | Some Runner.{rc=Failure _} as err when stop_on_first_error -> [err]
+    | e -> e :: map_stop_on_error stop_on_first_error f xs
+
+let main sources plot timeout promote executables namef catskip timetool seed stop_on_first_error =
   Random.init seed;
   let filter_name =
     let rex = Str.regexp (".*"^namef) in
@@ -156,7 +163,7 @@ let main sources plot timeout promote executables namef catskip timetool seed =
     tests |> List.map (Suite.Runner.jobs ~timetool ~executables ~promote)
           |> List.concat in
   let results =
-    List.map (run timeout seed sources promote env) jobs in
+    map_stop_on_error stop_on_first_error (run stop_on_first_error timeout seed sources promote env) jobs in
   let total, ok, ko_list, skipped, timeout =
     let rec part total ok ko_list skipped timeout = function
       | [] -> (total, ok, List.rev ko_list, skipped, timeout)
@@ -223,6 +230,10 @@ let promote =
   let doc = "Promotes the tests (if failing)" in
   Arg.(value & opt bool false & info ["promote"] ~docv:"PATH" ~doc)
 
+let stop_on_first_error = 
+  let doc = "Stops test execution on first error" in
+  Arg.(value & opt bool false & info ["stop-on-first-error"] ~docv:"PATH" ~doc)
+
 let info =
   let doc = "run the test suite" in
   let tests = Test.names ()
@@ -234,5 +245,5 @@ let info =
 ;;
 
 let () =
-  (Term.exit @@ Term.eval (Term.(const main $ src $ plot $ timeout $ promote $ runners $ namef $ catskip $ mem $ seed),info)) [@ warning "-A"]
+  (Term.exit @@ Term.eval (Term.(const main $ src $ plot $ timeout $ promote $ runners $ namef $ catskip $ mem $ seed $ stop_on_first_error),info)) [@ warning "-A"]
   (* ocaml >= 4.08 | exit @@ Cmd.eval (Cmd.v info Term.(const main $ src $ plot $ timeout $ runners $ namef $ catskip $ mem $ seed)) *)