Collect test failure details at the end

tests/_utils/stages/__init__.py

@@ -23,6 +23,7 @@
 
 from .. import FeatureType
 from .test_stage import TestStage
+from .util import log_proc
 
 if sys.platform == "darwin":
     from ._osx import CPU, Eager, GPU, OMP

tests/_utils/stages/test_stage.py

@@ -231,8 +231,8 @@ def run(
 
         self.delay(shard, config, system)
 
-        result = system.run(cmd, env=self._env(config, system))
-        log_proc(self.name, result, test_file, config)
+        result = system.run(cmd, test_file, env=self._env(config, system))
+        log_proc(self.name, result, config, verbose=config.verbose)
 
         self.shards.put(shard)
 

tests/_utils/stages/util.py

@@ -16,7 +16,6 @@
 
 from dataclasses import dataclass
 from datetime import timedelta
-from pathlib import Path
 from typing import Tuple, Union
 
 from typing_extensions import TypeAlias
@@ -66,6 +65,24 @@ def passed(self) -> int:
 
 
 def adjust_workers(workers: int, requested_workers: Union[int, None]) -> int:
+    """Adjust computed workers according to command line requested workers.
+
+    The final number of workers will only be adjusted down by this function.
+
+    Parameters
+    ----------
+    workers: int
+        The computed number of workers to use
+
+    requested_workers: int | None, optional
+        Requested number of workers from the user, if supplied (default: None)
+
+    Returns
+    -------
+    int
+        The number of workers to actually use
+
+    """
     if requested_workers is not None and requested_workers < 0:
         raise ValueError("requested workers must be non-negative")
 
@@ -83,12 +100,13 @@ def adjust_workers(workers: int, requested_workers: Union[int, None]) -> int:
 
 
 def log_proc(
-    name: str, proc: ProcessResult, test_file: Path, config: Config
+    name: str, proc: ProcessResult, config: Config, *, verbose: bool
 ) -> None:
+    """Log a process result according to the current configuration"""
     if config.debug or config.dry_run:
         LOG(shell(proc.invocation))
-    msg = f"({name}) {test_file}"
-    details = proc.output.split("\n") if config.verbose else None
+    msg = f"({name}) {proc.test_file}"
+    details = proc.output.split("\n") if verbose else None
     if proc.skipped:
         LOG(skipped(msg))
     elif proc.returncode == 0:

tests/_utils/system.py

@@ -23,6 +23,7 @@
 import sys
 from dataclasses import dataclass
 from functools import cached_property
+from pathlib import Path
 from subprocess import PIPE, STDOUT, run as stdlib_run
 from typing import Sequence
 
@@ -35,6 +36,9 @@ class ProcessResult:
     #: The command invovation, including relevant environment vars
     invocation: str
 
+    #  User-friendly test file path to use in reported output
+    test_file: Path
+
     #: Whether this process was actually invoked
     skipped: bool = False
 
@@ -67,6 +71,7 @@ def __init__(
     def run(
         self,
         cmd: Sequence[str],
+        test_file: Path,
         *,
         env: EnvDict | None = None,
         cwd: str | None = None,
@@ -79,6 +84,9 @@ def run(
             The command to run, split on whitespace into a sequence
             of strings
 
+        test_file : Path
+            User-friendly test file path to use in reported output
+
         env : dict[str, str] or None, optional, default: None
             Environment variables to apply when running the command
 
@@ -97,7 +105,7 @@ def run(
         invocation = envstr + " ".join(cmd)
 
         if self.dry_run:
-            return ProcessResult(invocation, skipped=True)
+            return ProcessResult(invocation, test_file, skipped=True)
 
         full_env = dict(os.environ)
         full_env.update(env)
@@ -107,7 +115,10 @@ def run(
         )
 
         return ProcessResult(
-            invocation, returncode=proc.returncode, output=proc.stdout
+            invocation,
+            test_file,
+            returncode=proc.returncode,
+            output=proc.stdout,
         )
 
     @cached_property

tests/_utils/test_plan.py

@@ -22,7 +22,7 @@
 
 from .config import Config
 from .logger import LOG
-from .stages import STAGES
+from .stages import STAGES, log_proc
 from .system import System
 from .ui import banner, rule, summary, yellow
 
@@ -64,6 +64,10 @@ def execute(self) -> int:
         total = len(all_procs)
         passed = sum(proc.returncode == 0 for proc in all_procs)
 
+        LOG(f"\n{rule()}")
+
+        self._log_failures(total, passed)
+
         LOG(self.outro(total, passed))
 
         return int((total - passed) > 0)
@@ -111,6 +115,17 @@ def outro(self, total: int, passed: int) -> str:
             summary("All tests", total, passed, time, justify=False)
         )
 
-        result = banner("Test Suite Summary", details=details)
+        overall = banner("Overall summary", details=details)
+
+        return f"{overall}\n"
 
-        return f"\n{rule()}\n{result}\n"
+    def _log_failures(self, total: int, passed: int) -> None:
+        if total == passed:
+            return
+
+        LOG(f"{banner('FAILURES')}\n")
+
+        for stage in self._stages:
+            procs = (proc for proc in stage.result.procs if proc.returncode)
+            for proc in procs:
+                log_proc(stage.name, proc, self._config, verbose=True)

tests/_utils/tests/stages/test_test_stage.py

@@ -60,7 +60,8 @@ def test_outro(self) -> None:
         c = Config([])
         stage = MockTestStage(c, s)
         stage.result = StageResult(
-            [ProcessResult("invoke")], timedelta(seconds=2.123)
+            [ProcessResult("invoke", Path("test/file"))],
+            timedelta(seconds=2.123),
         )
         outro = stage.outro
         assert "Exiting stage: mock" in outro

tests/_utils/tests/test_system.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import sys
+from pathlib import Path
 from subprocess import CompletedProcess
 from unittest.mock import MagicMock
 
@@ -43,20 +44,22 @@ def test_init(self) -> None:
     def test_run(self, mock_subprocess_run: MagicMock) -> None:
         s = m.System()
 
-        expected = m.ProcessResult(CMD, returncode=10, output="<output>")
+        expected = m.ProcessResult(
+            CMD, Path("test/file"), returncode=10, output="<output>"
+        )
         mock_subprocess_run.return_value = CompletedProcess(
             CMD, 10, stdout="<output>"
         )
 
-        result = s.run(CMD.split())
+        result = s.run(CMD.split(), Path("test/file"))
         mock_subprocess_run.assert_called()
 
         assert result == expected
 
     def test_dry_run(self, mock_subprocess_run: MagicMock) -> None:
         s = m.System(dry_run=True)
 
-        result = s.run(CMD.split())
+        result = s.run(CMD.split(), Path("test/file"))
         mock_subprocess_run.assert_not_called()
 
         assert result.output == ""