From 393f7042b5439054c5a74fb0feb1a778c6678e1a Mon Sep 17 00:00:00 2001 From: Bryan Van de Ven Date: Tue, 23 Aug 2022 08:15:40 -0700 Subject: [PATCH] collect test failure details at the end --- tests/_utils/stages/__init__.py | 1 + tests/_utils/stages/test_stage.py | 4 +-- tests/_utils/stages/util.py | 26 +++++++++++++++++--- tests/_utils/system.py | 15 +++++++++-- tests/_utils/test_plan.py | 21 +++++++++++++--- tests/_utils/tests/stages/test_test_stage.py | 3 ++- tests/_utils/tests/test_system.py | 9 ++++--- 7 files changed, 64 insertions(+), 15 deletions(-) diff --git a/tests/_utils/stages/__init__.py b/tests/_utils/stages/__init__.py index ed960219d..fa8f916d5 100644 --- a/tests/_utils/stages/__init__.py +++ b/tests/_utils/stages/__init__.py @@ -23,6 +23,7 @@ from .. import FeatureType from .test_stage import TestStage +from .util import log_proc if sys.platform == "darwin": from ._osx import CPU, Eager, GPU, OMP diff --git a/tests/_utils/stages/test_stage.py b/tests/_utils/stages/test_stage.py index 3ad7e5111..848f74b42 100644 --- a/tests/_utils/stages/test_stage.py +++ b/tests/_utils/stages/test_stage.py @@ -231,8 +231,8 @@ def run( self.delay(shard, config, system) - result = system.run(cmd, env=self._env(config, system)) - log_proc(self.name, result, test_file, config) + result = system.run(cmd, test_file, env=self._env(config, system)) + log_proc(self.name, result, config, verbose=config.verbose) self.shards.put(shard) diff --git a/tests/_utils/stages/util.py b/tests/_utils/stages/util.py index 1733654ef..357474c90 100644 --- a/tests/_utils/stages/util.py +++ b/tests/_utils/stages/util.py @@ -16,7 +16,6 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import Tuple, Union from typing_extensions import TypeAlias @@ -66,6 +65,24 @@ def passed(self) -> int: def adjust_workers(workers: int, requested_workers: Union[int, None]) -> int: + """Adjust computed workers according to command line requested workers. + + The final number of workers will only be adjusted down by this function. + + Parameters + ---------- + workers: int + The computed number of workers to use + + requested_workers: int | None, optional + Requested number of workers from the user, if supplied (default: None) + + Returns + ------- + int + The number of workers to actually use + + """ if requested_workers is not None and requested_workers < 0: raise ValueError("requested workers must be non-negative") @@ -83,12 +100,13 @@ def adjust_workers(workers: int, requested_workers: Union[int, None]) -> int: def log_proc( - name: str, proc: ProcessResult, test_file: Path, config: Config + name: str, proc: ProcessResult, config: Config, *, verbose: bool ) -> None: + """Log a process result according to the current configuration""" if config.debug or config.dry_run: LOG(shell(proc.invocation)) - msg = f"({name}) {test_file}" - details = proc.output.split("\n") if config.verbose else None + msg = f"({name}) {proc.test_file}" + details = proc.output.split("\n") if verbose else None if proc.skipped: LOG(skipped(msg)) elif proc.returncode == 0: diff --git a/tests/_utils/system.py b/tests/_utils/system.py index 86de556f3..818da394f 100644 --- a/tests/_utils/system.py +++ b/tests/_utils/system.py @@ -23,6 +23,7 @@ import sys from dataclasses import dataclass from functools import cached_property +from pathlib import Path from subprocess import PIPE, STDOUT, run as stdlib_run from typing import Sequence @@ -35,6 +36,9 @@ class ProcessResult: #: The command invovation, including relevant environment vars invocation: str + # User-friendly test file path to use in reported output + test_file: Path + #: Whether this process was actually invoked skipped: bool = False @@ -67,6 +71,7 @@ def __init__( def run( self, cmd: Sequence[str], + test_file: Path, *, env: EnvDict | None = None, cwd: str | None = None, @@ -79,6 +84,9 @@ def run( The command to run, split on whitespace into a sequence of strings + test_file : Path + User-friendly test file path to use in reported output + env : dict[str, str] or None, optional, default: None Environment variables to apply when running the command @@ -97,7 +105,7 @@ def run( invocation = envstr + " ".join(cmd) if self.dry_run: - return ProcessResult(invocation, skipped=True) + return ProcessResult(invocation, test_file, skipped=True) full_env = dict(os.environ) full_env.update(env) @@ -107,7 +115,10 @@ def run( ) return ProcessResult( - invocation, returncode=proc.returncode, output=proc.stdout + invocation, + test_file, + returncode=proc.returncode, + output=proc.stdout, ) @cached_property diff --git a/tests/_utils/test_plan.py b/tests/_utils/test_plan.py index cc1ad6004..9e2a92532 100644 --- a/tests/_utils/test_plan.py +++ b/tests/_utils/test_plan.py @@ -22,7 +22,7 @@ from .config import Config from .logger import LOG -from .stages import STAGES +from .stages import STAGES, log_proc from .system import System from .ui import banner, rule, summary, yellow @@ -64,6 +64,10 @@ def execute(self) -> int: total = len(all_procs) passed = sum(proc.returncode == 0 for proc in all_procs) + LOG(f"\n{rule()}") + + self._log_failures(total, passed) + LOG(self.outro(total, passed)) return int((total - passed) > 0) @@ -111,6 +115,17 @@ def outro(self, total: int, passed: int) -> str: summary("All tests", total, passed, time, justify=False) ) - result = banner("Test Suite Summary", details=details) + overall = banner("Overall summary", details=details) + + return f"{overall}\n" - return f"\n{rule()}\n{result}\n" + def _log_failures(self, total: int, passed: int) -> None: + if total == passed: + return + + LOG(f"{banner('FAILURES')}\n") + + for stage in self._stages: + procs = (proc for proc in stage.result.procs if proc.returncode) + for proc in procs: + log_proc(stage.name, proc, self._config, verbose=True) diff --git a/tests/_utils/tests/stages/test_test_stage.py b/tests/_utils/tests/stages/test_test_stage.py index 1e9731442..393ac18bc 100644 --- a/tests/_utils/tests/stages/test_test_stage.py +++ b/tests/_utils/tests/stages/test_test_stage.py @@ -60,7 +60,8 @@ def test_outro(self) -> None: c = Config([]) stage = MockTestStage(c, s) stage.result = StageResult( - [ProcessResult("invoke")], timedelta(seconds=2.123) + [ProcessResult("invoke", Path("test/file"))], + timedelta(seconds=2.123), ) outro = stage.outro assert "Exiting stage: mock" in outro diff --git a/tests/_utils/tests/test_system.py b/tests/_utils/tests/test_system.py index d883f3e52..d110e260f 100644 --- a/tests/_utils/tests/test_system.py +++ b/tests/_utils/tests/test_system.py @@ -18,6 +18,7 @@ from __future__ import annotations import sys +from pathlib import Path from subprocess import CompletedProcess from unittest.mock import MagicMock @@ -43,12 +44,14 @@ def test_init(self) -> None: def test_run(self, mock_subprocess_run: MagicMock) -> None: s = m.System() - expected = m.ProcessResult(CMD, returncode=10, output="") + expected = m.ProcessResult( + CMD, Path("test/file"), returncode=10, output="" + ) mock_subprocess_run.return_value = CompletedProcess( CMD, 10, stdout="" ) - result = s.run(CMD.split()) + result = s.run(CMD.split(), Path("test/file")) mock_subprocess_run.assert_called() assert result == expected @@ -56,7 +59,7 @@ def test_run(self, mock_subprocess_run: MagicMock) -> None: def test_dry_run(self, mock_subprocess_run: MagicMock) -> None: s = m.System(dry_run=True) - result = s.run(CMD.split()) + result = s.run(CMD.split(), Path("test/file")) mock_subprocess_run.assert_not_called() assert result.output == ""