diff --git a/README.rst b/README.rst index 901ef79..9d41046 100644 --- a/README.rst +++ b/README.rst @@ -314,9 +314,13 @@ of all test suites in the file are taken into account. If you only care about one specific test suite, you can use ``--name <>``. If this suite name doesn't exist in the input file, an error is raised. The warning limits can be configured for multiple test suites individually by means of a -`configuration file to pass options`_. If the setting ``"check_suite_names"`` -is false, no error is raised when a suite name doesn't exist in the -input file. When this setting is missing, the default value ``true`` is used. +`configuration file to pass options`_, which supports additional configuration parameters: + +- ``"check_suite_names"``, which is ``true`` by default, raises an error when a configured suite name + does not exist in the input XML file. This helps you verify that all of the test suites in your configuration file + were passed to the `robot` CLI. +- ``"allow_unconfigured"``, which is ``true`` by default, raises an error when a suite name in the input + XML file is missing from your configuration file. Note: a configuration value ``""`` matches all suite names. .. code-block:: bash diff --git a/src/mlx/warnings/robot_checker.py b/src/mlx/warnings/robot_checker.py index 93146fd..61eb617 100644 --- a/src/mlx/warnings/robot_checker.py +++ b/src/mlx/warnings/robot_checker.py @@ -4,15 +4,21 @@ from junitparser import Error, Failure +from .exceptions import WarningsConfigError from .junit_checker import JUnitChecker from .warnings_checker import WarningsChecker class RobotChecker(WarningsChecker): name = "robot" - checkers = [] logging_fmt = "{checker.name_repr}: {message}" + def __init__(self, *logging_args): + ''' Constructor ''' + super().__init__(*logging_args) + self.checkers = [] + self.allow_unconfigured = True + @property def minimum(self): """Gets the lowest minimum amount of warnings @@ -45,6 +51,11 @@ def maximum(self, maximum): for checker in self.checkers: checker.maximum = maximum + @property + def ignored_testsuites(self): + ignored_testcases = set.intersection(*(c.ignored_testsuites for c in self.checkers)) + return sorted({t.classname.split(".")[-1] for t in ignored_testcases}) + def check(self, content): """ Function for counting the number of failures in a specific Robot @@ -65,6 +76,9 @@ def return_count(self): self.count = 0 for checker in self.checkers: self.count += checker.return_count() + if not self.allow_unconfigured and self.ignored_testsuites: + raise WarningsConfigError(f"{len(self.ignored_testsuites)} test suites have been ignored due to " + f"incomplete configuration: {self.ignored_testsuites}") return self.count def return_check_limits(self): @@ -82,7 +96,7 @@ def return_check_limits(self): return count def parse_config(self, config): - self.checkers = [] + self.allow_unconfigured = config.get("allow_unconfigured", True) check_suite_name = config.get("check_suite_names", True) for suite_config in config["suites"]: checker = RobotSuiteChecker(suite_config["name"], *self.logging_args, check_suite_name=check_suite_name) @@ -105,6 +119,7 @@ def __init__(self, suite_name, *logging_args, check_suite_name=False): self.suite_name = suite_name self.check_suite_name = check_suite_name self.is_valid_suite_name = False + self.ignored_testsuites = set() @property def suite_name_repr(self): @@ -126,7 +141,8 @@ def _check_testcase(self, testcase): if testcase.classname.endswith(self.suite_name): self.is_valid_suite_name = True return super()._check_testcase(testcase) - return int(self.suite_name and isinstance(testcase.result, (Failure, Error))) + self.ignored_testsuites.add(testcase) + return int(isinstance(testcase.result, (Failure, Error))) def check(self, content): """Function for counting the number of JUnit failures in a specific text diff --git a/tests/test_in/config_example_robot.json b/tests/test_in/config_example_robot.json index 1bc6ef9..dcdf539 100644 --- a/tests/test_in/config_example_robot.json +++ b/tests/test_in/config_example_robot.json @@ -27,6 +27,7 @@ "robot": { "enabled": true, "check_suite_names": false, + "allow_unconfigured": false, "suites": [ { "name": "Suite One", diff --git a/tests/test_robot.py b/tests/test_robot.py index 096797c..ef6d437 100644 --- a/tests/test_robot.py +++ b/tests/test_robot.py @@ -2,7 +2,7 @@ import pytest -from mlx.warnings import RobotSuiteChecker, WarningsPlugin, warnings_wrapper +from mlx.warnings import RobotSuiteChecker, WarningsConfigError, WarningsPlugin, warnings_wrapper class TestRobotWarnings(unittest.TestCase): @@ -77,6 +77,7 @@ def test_check_suite_name(self): self.assertEqual(c_m.exception.code, -1) def test_robot_version_5(self): + self.dut.allow_unconfigured = True self.dut.checkers = [ RobotSuiteChecker("Empty Flash Product Id", *self.dut.logging_args, check_suite_name=True), ] @@ -85,6 +86,38 @@ def test_robot_version_5(self): count = self.warnings.return_count() self.assertEqual(count, 6) + def test_disallow_unconfigured_pass(self): + self.dut.allow_unconfigured = False + self.dut.checkers = [ + RobotSuiteChecker('Empty Flash Product Id', *self.dut.logging_args), + RobotSuiteChecker('Empty Flash Mlx Device Project Id', *self.dut.logging_args), + ] + with open('tests/test_in/robot_version_5.xml') as xmlfile: + self.warnings.check(xmlfile.read()) + count = self.warnings.return_count() + self.assertEqual(count, 8) + + def test_disallow_unconfigured_pass_wildcard(self): + self.dut.allow_unconfigured = False + self.dut.checkers = [ + RobotSuiteChecker('', *self.dut.logging_args), + ] + with open('tests/test_in/robot_version_5.xml') as xmlfile: + self.warnings.check(xmlfile.read()) + count = self.warnings.return_count() + self.assertEqual(count, 8) + + def test_disallow_unconfigured_fail(self): + self.dut.allow_unconfigured = False + self.dut.checkers = [ + RobotSuiteChecker('Empty Flash Mlx Device Project Id', *self.dut.logging_args), + ] + with open('tests/test_in/robot_version_5.xml') as xmlfile: + with self.assertRaises(WarningsConfigError) as exc: + self.warnings.check(xmlfile.read()) + self.warnings.return_count() + self.assertEqual(str(exc.exception), "1 test suites have been ignored due to incomplete configuration: ['Empty Flash Product Id']") + if __name__ == "__main__": unittest.main()