From e03fd7bec1d1add1e8c3fba516ed88ce13a96af8 Mon Sep 17 00:00:00 2001 From: jce Date: Sat, 7 Dec 2024 15:05:29 +0100 Subject: [PATCH 1/4] Add config param 'allow_unconfigured' --- README.rst | 10 +++++++--- src/mlx/warnings/robot_checker.py | 30 +++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index a4608bec..2c481e62 100644 --- a/README.rst +++ b/README.rst @@ -314,9 +314,13 @@ of all test suites in the file are taken into account. If you only care about one specific test suite, you can use ``--name <>``. If this suite name doesn't exist in the input file, an error is raised. The warning limits can be configured for multiple test suites individually by means of a -`configuration file to pass options`_. If the setting ``"check_suite_names"`` -is false, no error is raised when a suite name doesn't exist in the -input file. When this setting is missing, the default value ``true`` is used. +`configuration file to pass options`_, which supports additional configuration parameters: + +- ``"check_suite_names"``, which is ``true`` by default, raises an error when a configured suite name + does not exist in the input XML file. This helps you verify that all of the test suites in your configuration file + were passed to the `robot` CLI. +- ``"allow_unconfigured"``, which is ``true`` by default, raises an error when a suite name in the input + XML file is missing from your configuration file. Note: a configuration value ``""`` matches all suite names. .. code-block:: bash diff --git a/src/mlx/warnings/robot_checker.py b/src/mlx/warnings/robot_checker.py index 9dc57e6e..967cb372 100644 --- a/src/mlx/warnings/robot_checker.py +++ b/src/mlx/warnings/robot_checker.py @@ -5,13 +5,23 @@ from junitparser import Error, Failure +from .exceptions import WarningsConfigError from .junit_checker import JUnitChecker from .warnings_checker import WarningsChecker class RobotChecker(WarningsChecker): name = 'robot' - checkers = [] + + def __init__(self, verbose=False): + ''' Constructor + + Args: + verbose (bool): Enable/disable verbose logging + ''' + super().__init__(verbose) + self.checkers = [] + self.allow_unconfigured = True @property def counted_warnings(self): @@ -53,6 +63,11 @@ def maximum(self, maximum): for checker in self.checkers: checker.maximum = maximum + @property + def ignored_testsuites(self): + ignored_testcases = set.intersection(*(c.ignored_testsuites for c in self.checkers)) + return sorted({t.classname.split(".")[-1] for t in ignored_testcases}) + def check(self, content): ''' Function for counting the number of failures in a specific Robot @@ -73,6 +88,9 @@ def return_count(self): self.count = 0 for checker in self.checkers: self.count += checker.return_count() + if not self.allow_unconfigured and self.ignored_testsuites: + raise WarningsConfigError(f"{len(self.ignored_testsuites)} test suites have been ignored due to " + f"incomplete configuration: {self.ignored_testsuites}") return self.count def return_check_limits(self): @@ -96,10 +114,10 @@ def return_check_limits(self): return count def parse_config(self, config): - self.checkers = [] - check_suite_name = config.get('check_suite_names', True) + self.allow_unconfigured = config.get('allow_unconfigured', True) for suite_config in config['suites']: - checker = RobotSuiteChecker(suite_config['name'], check_suite_name=check_suite_name, + checker = RobotSuiteChecker(suite_config['name'], + check_suite_name=config.get('check_suite_names', True), verbose=self.verbose) checker.parse_config(suite_config) self.checkers.append(checker) @@ -119,6 +137,7 @@ def __init__(self, suite_name, check_suite_name=False, **kwargs): self.suite_name = suite_name self.check_suite_name = check_suite_name self.is_valid_suite_name = False + self.ignored_testsuites = set() def _check_testcase(self, testcase): """ Handles the check of a test case element by checking if the result is a failure/error. @@ -135,7 +154,8 @@ def _check_testcase(self, testcase): if testcase.classname.endswith(self.suite_name): self.is_valid_suite_name = True return super()._check_testcase(testcase) - return int(self.suite_name and isinstance(testcase.result, (Failure, Error))) + self.ignored_testsuites.add(testcase) + return int(isinstance(testcase.result, (Failure, Error))) def check(self, content): """ Function for counting the number of JUnit failures in a specific text From 71c78e10504dca5399ba2f3a4315e74e0e8462e1 Mon Sep 17 00:00:00 2001 From: jce Date: Sat, 7 Dec 2024 15:08:14 +0100 Subject: [PATCH 2/4] test e03fd7bec1d1add1e8c3fba516ed88ce13a96af8 --- tests/test_in/config_example_robot.json | 1 + tests/test_robot.py | 35 ++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_in/config_example_robot.json b/tests/test_in/config_example_robot.json index 1bc6ef96..dcdf5395 100644 --- a/tests/test_in/config_example_robot.json +++ b/tests/test_in/config_example_robot.json @@ -27,6 +27,7 @@ "robot": { "enabled": true, "check_suite_names": false, + "allow_unconfigured": false, "suites": [ { "name": "Suite One", diff --git a/tests/test_robot.py b/tests/test_robot.py index 7bd19e0d..82e62b2e 100644 --- a/tests/test_robot.py +++ b/tests/test_robot.py @@ -1,6 +1,6 @@ import unittest -from mlx.warnings import RobotSuiteChecker, WarningsPlugin +from mlx.warnings import RobotSuiteChecker, WarningsPlugin, WarningsConfigError from test_integration import run_test_with_logging @@ -67,6 +67,7 @@ def test_check_suite_name(self): self.assertEqual(c_m.exception.code, -1) def test_robot_version_5(self): + self.dut.allow_unconfigured = False self.dut.checkers = [ RobotSuiteChecker('Empty Flash Product Id', check_suite_name=True), ] @@ -75,6 +76,38 @@ def test_robot_version_5(self): count = self.warnings.return_count() self.assertEqual(count, 6) + def test_disallow_unconfigured_pass(self): + self.dut.allow_unconfigured = False + self.dut.checkers = [ + RobotSuiteChecker('Empty Flash Product Id'), + RobotSuiteChecker('Empty Flash Mlx Device Project Id'), + ] + with open('tests/test_in/robot_version_5.xml') as xmlfile: + self.warnings.check(xmlfile.read()) + count = self.warnings.return_count() + self.assertEqual(count, 8) + + def test_disallow_unconfigured_pass_wildcard(self): + self.dut.allow_unconfigured = False + self.dut.checkers = [ + RobotSuiteChecker(''), + ] + with open('tests/test_in/robot_version_5.xml') as xmlfile: + self.warnings.check(xmlfile.read()) + count = self.warnings.return_count() + self.assertEqual(count, 8) + + def test_disallow_unconfigured_fail(self): + self.dut.allow_unconfigured = False + self.dut.checkers = [ + RobotSuiteChecker('Empty Flash Mlx Device Project Id'), + ] + with open('tests/test_in/robot_version_5.xml') as xmlfile: + with self.assertRaises(WarningsConfigError) as exc: + self.warnings.check(xmlfile.read()) + self.warnings.return_count() + self.assertEqual(str(exc.exception), "1 test suites have been ignored due to incomplete configuration: ['Empty Flash Product Id']") + if __name__ == '__main__': unittest.main() From 5e1c818b759a9f6518de0ba59ffad48a832c42f1 Mon Sep 17 00:00:00 2001 From: jce Date: Sat, 14 Dec 2024 17:16:33 +0100 Subject: [PATCH 3/4] correct test --- tests/test_robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_robot.py b/tests/test_robot.py index 407a1443..b2eaf722 100644 --- a/tests/test_robot.py +++ b/tests/test_robot.py @@ -77,7 +77,7 @@ def test_check_suite_name(self): self.assertEqual(c_m.exception.code, -1) def test_robot_version_5(self): - self.dut.allow_unconfigured = False + self.dut.allow_unconfigured = True self.dut.checkers = [ RobotSuiteChecker('Empty Flash Product Id', *self.dut.logging_args, check_suite_name=True), ] From 44c6dc0a04cc111e80c11b52e73f24ba283f1cc0 Mon Sep 17 00:00:00 2001 From: jce Date: Sat, 14 Dec 2024 17:17:44 +0100 Subject: [PATCH 4/4] style fix --- src/mlx/warnings/robot_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlx/warnings/robot_checker.py b/src/mlx/warnings/robot_checker.py index 1758b152..5e31eaff 100644 --- a/src/mlx/warnings/robot_checker.py +++ b/src/mlx/warnings/robot_checker.py @@ -97,8 +97,8 @@ def return_check_limits(self): def parse_config(self, config): self.allow_unconfigured = config.get('allow_unconfigured', True) + check_suite_name = config.get('check_suite_names', True) for suite_config in config['suites']: - check_suite_name=config.get('check_suite_names', True) checker = RobotSuiteChecker(suite_config['name'], *self.logging_args, check_suite_name=check_suite_name) checker.parse_config(suite_config) self.checkers.append(checker)