Merge pull request #166 from melexis/check_robot

Robot checker: add configuration option to disallow unconfigured suite names

README.rst

@@ -314,9 +314,13 @@ of all test suites in the file are taken into account. If you only care about
 one specific test suite, you can use ``--name <<suite name>>``. If this suite
 name doesn't exist in the input file, an error is raised. The warning
 limits can be configured for multiple test suites individually by means of a
-`configuration file to pass options`_. If the setting ``"check_suite_names"``
-is false, no error is raised when a suite name doesn't exist in the
-input file. When this setting is missing, the default value ``true`` is used.
+`configuration file to pass options`_, which supports additional configuration parameters:
+
+- ``"check_suite_names"``, which is ``true`` by default, raises an error when a configured suite name
+  does not exist in the input XML file. This helps you verify that all of the test suites in your configuration file
+  were passed to the `robot` CLI.
+- ``"allow_unconfigured"``, which is ``true`` by default, raises an error when a suite name in the input
+  XML file is missing from your configuration file. Note: a configuration value ``""`` matches all suite names.
 
 .. code-block:: bash
 

src/mlx/warnings/robot_checker.py

@@ -4,15 +4,21 @@
 
 from junitparser import Error, Failure
 
+from .exceptions import WarningsConfigError
 from .junit_checker import JUnitChecker
 from .warnings_checker import WarningsChecker
 
 
 class RobotChecker(WarningsChecker):
     name = "robot"
-    checkers = []
     logging_fmt = "{checker.name_repr}: {message}"
 
+    def __init__(self, *logging_args):
+        ''' Constructor '''
+        super().__init__(*logging_args)
+        self.checkers = []
+        self.allow_unconfigured = True
+
     @property
     def minimum(self):
         """Gets the lowest minimum amount of warnings
@@ -45,6 +51,11 @@ def maximum(self, maximum):
         for checker in self.checkers:
             checker.maximum = maximum
 
+    @property
+    def ignored_testsuites(self):
+        ignored_testcases = set.intersection(*(c.ignored_testsuites for c in self.checkers))
+        return sorted({t.classname.split(".")[-1] for t in ignored_testcases})
+
     def check(self, content):
         """
         Function for counting the number of failures in a specific Robot
@@ -65,6 +76,9 @@ def return_count(self):
         self.count = 0
         for checker in self.checkers:
             self.count += checker.return_count()
+        if not self.allow_unconfigured and self.ignored_testsuites:
+            raise WarningsConfigError(f"{len(self.ignored_testsuites)} test suites have been ignored due to "
+                                      f"incomplete configuration: {self.ignored_testsuites}")
         return self.count
 
     def return_check_limits(self):
@@ -82,7 +96,7 @@ def return_check_limits(self):
         return count
 
     def parse_config(self, config):
-        self.checkers = []
+        self.allow_unconfigured = config.get("allow_unconfigured", True)
         check_suite_name = config.get("check_suite_names", True)
         for suite_config in config["suites"]:
             checker = RobotSuiteChecker(suite_config["name"], *self.logging_args, check_suite_name=check_suite_name)
@@ -105,6 +119,7 @@ def __init__(self, suite_name, *logging_args, check_suite_name=False):
         self.suite_name = suite_name
         self.check_suite_name = check_suite_name
         self.is_valid_suite_name = False
+        self.ignored_testsuites = set()
 
     @property
     def suite_name_repr(self):
@@ -126,7 +141,8 @@ def _check_testcase(self, testcase):
         if testcase.classname.endswith(self.suite_name):
             self.is_valid_suite_name = True
             return super()._check_testcase(testcase)
-        return int(self.suite_name and isinstance(testcase.result, (Failure, Error)))
+        self.ignored_testsuites.add(testcase)
+        return int(isinstance(testcase.result, (Failure, Error)))
 
     def check(self, content):
         """Function for counting the number of JUnit failures in a specific text

tests/test_in/config_example_robot.json

@@ -27,6 +27,7 @@
     "robot": {
         "enabled": true,
         "check_suite_names": false,
+        "allow_unconfigured": false,
         "suites": [
             {
                 "name": "Suite One",

tests/test_robot.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from mlx.warnings import RobotSuiteChecker, WarningsPlugin, warnings_wrapper
+from mlx.warnings import RobotSuiteChecker, WarningsConfigError, WarningsPlugin, warnings_wrapper
 
 
 class TestRobotWarnings(unittest.TestCase):
@@ -77,6 +77,7 @@ def test_check_suite_name(self):
         self.assertEqual(c_m.exception.code, -1)
 
     def test_robot_version_5(self):
+        self.dut.allow_unconfigured = True
         self.dut.checkers = [
             RobotSuiteChecker("Empty Flash Product Id", *self.dut.logging_args, check_suite_name=True),
         ]
@@ -85,6 +86,38 @@ def test_robot_version_5(self):
             count = self.warnings.return_count()
         self.assertEqual(count, 6)
 
+    def test_disallow_unconfigured_pass(self):
+        self.dut.allow_unconfigured = False
+        self.dut.checkers = [
+            RobotSuiteChecker('Empty Flash Product Id', *self.dut.logging_args),
+            RobotSuiteChecker('Empty Flash Mlx Device Project Id', *self.dut.logging_args),
+        ]
+        with open('tests/test_in/robot_version_5.xml') as xmlfile:
+            self.warnings.check(xmlfile.read())
+            count = self.warnings.return_count()
+        self.assertEqual(count, 8)
+
+    def test_disallow_unconfigured_pass_wildcard(self):
+        self.dut.allow_unconfigured = False
+        self.dut.checkers = [
+            RobotSuiteChecker('', *self.dut.logging_args),
+        ]
+        with open('tests/test_in/robot_version_5.xml') as xmlfile:
+            self.warnings.check(xmlfile.read())
+            count = self.warnings.return_count()
+        self.assertEqual(count, 8)
+
+    def test_disallow_unconfigured_fail(self):
+        self.dut.allow_unconfigured = False
+        self.dut.checkers = [
+            RobotSuiteChecker('Empty Flash Mlx Device Project Id', *self.dut.logging_args),
+        ]
+        with open('tests/test_in/robot_version_5.xml') as xmlfile:
+            with self.assertRaises(WarningsConfigError) as exc:
+                self.warnings.check(xmlfile.read())
+                self.warnings.return_count()
+        self.assertEqual(str(exc.exception), "1 test suites have been ignored due to incomplete configuration: ['Empty Flash Product Id']")
+
 
 if __name__ == "__main__":
     unittest.main()