Robot checker: add configuration option to disallow unconfigured suite names

README.rst

@@ -314,9 +314,13 @@ of all test suites in the file are taken into account. If you only care about
 one specific test suite, you can use ``--name <<suite name>>``. If this suite
 name doesn't exist in the input file, an error is raised. The warning
 limits can be configured for multiple test suites individually by means of a
-`configuration file to pass options`_. If the setting ``"check_suite_names"``
-is false, no error is raised when a suite name doesn't exist in the
-input file. When this setting is missing, the default value ``true`` is used.
+`configuration file to pass options`_, which supports additional configuration parameters:
+
+- ``"check_suite_names"``, which is ``true`` by default, raises an error when a configured suite name
+  does not exist in the input XML file. This helps you verify that all of the test suites in your configuration file
+  were passed to the `robot` CLI.
+- ``"allow_unconfigured"``, which is ``true`` by default, raises an error when a suite name in the input
+  XML file is missing from your configuration file. Note: a configuration value ``""`` matches all suite names.
 
 .. code-block:: bash
 

src/mlx/warnings/robot_checker.py

@@ -5,13 +5,23 @@
 
 from junitparser import Error, Failure
 
+from .exceptions import WarningsConfigError
 from .junit_checker import JUnitChecker
 from .warnings_checker import WarningsChecker
 
 
 class RobotChecker(WarningsChecker):
     name = 'robot'
-    checkers = []
+
+    def __init__(self, verbose=False):
+        ''' Constructor
+
+        Args:
+            verbose (bool): Enable/disable verbose logging
+        '''
+        super().__init__(verbose)
+        self.checkers = []
+        self.allow_unconfigured = True
 
     @property
     def counted_warnings(self):
@@ -53,6 +63,11 @@ def maximum(self, maximum):
         for checker in self.checkers:
             checker.maximum = maximum
 
+    @property
+    def ignored_testsuites(self):
+        ignored_testcases = set.intersection(*(c.ignored_testsuites for c in self.checkers))
+        return sorted({t.classname.split(".")[-1] for t in ignored_testcases})
+
     def check(self, content):
         '''
         Function for counting the number of failures in a specific Robot
@@ -73,6 +88,9 @@ def return_count(self):
         self.count = 0
         for checker in self.checkers:
             self.count += checker.return_count()
+        if not self.allow_unconfigured and self.ignored_testsuites:
+            raise WarningsConfigError(f"{len(self.ignored_testsuites)} test suites have been ignored due to "
+                                      f"incomplete configuration: {self.ignored_testsuites}")
         return self.count
 
     def return_check_limits(self):
@@ -96,10 +114,10 @@ def return_check_limits(self):
         return count
 
     def parse_config(self, config):
-        self.checkers = []
-        check_suite_name = config.get('check_suite_names', True)
+        self.allow_unconfigured = config.get('allow_unconfigured', True)
         for suite_config in config['suites']:
-            checker = RobotSuiteChecker(suite_config['name'], check_suite_name=check_suite_name,
+            checker = RobotSuiteChecker(suite_config['name'],
+                                        check_suite_name=config.get('check_suite_names', True),
                                         verbose=self.verbose)
             checker.parse_config(suite_config)
             self.checkers.append(checker)
@@ -119,6 +137,7 @@ def __init__(self, suite_name, check_suite_name=False, **kwargs):
         self.suite_name = suite_name
         self.check_suite_name = check_suite_name
         self.is_valid_suite_name = False
+        self.ignored_testsuites = set()
 
     def _check_testcase(self, testcase):
         """ Handles the check of a test case element by checking if the result is a failure/error.
@@ -135,7 +154,8 @@ def _check_testcase(self, testcase):
         if testcase.classname.endswith(self.suite_name):
             self.is_valid_suite_name = True
             return super()._check_testcase(testcase)
-        return int(self.suite_name and isinstance(testcase.result, (Failure, Error)))
+        self.ignored_testsuites.add(testcase)
+        return int(isinstance(testcase.result, (Failure, Error)))
 
     def check(self, content):
         """ Function for counting the number of JUnit failures in a specific text

tests/test_in/config_example_robot.json

@@ -27,6 +27,7 @@
     "robot": {
         "enabled": true,
         "check_suite_names": false,
+        "allow_unconfigured": false,
         "suites": [
             {
                 "name": "Suite One",

tests/test_robot.py

@@ -1,6 +1,6 @@
 import unittest
 
-from mlx.warnings import RobotSuiteChecker, WarningsPlugin
+from mlx.warnings import RobotSuiteChecker, WarningsPlugin, WarningsConfigError
 from test_integration import run_test_with_logging
 
 
@@ -67,6 +67,7 @@ def test_check_suite_name(self):
         self.assertEqual(c_m.exception.code, -1)
 
     def test_robot_version_5(self):
+        self.dut.allow_unconfigured = False
         self.dut.checkers = [
             RobotSuiteChecker('Empty Flash Product Id', check_suite_name=True),
         ]
@@ -75,6 +76,38 @@ def test_robot_version_5(self):
             count = self.warnings.return_count()
         self.assertEqual(count, 6)
 
+    def test_disallow_unconfigured_pass(self):
+        self.dut.allow_unconfigured = False
+        self.dut.checkers = [
+            RobotSuiteChecker('Empty Flash Product Id'),
+            RobotSuiteChecker('Empty Flash Mlx Device Project Id'),
+        ]
+        with open('tests/test_in/robot_version_5.xml') as xmlfile:
+            self.warnings.check(xmlfile.read())
+            count = self.warnings.return_count()
+        self.assertEqual(count, 8)
+
+    def test_disallow_unconfigured_pass_wildcard(self):
+        self.dut.allow_unconfigured = False
+        self.dut.checkers = [
+            RobotSuiteChecker(''),
+        ]
+        with open('tests/test_in/robot_version_5.xml') as xmlfile:
+            self.warnings.check(xmlfile.read())
+            count = self.warnings.return_count()
+        self.assertEqual(count, 8)
+
+    def test_disallow_unconfigured_fail(self):
+        self.dut.allow_unconfigured = False
+        self.dut.checkers = [
+            RobotSuiteChecker('Empty Flash Mlx Device Project Id'),
+        ]
+        with open('tests/test_in/robot_version_5.xml') as xmlfile:
+            with self.assertRaises(WarningsConfigError) as exc:
+                self.warnings.check(xmlfile.read())
+                self.warnings.return_count()
+        self.assertEqual(str(exc.exception), "1 test suites have been ignored due to incomplete configuration: ['Empty Flash Product Id']")
+
 
 if __name__ == '__main__':
     unittest.main()