From 986176996290fe71b84c02bf6c75f73b8627bac0 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Tue, 24 Jan 2023 16:02:31 -0500 Subject: [PATCH] Add all passing chip-repl yaml tests to CI along with some fixes required (#24597) * Add all passing chip-repl yaml tests to CI This will prevent future breakages like some recently added due to incorrect naming. * Address PR comment * Address PR comment * Remove test that is now failing * Remove test that are no longer passing --- .github/workflows/tests.yaml | 2 +- .../matter_yamltests/parser.py | 2 + scripts/tests/chiptest/__init__.py | 91 ++++++++++++++++++- .../yamltest_with_chip_repl_tester.py | 49 +++++----- scripts/tests/run_test_suite.py | 1 + .../python/chip/yaml/data_model_lookup.py | 21 +++-- src/controller/python/chip/yaml/runner.py | 9 +- 7 files changed, 142 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e002203160cd00..b6b698bb2da748 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -214,7 +214,7 @@ jobs: --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \ " - name: Run Tests using chip-repl - timeout-minutes: 5 + timeout-minutes: 10 run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser.py b/scripts/py_matter_yamltests/matter_yamltests/parser.py index 9e870019241eea..f7b9ec9b0729a2 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/parser.py +++ b/scripts/py_matter_yamltests/matter_yamltests/parser.py @@ -400,6 +400,8 @@ def __init__(self, test: _TestStepWithPlaceholders, runtime_config_variable_stor self.response = copy.deepcopy(test.response_with_placeholders) self._update_placeholder_values(self.arguments) self._update_placeholder_values(self.response) + self._test.node_id = self._config_variable_substitution( + self._test.node_id) test.update_arguments(self.arguments) test.update_response(self.response) diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index a8f7450db437ea..e3cf7fa7594676 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -77,8 +77,97 @@ def tests_with_command(chip_tool: str, is_manual: bool): # parser/runner reaches parity with the code gen version. def _hardcoded_python_yaml_tests(): currently_supported_yaml_tests = [ - "TestClusterComplexTypes.yaml", + "Test_TC_ACL_1_1.yaml", + "Test_TC_ACL_2_1.yaml", + "Test_TC_BOOL_1_1.yaml", + "Test_TC_ACT_1_1.yaml", + "Test_TC_BIND_1_1.yaml", + "Test_TC_OPCREDS_1_2.yaml", + "Test_TC_BINFO_1_1.yaml", + "Test_TC_DESC_1_1.yaml", + "Test_TC_DLOG_1_1.yaml", + "Test_TC_FLW_1_1.yaml", + "Test_TC_FLW_2_1.yaml", + "Test_TC_FLABEL_1_1.yaml", + "Test_TC_CGEN_1_1.yaml", + "Test_TC_DGGEN_1_1.yaml", + "Test_TC_I_1_1.yaml", + "Test_TC_I_2_1.yaml", + "Test_TC_ILL_1_1.yaml", + "Test_TC_ILL_2_1.yaml", + "Test_TC_LVL_2_1.yaml", + "Test_TC_LVL_2_2.yaml", + "Test_TC_LCFG_1_1.yaml", + "Test_TC_LTIME_1_2.yaml", + "Test_TC_LOWPOWER_1_1.yaml", + "Test_TC_WAKEONLAN_1_5.yaml", + "Test_TC_AUDIOOUTPUT_1_8.yaml", + "Test_TC_TGTNAV_1_9.yaml", + "Test_TC_TGTNAV_8_2.yaml", + "Test_TC_APBSC_1_10.yaml", + "Test_TC_ALOGIN_1_12.yaml", + "Test_TC_KEYPADINPUT_3_2.yaml", + "Test_TC_KEYPADINPUT_3_3.yaml", + "Test_TC_APPLAUNCHER_3_5.yaml", + "Test_TC_APPLAUNCHER_3_6.yaml", + "Test_TC_MEDIAINPUT_3_10.yaml", + "Test_TC_MEDIAINPUT_3_11.yaml", + "Test_TC_CHANNEL_5_1.yaml", + "Test_TC_CONTENTLAUNCHER_10_1.yaml", + "Test_TC_OCC_1_1.yaml", + "Test_TC_PSCFG_1_1.yaml", + "Test_TC_PSCFG_2_1.yaml", + "Test_TC_RH_1_1.yaml", + "Test_TC_RH_2_1.yaml", + "Test_TC_SWTCH_2_1.yaml", + "Test_TC_TMP_1_1.yaml", + "Test_TC_TMP_2_1.yaml", + "Test_TC_TSUIC_1_1.yaml", + "Test_TC_TSUIC_2_1.yaml", + "Test_TC_DGTHREAD_2_2.yaml", + "Test_TC_DGTHREAD_2_4.yaml", + "Test_TC_ULABEL_1_1.yaml", + "Test_TC_ULABEL_2_1.yaml", + "Test_TC_ULABEL_2_2.yaml", + "Test_TC_ULABEL_2_3.yaml", + "Test_TC_ULABEL_2_4.yaml", + "Test_TC_DGWIFI_2_3.yaml", + "TV_TargetNavigatorCluster.yaml", + "TV_AudioOutputCluster.yaml", + "TV_ApplicationLauncherCluster.yaml", + "TV_KeypadInputCluster.yaml", + "TV_AccountLoginCluster.yaml", + "TV_WakeOnLanCluster.yaml", + "TV_ApplicationBasicCluster.yaml", + "TV_ChannelCluster.yaml", + "TV_LowPowerCluster.yaml", + "TV_ContentLauncherCluster.yaml", + "TV_MediaInputCluster.yaml", + "TestCluster.yaml", "TestConstraints.yaml", + "TestSaveAs.yaml", + "TestConfigVariables.yaml", + "TestFabricRemovalWhileSubscribed.yaml", + "TestIdentifyCluster.yaml", + "TestSelfFabricRemoval.yaml", + "TestBinding.yaml", + "TestUserLabelClusterConstraints.yaml", + "TestFanControl.yaml", + "TestAccessControlConstraints.yaml", + "TestCommissioningWindow.yaml", + "TestSubscribe_OnOff.yaml", + "TestClusterComplexTypes.yaml", + "TestGroupsCluster.yaml", + "TestOperationalCredentialsCluster.yaml", + "Test_TC_AUDIOOUTPUT_7_1.yaml", + "Test_TC_BOOL_2_1.yaml", + "Test_TC_OO_2_1.yaml", + "Test_TC_TGTNAV_8_1.yaml", + "Test_TC_WNCV_2_3.yaml", + "Test_TC_WNCV_4_3.yaml", + "Test_TC_WNCV_4_4.yaml", + "DL_Schedules.yaml", + "DL_UsersAndCredentials.yaml", ] for name in currently_supported_yaml_tests: diff --git a/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py b/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py index ce1db0330d18d8..2cf03caf9a858b 100644 --- a/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py +++ b/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py @@ -18,6 +18,7 @@ import glob import os import tempfile +import traceback # isort: off @@ -88,28 +89,32 @@ def main(setup_code, yaml_path, node_id): dev_ctrl = ca_list[0].adminList[0].NewController() dev_ctrl.CommissionWithCode(setup_code, node_id) - # Creating Cluster definition. - cluster_xml_filenames = glob.glob(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml', recursive=False) - cluster_xml_filenames.sort(key=functools.cmp_to_key(_sort_with_global_attribute_first)) - sources = [ParseSource(source=name) for name in cluster_xml_filenames] - clusters_definitions = SpecDefinitions(sources) - - # Parsing YAML test and setting up chip-repl yamltests runner. - yaml = TestParser(yaml_path, None, clusters_definitions) - runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl) - - # Executing and validating test - for test_step in yaml.tests: - test_action = runner.encode(test_step) - # TODO if test_action is None we should see if it is a pseudo cluster. - if test_action is not None: - response = runner.execute(test_action) - decoded_response = runner.decode(response) - post_processing_result = test_step.post_process_response(decoded_response) - if not post_processing_result.is_success(): - exit(-2) - else: - exit(-2) + try: + # Creating Cluster definition. + cluster_xml_filenames = glob.glob(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml', recursive=False) + cluster_xml_filenames.sort(key=functools.cmp_to_key(_sort_with_global_attribute_first)) + sources = [ParseSource(source=name) for name in cluster_xml_filenames] + clusters_definitions = SpecDefinitions(sources) + + # Parsing YAML test and setting up chip-repl yamltests runner. + yaml = TestParser(yaml_path, None, clusters_definitions) + runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl) + + # Executing and validating test + for test_step in yaml.tests: + test_action = runner.encode(test_step) + # TODO if test_action is None we should see if it is a pseudo cluster. + if test_action is not None: + response = runner.execute(test_action) + decoded_response = runner.decode(response) + post_processing_result = test_step.post_process_response(decoded_response) + if not post_processing_result.is_success(): + raise Exception(f'Test step failed {test_step.label}') + else: + raise Exception(f'Failed to encode test step {test_step.label}') + except Exception: + print(traceback.format_exc()) + exit(-2) runner.shutdown() # Tearing down chip stack. If not done in the correct order test will fail. diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 878327ada2d109..1afca2600e3a6d 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -262,6 +262,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o if context.obj.dry_run: logging.info("Would run test %s:" % test.name) + logging.info('%-20s - Starting test' % (test.name)) test.Run(runner, apps_register, paths, pics_file, test_timeout_seconds, context.obj.dry_run) test_end = time.monotonic() logging.info('%-20s - Completed in %0.2f seconds' % diff --git a/src/controller/python/chip/yaml/data_model_lookup.py b/src/controller/python/chip/yaml/data_model_lookup.py index afabeb15278ace..b4c993342f2687 100644 --- a/src/controller/python/chip/yaml/data_model_lookup.py +++ b/src/controller/python/chip/yaml/data_model_lookup.py @@ -20,6 +20,13 @@ import chip.clusters as Clusters +def _case_insensitive_getattr(object, attr_name, default): + for attr in dir(object): + if attr.lower() == attr_name.lower(): + return getattr(object, attr) + return default + + class DataModelLookup(ABC): @abstractmethod def get_cluster(self, cluster: str): @@ -41,27 +48,27 @@ def get_event(self, cluster: str, event: str): class PreDefinedDataModelLookup(DataModelLookup): def get_cluster(self, cluster: str): try: - return getattr(Clusters, cluster, None) + return _case_insensitive_getattr(Clusters, cluster, None) except AttributeError: return None def get_command(self, cluster: str, command: str): try: - commands = getattr(Clusters, cluster, None).Commands - return getattr(commands, command, None) + commands = _case_insensitive_getattr(Clusters, cluster, None).Commands + return _case_insensitive_getattr(commands, command, None) except AttributeError: return None def get_attribute(self, cluster: str, attribute: str): try: - attributes = getattr(Clusters, cluster, None).Attributes - return getattr(attributes, attribute, None) + attributes = _case_insensitive_getattr(Clusters, cluster, None).Attributes + return _case_insensitive_getattr(attributes, attribute, None) except AttributeError: return None def get_event(self, cluster: str, event: str): try: - events = getattr(Clusters, cluster, None).Events - return getattr(events, event, None) + events = _case_insensitive_getattr(Clusters, cluster, None).Events + return _case_insensitive_getattr(events, event, None) except AttributeError: return None diff --git a/src/controller/python/chip/yaml/runner.py b/src/controller/python/chip/yaml/runner.py index f56f08d740dbf1..b011b8817d692c 100644 --- a/src/controller/python/chip/yaml/runner.py +++ b/src/controller/python/chip/yaml/runner.py @@ -644,8 +644,13 @@ def decode(self, result: _ActionResult): if attribute is None: # When we cannot find the attribute it is because it is a global attribute like # FeatureMap. Fortunately for these types we can get away with using - # 'response.value' directly for the time being. - decoded_response['value'] = response.value + # 'response.value' directly if it is a list and mapping to int if not a list. + if isinstance(response.value, list): + decoded_response['value'] = response.value + else: + decoded_response['value'] = Converter.from_data_model_to_test_definition( + self._test_spec_definition, cluster_name, int, response.value) + else: decoded_response['value'] = Converter.from_data_model_to_test_definition( self._test_spec_definition, cluster_name, attribute.definition, response.value)