Skip to content

Commit

Permalink
Add all passing chip-repl yaml tests to CI along with some fixes requ…
Browse files Browse the repository at this point in the history
…ired (#24597)

* Add all passing chip-repl yaml tests to CI

This will prevent future breakages like some recently added due to
incorrect naming.

* Address PR comment

* Address PR comment

* Remove test that is now failing

* Remove test that are no longer passing
  • Loading branch information
tehampson authored and pull[bot] committed Jan 11, 2024
1 parent 6bf3e0e commit 4256857
Show file tree
Hide file tree
Showing 7 changed files with 142 additions and 33 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ jobs:
--bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \
"
- name: Run Tests using chip-repl
timeout-minutes: 5
timeout-minutes: 10
run: |
./scripts/run_in_build_env.sh \
"./scripts/tests/run_test_suite.py \
Expand Down
2 changes: 2 additions & 0 deletions scripts/py_matter_yamltests/matter_yamltests/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,8 @@ def __init__(self, test: _TestStepWithPlaceholders, runtime_config_variable_stor
self.response = copy.deepcopy(test.response_with_placeholders)
self._update_placeholder_values(self.arguments)
self._update_placeholder_values(self.response)
self._test.node_id = self._config_variable_substitution(
self._test.node_id)
test.update_arguments(self.arguments)
test.update_response(self.response)

Expand Down
91 changes: 90 additions & 1 deletion scripts/tests/chiptest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,97 @@ def tests_with_command(chip_tool: str, is_manual: bool):
# parser/runner reaches parity with the code gen version.
def _hardcoded_python_yaml_tests():
currently_supported_yaml_tests = [
"TestClusterComplexTypes.yaml",
"Test_TC_ACL_1_1.yaml",
"Test_TC_ACL_2_1.yaml",
"Test_TC_BOOL_1_1.yaml",
"Test_TC_ACT_1_1.yaml",
"Test_TC_BIND_1_1.yaml",
"Test_TC_OPCREDS_1_2.yaml",
"Test_TC_BINFO_1_1.yaml",
"Test_TC_DESC_1_1.yaml",
"Test_TC_DLOG_1_1.yaml",
"Test_TC_FLW_1_1.yaml",
"Test_TC_FLW_2_1.yaml",
"Test_TC_FLABEL_1_1.yaml",
"Test_TC_CGEN_1_1.yaml",
"Test_TC_DGGEN_1_1.yaml",
"Test_TC_I_1_1.yaml",
"Test_TC_I_2_1.yaml",
"Test_TC_ILL_1_1.yaml",
"Test_TC_ILL_2_1.yaml",
"Test_TC_LVL_2_1.yaml",
"Test_TC_LVL_2_2.yaml",
"Test_TC_LCFG_1_1.yaml",
"Test_TC_LTIME_1_2.yaml",
"Test_TC_LOWPOWER_1_1.yaml",
"Test_TC_WAKEONLAN_1_5.yaml",
"Test_TC_AUDIOOUTPUT_1_8.yaml",
"Test_TC_TGTNAV_1_9.yaml",
"Test_TC_TGTNAV_8_2.yaml",
"Test_TC_APBSC_1_10.yaml",
"Test_TC_ALOGIN_1_12.yaml",
"Test_TC_KEYPADINPUT_3_2.yaml",
"Test_TC_KEYPADINPUT_3_3.yaml",
"Test_TC_APPLAUNCHER_3_5.yaml",
"Test_TC_APPLAUNCHER_3_6.yaml",
"Test_TC_MEDIAINPUT_3_10.yaml",
"Test_TC_MEDIAINPUT_3_11.yaml",
"Test_TC_CHANNEL_5_1.yaml",
"Test_TC_CONTENTLAUNCHER_10_1.yaml",
"Test_TC_OCC_1_1.yaml",
"Test_TC_PSCFG_1_1.yaml",
"Test_TC_PSCFG_2_1.yaml",
"Test_TC_RH_1_1.yaml",
"Test_TC_RH_2_1.yaml",
"Test_TC_SWTCH_2_1.yaml",
"Test_TC_TMP_1_1.yaml",
"Test_TC_TMP_2_1.yaml",
"Test_TC_TSUIC_1_1.yaml",
"Test_TC_TSUIC_2_1.yaml",
"Test_TC_DGTHREAD_2_2.yaml",
"Test_TC_DGTHREAD_2_4.yaml",
"Test_TC_ULABEL_1_1.yaml",
"Test_TC_ULABEL_2_1.yaml",
"Test_TC_ULABEL_2_2.yaml",
"Test_TC_ULABEL_2_3.yaml",
"Test_TC_ULABEL_2_4.yaml",
"Test_TC_DGWIFI_2_3.yaml",
"TV_TargetNavigatorCluster.yaml",
"TV_AudioOutputCluster.yaml",
"TV_ApplicationLauncherCluster.yaml",
"TV_KeypadInputCluster.yaml",
"TV_AccountLoginCluster.yaml",
"TV_WakeOnLanCluster.yaml",
"TV_ApplicationBasicCluster.yaml",
"TV_ChannelCluster.yaml",
"TV_LowPowerCluster.yaml",
"TV_ContentLauncherCluster.yaml",
"TV_MediaInputCluster.yaml",
"TestCluster.yaml",
"TestConstraints.yaml",
"TestSaveAs.yaml",
"TestConfigVariables.yaml",
"TestFabricRemovalWhileSubscribed.yaml",
"TestIdentifyCluster.yaml",
"TestSelfFabricRemoval.yaml",
"TestBinding.yaml",
"TestUserLabelClusterConstraints.yaml",
"TestFanControl.yaml",
"TestAccessControlConstraints.yaml",
"TestCommissioningWindow.yaml",
"TestSubscribe_OnOff.yaml",
"TestClusterComplexTypes.yaml",
"TestGroupsCluster.yaml",
"TestOperationalCredentialsCluster.yaml",
"Test_TC_AUDIOOUTPUT_7_1.yaml",
"Test_TC_BOOL_2_1.yaml",
"Test_TC_OO_2_1.yaml",
"Test_TC_TGTNAV_8_1.yaml",
"Test_TC_WNCV_2_3.yaml",
"Test_TC_WNCV_4_3.yaml",
"Test_TC_WNCV_4_4.yaml",
"DL_Schedules.yaml",
"DL_UsersAndCredentials.yaml",
]

for name in currently_supported_yaml_tests:
Expand Down
49 changes: 27 additions & 22 deletions scripts/tests/chiptest/yamltest_with_chip_repl_tester.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import glob
import os
import tempfile
import traceback

# isort: off

Expand Down Expand Up @@ -88,28 +89,32 @@ def main(setup_code, yaml_path, node_id):
dev_ctrl = ca_list[0].adminList[0].NewController()
dev_ctrl.CommissionWithCode(setup_code, node_id)

# Creating Cluster definition.
cluster_xml_filenames = glob.glob(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml', recursive=False)
cluster_xml_filenames.sort(key=functools.cmp_to_key(_sort_with_global_attribute_first))
sources = [ParseSource(source=name) for name in cluster_xml_filenames]
clusters_definitions = SpecDefinitions(sources)

# Parsing YAML test and setting up chip-repl yamltests runner.
yaml = TestParser(yaml_path, None, clusters_definitions)
runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl)

# Executing and validating test
for test_step in yaml.tests:
test_action = runner.encode(test_step)
# TODO if test_action is None we should see if it is a pseudo cluster.
if test_action is not None:
response = runner.execute(test_action)
decoded_response = runner.decode(response)
post_processing_result = test_step.post_process_response(decoded_response)
if not post_processing_result.is_success():
exit(-2)
else:
exit(-2)
try:
# Creating Cluster definition.
cluster_xml_filenames = glob.glob(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml', recursive=False)
cluster_xml_filenames.sort(key=functools.cmp_to_key(_sort_with_global_attribute_first))
sources = [ParseSource(source=name) for name in cluster_xml_filenames]
clusters_definitions = SpecDefinitions(sources)

# Parsing YAML test and setting up chip-repl yamltests runner.
yaml = TestParser(yaml_path, None, clusters_definitions)
runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl)

# Executing and validating test
for test_step in yaml.tests:
test_action = runner.encode(test_step)
# TODO if test_action is None we should see if it is a pseudo cluster.
if test_action is not None:
response = runner.execute(test_action)
decoded_response = runner.decode(response)
post_processing_result = test_step.post_process_response(decoded_response)
if not post_processing_result.is_success():
raise Exception(f'Test step failed {test_step.label}')
else:
raise Exception(f'Failed to encode test step {test_step.label}')
except Exception:
print(traceback.format_exc())
exit(-2)

runner.shutdown()
# Tearing down chip stack. If not done in the correct order test will fail.
Expand Down
1 change: 1 addition & 0 deletions scripts/tests/run_test_suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o
if context.obj.dry_run:
logging.info("Would run test %s:" % test.name)

logging.info('%-20s - Starting test' % (test.name))
test.Run(runner, apps_register, paths, pics_file, test_timeout_seconds, context.obj.dry_run)
test_end = time.monotonic()
logging.info('%-20s - Completed in %0.2f seconds' %
Expand Down
21 changes: 14 additions & 7 deletions src/controller/python/chip/yaml/data_model_lookup.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
import chip.clusters as Clusters


def _case_insensitive_getattr(object, attr_name, default):
for attr in dir(object):
if attr.lower() == attr_name.lower():
return getattr(object, attr)
return default


class DataModelLookup(ABC):
@abstractmethod
def get_cluster(self, cluster: str):
Expand All @@ -41,27 +48,27 @@ def get_event(self, cluster: str, event: str):
class PreDefinedDataModelLookup(DataModelLookup):
def get_cluster(self, cluster: str):
try:
return getattr(Clusters, cluster, None)
return _case_insensitive_getattr(Clusters, cluster, None)
except AttributeError:
return None

def get_command(self, cluster: str, command: str):
try:
commands = getattr(Clusters, cluster, None).Commands
return getattr(commands, command, None)
commands = _case_insensitive_getattr(Clusters, cluster, None).Commands
return _case_insensitive_getattr(commands, command, None)
except AttributeError:
return None

def get_attribute(self, cluster: str, attribute: str):
try:
attributes = getattr(Clusters, cluster, None).Attributes
return getattr(attributes, attribute, None)
attributes = _case_insensitive_getattr(Clusters, cluster, None).Attributes
return _case_insensitive_getattr(attributes, attribute, None)
except AttributeError:
return None

def get_event(self, cluster: str, event: str):
try:
events = getattr(Clusters, cluster, None).Events
return getattr(events, event, None)
events = _case_insensitive_getattr(Clusters, cluster, None).Events
return _case_insensitive_getattr(events, event, None)
except AttributeError:
return None
9 changes: 7 additions & 2 deletions src/controller/python/chip/yaml/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -644,8 +644,13 @@ def decode(self, result: _ActionResult):
if attribute is None:
# When we cannot find the attribute it is because it is a global attribute like
# FeatureMap. Fortunately for these types we can get away with using
# 'response.value' directly for the time being.
decoded_response['value'] = response.value
# 'response.value' directly if it is a list and mapping to int if not a list.
if isinstance(response.value, list):
decoded_response['value'] = response.value
else:
decoded_response['value'] = Converter.from_data_model_to_test_definition(
self._test_spec_definition, cluster_name, int, response.value)

else:
decoded_response['value'] = Converter.from_data_model_to_test_definition(
self._test_spec_definition, cluster_name, attribute.definition, response.value)
Expand Down

0 comments on commit 4256857

Please sign in to comment.