Start of python YAML test parsers and executer (#23533)

* Start of python YAML test parsers and executer

Tested inside chip-repl with the following commands (with all-cluster
app running on separate terminal already commissioned):
  import chip.yaml
  foo = chip.yaml.Yaml.YamlTestParser("src/app/tests/suites/TestCluster.yaml")
  foo.ExecuteTests(devCtrl)

Co-authored-by: Jerry Johns <[email protected]>

* Address PR comments

* Reduce line length to 100

* Address PR comments

* Address PR comments

* Fix minor nit docstring issue

* Address more PR comments, rename module names to adhere to guidelines

* Fix style

* Apparently cluster is not always set in 'config'

* Fix up chip-repl CI failure

Co-authored-by: Jerry Johns <[email protected]>

src/controller/python/BUILD.gn

@@ -226,6 +226,11 @@ chip_python_wheel_action("chip-core") {
         "chip/storage/__init__.py",
         "chip/utils/CommissioningBuildingBlocks.py",
         "chip/utils/__init__.py",
+        "chip/yaml/__init__.py",
+        "chip/yaml/data_model_lookup.py",
+        "chip/yaml/errors.py",
+        "chip/yaml/format_converter.py",
+        "chip/yaml/parser.py",
       ]
 
       if (chip_controller) {
@@ -269,6 +274,7 @@ chip_python_wheel_action("chip-core") {
     "chip.internal",
     "chip.interaction_model",
     "chip.logging",
+    "chip.yaml",
     "chip.native",
     "chip.clusters",
     "chip.setup_payload",

src/controller/python/chip/yaml/__init__.py

@@ -0,0 +1,23 @@
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+#
+#    @file
+#      Provides Python APIs for Matter.
+
+"""Provides yaml parser Python APIs for Matter."""
+from . import parser

src/controller/python/chip/yaml/data_model_lookup.py

@@ -0,0 +1,55 @@
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from abc import ABC, abstractmethod
+import chip.clusters as Clusters
+
+
+class DataModelLookup(ABC):
+    @abstractmethod
+    def get_cluster(self, cluster: str):
+        pass
+
+    @abstractmethod
+    def get_command(self, cluster: str, command: str):
+        pass
+
+    @abstractmethod
+    def get_attribute(self, cluster: str, attribute: str):
+        pass
+
+
+class PreDefinedDataModelLookup(DataModelLookup):
+    def get_cluster(self, cluster: str):
+        try:
+            return getattr(Clusters, cluster, None)
+        except AttributeError:
+            return None
+
+    def get_command(self, cluster: str, command: str):
+        try:
+            commands = getattr(Clusters, cluster, None).Commands
+            return getattr(commands, command, None)
+        except AttributeError:
+            return None
+
+    def get_attribute(self, cluster: str, attribute: str):
+        try:
+            attributes = getattr(Clusters, cluster, None).Attributes
+            return getattr(attributes, attribute, None)
+        except AttributeError:
+            return None

src/controller/python/chip/yaml/errors.py

@@ -0,0 +1,30 @@
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+class ParsingError(ValueError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class UnexpectedParsingError(ParsingError):
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class ValidationError(Exception):
+    def __init__(self, message):
+        super().__init__(message)

src/controller/python/chip/yaml/format_converter.py

@@ -0,0 +1,126 @@
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+import typing
+from chip.clusters.Types import Nullable, NullValue
+from chip.tlv import uint, float32
+import enum
+from chip.yaml.errors import ValidationError
+
+
+_HEX_PREFIX = 'hex:'
+
+
+def convert_name_value_pair_to_dict(arg_values):
+    ''' Fix yaml command arguments.
+
+    For some reason, instead of treating the entire data payload of a
+    command as a singular struct, the top-level args are specified as 'name'
+    and 'value' pairs, while the payload of each argument is itself
+    correctly encapsulated. This fixes up this oddity to create a new
+    key/value pair with the key being the value of the 'name' field, and
+    the value being 'value' field.
+    '''
+    ret_value = {}
+
+    for item in arg_values:
+        ret_value[item['name']] = item['value']
+
+    return ret_value
+
+
+def convert_yaml_type(field_value, field_type, use_from_dict=False):
+    ''' Converts yaml value to expected type.
+
+    The YAML representation when converted to a Python dictionary does not
+    quite line up in terms of type (see each of the specific if branches
+    below for the rationale for the necessary fix-ups). This function does
+    a fix-up given a field value (as present in the YAML) and its matching
+    cluster object type and returns it.
+    '''
+    origin = typing.get_origin(field_type)
+
+    if field_value is None:
+        field_value = NullValue
+
+    if (origin == typing.Union or origin == typing.Optional or origin == Nullable):
+        underlying_field_type = None
+
+        if field_value is NullValue:
+            for t in typing.get_args(field_type):
+                if t == Nullable:
+                    return field_value
+
+        for t in typing.get_args(field_type):
+            # Comparison below explicitly not using 'isinstance' as that doesn't do what we want.
+            if t != Nullable and t != type(None):
+                underlying_field_type = t
+                break
+
+        if (underlying_field_type is None):
+            raise ValueError(f"Can't find the underling type for {field_type}")
+
+        field_type = underlying_field_type
+
+    # Dictionary represents a data model struct.
+    if (type(field_value) is dict):
+        return_field_value = {}
+        field_descriptors = field_type.descriptor
+        for item in field_value:
+            try:
+                # We search for a matching item in the list of field descriptors
+                # for this struct and ensure we can find a field with a matching
+                # label.
+                field_descriptor = next(
+                    x for x in field_descriptors.Fields if x.Label.lower() ==
+                    item.lower())
+            except StopIteration as exc:
+                raise ValidationError(
+                    f'Did not find field "{item}" in {str(field_type)}') from None
+
+            return_field_value[field_descriptor.Label] = convert_yaml_type(
+                field_value[item], field_descriptor.Type, use_from_dict)
+        if use_from_dict:
+            return field_type.FromDict(return_field_value)
+        return return_field_value
+    elif(type(field_value) is float):
+        return float32(field_value)
+    # list represents a data model list
+    elif(type(field_value) is list):
+        list_element_type = typing.get_args(field_type)[0]
+
+        # The field type passed in is the type of the list element and not list[T].
+        for idx, item in enumerate(field_value):
+            field_value[idx] = convert_yaml_type(item, list_element_type, use_from_dict)
+        return field_value
+    # YAML conversion treats all numbers as ints. Convert to a uint type if the schema
+    # type indicates so.
+    elif (field_type == uint):
+        # Longer number are stored as strings. Need to make this conversion first.
+        value = int(field_value)
+        return field_type(value)
+    # YAML treats enums as ints. Convert to the typed enum class.
+    elif (issubclass(field_type, enum.Enum)):
+        return field_type(field_value)
+    # YAML treats bytes as strings. Convert to a byte string.
+    elif (field_type == bytes and type(field_value) != bytes):
+        if isinstance(field_value, str) and field_value.startswith(_HEX_PREFIX):
+            return bytes.fromhex(field_value[len(_HEX_PREFIX):])
+        return str.encode(field_value)
+    # By default, just return the field_value casted to field_type.
+    else:
+        return field_type(field_value)