Decode TLV payload data and present it in a human readable format

BUILD.gn

@@ -54,6 +54,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         "${chip_root}/src/credentials/tests:fuzz-chip-cert",
         "${chip_root}/src/lib/core/tests:fuzz-tlv-reader",
         "${chip_root}/src/lib/dnssd/minimal_mdns/tests:fuzz-minmdns-packet-parsing",
+        "${chip_root}/src/lib/format/tests:fuzz-payload-decoder",
       ]
     }
   }

build/chip/chip_codegen.gni

@@ -53,9 +53,19 @@ template("_chip_build_time_codegen") {
       rebase_path(target_gen_dir, root_build_dir),
       "--expected-outputs",
       rebase_path(_expected_outputs, root_build_dir),
-      rebase_path(_idl_file, root_build_dir),
     ]
 
+    if (defined(invoker.options)) {
+      foreach(option, invoker.options) {
+        args += [
+          "--option",
+          option,
+        ]
+      }
+    }
+
+    args += [ rebase_path(_idl_file, root_build_dir) ]
+
     inputs = [
       _idl_file,
       _expected_outputs,
@@ -313,12 +323,15 @@ template("chip_codegen") {
                                "generator",
                                "input",
                                "outputs",
+                               "options",
                                "public_configs",
                              ])
     }
   } else {
     _name = target_name
 
+    not_needed(invoker, [ "options" ])
+
     # This constructs a path like:
     #  FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/)
     #  USING "cpp-app" for generator:

examples/chip-tool/args.gni

@@ -27,3 +27,6 @@ matter_enable_tracing_support = true
 
 # Perfetto requires C++17
 cpp_standard = "gnu++17"
+
+matter_log_json_payload_hex = true
+matter_log_json_payload_decode_full = true

scripts/pregenerate/__init__.py

@@ -20,7 +20,8 @@
 from typing import Iterator, List, Optional
 
 from .types import IdlFileType, InputIdlFile
-from .using_codegen import CodegenCppAppPregenerator, CodegenJavaClassPregenerator, CodegenJavaJNIPregenerator
+from .using_codegen import (CodegenCppAppPregenerator, CodegenCppClustersTLVMetaPregenerator,
+                            CodegenCppProtocolsTLVMetaPregenerator, CodegenJavaClassPregenerator, CodegenJavaJNIPregenerator)
 from .using_zap import ZapApplicationPregenerator
 
 
@@ -94,6 +95,8 @@ def FindPregenerationTargets(sdk_root: str, external_roots: Optional[List[str]],
         CodegenJavaJNIPregenerator(sdk_root),
         CodegenJavaClassPregenerator(sdk_root),
         CodegenCppAppPregenerator(sdk_root),
+        CodegenCppClustersTLVMetaPregenerator(sdk_root),
+        CodegenCppProtocolsTLVMetaPregenerator(sdk_root),
 
         # ZAP codegen
         ZapApplicationPregenerator(sdk_root),

scripts/pregenerate/using_codegen.py

@@ -27,11 +27,12 @@
 class CodegenTarget:
     """A target that uses `scripts/codegen.py` to generate files."""
 
-    def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str, runner):
+    def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str, runner, options=[]):
         self.idl = idl
         self.generator = generator
         self.sdk_root = sdk_root
         self.runner = runner
+        self.options = options
 
         if idl.file_type != IdlFileType.MATTER:
             raise Exception(
@@ -51,8 +52,12 @@ def Generate(self, output_root: str):
             '--log-level', 'fatal',
             '--generator', self.generator,
             '--output-dir', output_dir,
-            self.idl.full_path
         ]
+        for option in self.options:
+            cmd.append("--option")
+            cmd.append(option)
+
+        cmd.append(self.idl.full_path)
 
         logging.debug(f"Executing {cmd}")
         self.runner.run(cmd)
@@ -97,6 +102,9 @@ def Accept(self, idl: InputIdlFile):
         if idl.file_type != IdlFileType.MATTER:
             return False
 
+        if '/lib/format/' in idl.relative_path:
+            return False
+
         # we should not be checked for these, but verify just in case
         if '/tests/' in idl.relative_path:
             return False
@@ -105,3 +113,29 @@ def Accept(self, idl: InputIdlFile):
 
     def CreateTarget(self, idl: InputIdlFile, runner):
         return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app", runner=runner)
+
+
+class CodegenCppProtocolsTLVMetaPregenerator:
+    """Pregeneration logic for "cpp-app" codegen.py outputs"""
+
+    def __init__(self, sdk_root):
+        self.sdk_root = sdk_root
+
+    def Accept(self, idl: InputIdlFile):
+        return (idl.file_type == IdlFileType.MATTER) and idl.relative_path.endswith('/protocol_messages.matter')
+
+    def CreateTarget(self, idl: InputIdlFile, runner):
+        return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-tlvmeta", options=["table_name:protocols_meta"], runner=runner)
+
+
+class CodegenCppClustersTLVMetaPregenerator:
+    """Pregeneration logic for "cpp-app" codegen.py outputs"""
+
+    def __init__(self, sdk_root):
+        self.sdk_root = sdk_root
+
+    def Accept(self, idl: InputIdlFile):
+        return (idl.file_type == IdlFileType.MATTER) and idl.relative_path.endswith('/controller-clusters.matter')
+
+    def CreateTarget(self, idl: InputIdlFile, runner):
+        return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-tlvmeta", options=["table_name:clusters_meta"], runner=runner)

scripts/py_matter_idl/files.gni

@@ -3,24 +3,38 @@ import("//build_overrides/chip.gni")
 
 # Templates used for generation
 matter_idl_generator_templates = [
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/CallbackStubSource.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_cpp.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_h.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/CHIPCallbackTypes.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersCpp.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersRead.jinja",
-  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ClusterWriteMapping.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/CHIPGlobalCallbacks_cpp.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/CHIPReadCallbacks_h.jinja",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ClusterIDMapping.jinja",
-  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/CallbackStubSource.jinja",
-  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ClusterReadMapping.jinja",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ClusterWriteMapping.jinja",
 ]
 
 matter_idl_generator_sources = [
   "${chip_root}/scripts/py_matter_idl/matter_idl/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/__init__.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/filters.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/__init__.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/registry.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/types.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/lint/__init__.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/lint/lint_rules_parser.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/lint/types.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/matter_idl_parser.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/matter_idl_types.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/test_generators.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/test_matter_idl_parser.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/test_xml_parser.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/xml_parser.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/__init__.py",

scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_cpp.jinja

@@ -0,0 +1,44 @@
+#include <tlv/meta/{{table_name}}.h>
+
+namespace chip {
+namespace TLVMeta {
+namespace {
+
+using namespace chip::FlatTree;
+using namespace chip::TLV;
+
+{%- for table in sub_tables %}
+
+const Entry<ItemInfo> _{{table.full_name}}[] = {
+  {%-  for entry in table.entries %}
+  { { {{entry.code}}, "{{entry.name}}", ItemType::{{entry.item_type}} }, {{entry.reference | indexInTable(sub_tables)}} }, // {{entry.real_type}}
+  {%- endfor %}
+};
+{%- endfor %}
+
+const Entry<ItemInfo> _all_clusters[] = {
+{%- for cluster in clusters | sort(attribute='code') %}
+  { { ClusterTag({{"0x%02X" | format(cluster.code)}}), "{{cluster.name}}", ItemType::kDefault }, {{cluster.name | indexInTable(sub_tables)}} },
+{%- endfor %}
+
+};
+
+// For any non-structure list like u64[] or similar.
+const Entry<ItemInfo> _primitive_type_list[] = {
+  { { AnonymousTag(), "[]", ItemType::kDefault }, kInvalidNodeIndex },
+};
+
+} // namespace
+
+#define _ENTRY(n) { sizeof(n) / sizeof(n[0]), n}
+
+const std::array<const Node<ItemInfo>, {{ sub_tables | length }} + 2> {{table_name}} = { {
+  _ENTRY(_all_clusters), // 0
+  _ENTRY(_primitive_type_list), // 1
+{%- for table in sub_tables %}
+  _ENTRY(_{{table.full_name}}), // {{loop.index + 1}}
+{%- endfor %}
+} };
+
+} // namespace TLVMeta
+} // namespace chip

scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_h.jinja

@@ -0,0 +1,12 @@
+#include <lib/format/tlv_meta.h>
+#include <lib/format/FlatTree.h>
+
+#include <array>
+
+namespace chip {
+namespace TLVMeta {
+
+extern const std::array<const FlatTree::Node<ItemInfo>, {{ sub_tables | length }} + 2> {{table_name}};
+
+} // namespace TLVMeta
+} // namespace chip