Add a way to have tests run only in the python-chip-tool driver. (#28526

)

Switches TestGroupKeyManagementCluster to the new setup and enables the
disabled parts of it.

Fixes #28396

examples/chip-tool/templates/tests/commands.zapt

@@ -36,6 +36,18 @@ public:
     }
 };
 
+class PythonRunnerOnlyTestList : public Command
+{
+public:
+    PythonRunnerOnlyTestList() : Command("list-python-runner-only") {};
+    CHIP_ERROR Run() override
+    {
+        {{#chip_tests "../../../../src/app/tests/suites/pythonRunnerOnlyTests.json" includeAllClusters=true}}printf("{{filename}}\n");{{/chip_tests}}
+
+        return CHIP_NO_ERROR;
+    }
+};
+
 {{>test_cluster tests="../../../../src/app/tests/suites/ciTests.json" credsIssuerConfigArg=true needsWaitDuration=true}}
 {{>test_cluster tests="../../../../src/app/tests/suites/manualTests.json" credsIssuerConfigArg=true needsWaitDuration=true}}
 
@@ -49,6 +61,7 @@ void registerCommandsTests(Commands & commands, CredentialIssuerCommands * creds
 #if CONFIG_ENABLE_YAML_TESTS
         make_unique<TestList>(),
         make_unique<ManualTestList>(),
+        make_unique<PythonRunnerOnlyTestList>(),
       {{#chip_tests "../../../../src/app/tests/suites/ciTests.json" includeAllClusters=true}}
         make_unique<{{filename}}Suite>(credsIssuerConfig),
       {{/chip_tests}}

scripts/setup/zap.json

@@ -8,13 +8,13 @@
                 "mac-amd64",
                 "windows-amd64"
             ],
-            "tags": ["version:[email protected].01-nightly.1"]
+            "tags": ["version:[email protected].04-nightly.1"]
         },
         {
             "_comment": "Always get the amd64 version on mac until usable arm64 zap build is available",
             "path": "fuchsia/third_party/zap/mac-amd64",
             "platforms": ["mac-arm64"],
-            "tags": ["version:[email protected].01-nightly.1"]
+            "tags": ["version:[email protected].04-nightly.1"]
         }
     ]
 }

scripts/setup/zap.version

@@ -1 +1 @@
-v2023.08.01-nightly
+v2023.08.04-nightly

scripts/tests/chiptest/__init__.py

@@ -129,27 +129,48 @@ def _GetSlowTests() -> Set[str]:
 
 def _GetInDevelopmentTests() -> Set[str]:
     """Tests that fail in YAML for some reason."""
+    return {
+        "Test_TC_TIMESYNC_1_1.yaml",         # Time sync SDK is not yet ready
+        "Test_TC_TIMESYNC_2_3.yaml",         # Time sync SDK is not yet ready
+        "Test_TC_PSCFG_1_1.yaml",  # Power source configuration cluster is deprecated and removed from all-clusters
+        "Test_TC_PSCFG_2_1.yaml",  # Power source configuration cluster is deprecated and removed from all-clusters
+        "Test_TC_PSCFG_2_2.yaml",  # Power source configuration cluster is deprecated and removed from all-clusters
+        "Test_TC_SMOKECO_2_6.yaml",          # chip-repl does not support local timeout (07/20/2023) and test assumes
+                                             # TestEventTriggersEnabled is true, which it's not in CI. Also, test
+                                             # has wrong key for eventNumber: because using the right key leads to
+                                             # codegen that does not compile.
+        "Test_TC_SMOKECO_2_2.yaml",          # chip-repl does not support local timeout (07/20/2023) and test assumes
+                                             # TestEventTriggersEnabled is true, which it's not in CI. Also, test
+                                             # has wrong key for eventNumber: because using the right key leads to
+                                             # codegen that does not compile.
+        "Test_TC_SMOKECO_2_3.yaml",          # chip-repl does not support local timeout (07/20/2023) and test assumes
+                                             # TestEventTriggersEnabled is true, which it's not in CI. Also, test
+                                             # has wrong key for eventNumber: because using the right key leads to
+                                             # codegen that does not compile.
+        "Test_TC_SMOKECO_2_4.yaml",          # chip-repl does not support local timeout (07/20/2023) and test assumes
+                                             # TestEventTriggersEnabled is true, which it's not in CI. Also, test
+                                             # has wrong key for eventNumber: because using the right key leads to
+                                             # codegen that does not compile.
+        "Test_TC_SMOKECO_2_5.yaml",          # chip-repl does not support local timeout (07/20/2023) and test uses unknown
+                                             # keepSubscriptions key in the YAML. Also, test has wrong key for eventNumber:
+                                             # because using the right key leads to codegen that does not compile.
+    }
+
+
+def _GetChipReplUnsupportedTests() -> Set[str]:
+    """Tests that fail in chip-repl for some reason"""
     return {
         "Test_AddNewFabricFromExistingFabric.yaml",     # chip-repl does not support GetCommissionerRootCertificate and IssueNocChain command
         "TestEqualities.yaml",              # chip-repl does not support pseudo-cluster commands that return a value
         "TestExampleCluster.yaml",          # chip-repl does not load custom pseudo clusters
-        "Test_TC_TIMESYNC_1_1.yaml",         # Time sync SDK is not yet ready
-        "Test_TC_TIMESYNC_2_3.yaml",         # Time sync SDK is not yet ready
         "TestAttributesById.yaml",           # chip-repl does not support AnyCommands (06/06/2023)
         "TestCommandsById.yaml",             # chip-repl does not support AnyCommands (06/06/2023)
         "TestEventsById.yaml",               # chip-repl does not support AnyCommands (06/06/2023)
         "TestReadNoneSubscribeNone.yaml",    # chip-repl does not support AnyCommands (07/27/2023)
         "Test_TC_DRLK_2_8.yaml",   # Test fails only in chip-repl: Refer--> https://github.com/project-chip/connectedhomeip/pull/27011#issuecomment-1593339855
         "Test_TC_ACE_1_6.yaml",    # Test fails only in chip-repl: Refer--> https://github.com/project-chip/connectedhomeip/pull/27910#issuecomment-1632485584
-        "Test_TC_PSCFG_1_1.yaml",  # Power source configuration cluster is deprecated and removed from all-clusters
-        "Test_TC_PSCFG_2_1.yaml",  # Power source configuration cluster is deprecated and removed from all-clusters
-        "Test_TC_PSCFG_2_2.yaml",  # Power source configuration cluster is deprecated and removed from all-clusters
-        "Test_TC_SMOKECO_2_2.yaml",          # chip-repl does not support local timeout (07/20/2023)
-        "Test_TC_SMOKECO_2_3.yaml",          # chip-repl does not support local timeout (07/20/2023)
-        "Test_TC_SMOKECO_2_4.yaml",          # chip-repl does not support local timeout (07/20/2023)
-        "Test_TC_SMOKECO_2_5.yaml",          # chip-repl does not support local timeout (07/20/2023)
-        "Test_TC_SMOKECO_2_6.yaml",          # chip-repl does not support local timeout (07/20/2023)
         "Test_TC_IDM_1_2.yaml",              # chip-repl does not support AnyCommands (19/07/2023)
+        "TestGroupKeyManagementCluster.yaml",  # chip-repl does not support EqualityCommands (2023-08-04)
     }
 
 
@@ -185,19 +206,23 @@ def target_for_name(name: str):
     return TestTarget.ALL_CLUSTERS
 
 
-def tests_with_command(chip_tool: str, is_manual: bool):
+def tests_with_command(chip_tool: str, is_manual: bool, is_chip_tool_python_only: bool = False):
     """Executes `chip_tool` binary to see what tests are available, using cmd
     to get the list.
     """
     cmd = "list"
     if is_manual:
         cmd += "-manual"
+    elif is_chip_tool_python_only:
+        cmd += "-python-runner-only"
 
     result = subprocess.run([chip_tool, "tests", cmd], capture_output=True)
 
     test_tags = set()
     if is_manual:
         test_tags.add(TestTag.MANUAL)
+    if is_chip_tool_python_only:
+        test_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)
 
     in_development_tests = [s.replace(".yaml", "") for s in _GetInDevelopmentTests()]
 
@@ -222,6 +247,7 @@ def _hardcoded_python_yaml_tests():
     flaky_tests = _GetFlakyTests()
     slow_tests = _GetSlowTests()
     in_development_tests = _GetInDevelopmentTests()
+    chip_repl_unsupported_tests = _GetChipReplUnsupportedTests()
 
     for path in _AllYamlTests():
         if not _IsValidYamlTest(path.name):
@@ -240,6 +266,9 @@ def _hardcoded_python_yaml_tests():
         if path.name in in_development_tests:
             tags.add(TestTag.IN_DEVELOPMENT)
 
+        if path.name in chip_repl_unsupported_tests:
+            tags.add(TestTag.IN_DEVELOPMENT)
+
         yield TestDefinition(
             run_name=str(path),
             name=path.stem,  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
@@ -260,6 +289,9 @@ def AllChipToolTests(chip_tool: str):
     for test in tests_with_command(chip_tool, is_manual=True):
         yield test
 
+    for test in tests_with_command(chip_tool, is_manual=False, is_chip_tool_python_only=True):
+        yield test
+
 
 __all__ = [
     "TestTarget",

scripts/tests/chiptest/test_definition.py

@@ -217,6 +217,7 @@ class TestTag(Enum):
     SLOW = auto()            # test uses Sleep and is generally slow (>=10s is a typical threshold)
     FLAKY = auto()           # test is considered flaky (usually a bug/time dependent issue)
     IN_DEVELOPMENT = auto()  # test may not pass or undergoes changes
+    CHIP_TOOL_PYTHON_ONLY = auto()  # test uses YAML features only supported by the CHIP_TOOL_PYTHON runner.
 
     def to_s(self):
         for (k, v) in TestTag.__members__.items():

scripts/tests/run_test_suite.py

@@ -174,6 +174,9 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
             TestTag.FLAKY,
         }
 
+        if runtime != TestRunTime.CHIP_TOOL_PYTHON:
+            exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)
+
     if 'all' not in target:
         tests = []
         for name in target:

scripts/tools/zap/zap_execution.py

@@ -23,7 +23,7 @@
 # Use scripts/tools/zap/version_update.py to manage ZAP versioning as many
 # files may need updating for versions
 #
-MIN_ZAP_VERSION = '2023.8.1'
+MIN_ZAP_VERSION = '2023.8.4'
 
 
 class ZapTool:

src/app/tests/suites/TestEqualities.yaml

@@ -18,8 +18,12 @@ config:
     nodeId: 0x12344321
     cluster: "EqualityCommands"
     endpoint: 1
-    expectedValue: 20003
-    unexpectedValue: 20004
+    expectedValue:
+        type: INT16U
+        defaultValue: 20003
+    unexpectedValue:
+        type: INT16U
+        defaultValue: 20004
 
 tests:
     - label: "Wait for the commissioned device to be retrieved"

src/app/tests/suites/TestGroupKeyManagementCluster.yaml

@@ -74,30 +74,29 @@ tests:
           constraints:
               minValue: 3
 
-    # TODO(#28396): Re-enabled when CI wouldn't break on this syntax
-    #
-    #   command: "readAttribute"
-    #   attribute: "ClusterRevision"
-    #   response:
-    #       saveAs: ClusterRevisionValue
-    #       constraints:
-    #           minValue: 1
-    #           maxValue: 2
-    #           type: int16u
-
-    # - label: "Creates a new variable to hold if the cluster revision is == 2"
-    #   cluster: EqualityCommands
-    #   command: UnsignedNumberEquals
-    #   arguments:
-    #       values:
-    #           - name: Value1
-    #             value: ClusterRevisionValue
-    #           - name: Value2
-    #             value: 2
-    #   response:
-    #       - values:
-    #             - name: Equals
-    #               saveAs: ClusterIsRevision2
+    - label: "Read the ClusterRevision attribute"
+      command: "readAttribute"
+      attribute: "ClusterRevision"
+      response:
+          saveAs: ClusterRevisionValue
+          constraints:
+              minValue: 1
+              maxValue: 2
+              type: int16u
+
+    - label: "Creates a new variable to hold if the cluster revision is == 2"
+      cluster: EqualityCommands
+      command: UnsignedNumberEquals
+      arguments:
+          values:
+              - name: Value1
+                value: ClusterRevisionValue
+              - name: Value2
+                value: 2
+      response:
+          - values:
+                - name: Equals
+                  saveAs: ClusterIsRevision2
 
     ########### KeySetWrite Epoch0 field validations
     - label: "KeySetWrite with EpochKey0 null fails INVALID_COMMAND"
@@ -161,7 +160,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey0 with length 1 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -183,7 +182,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey0 with length 0 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -204,7 +203,7 @@ tests:
 
     ########### KeySetWrite Epoch1 field validations
     - label: "KeySetWrite with EpochStartTime1 null fails INVALID_COMMAND"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -245,7 +244,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey1 with length 1 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -267,7 +266,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey1 with length 0 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -310,7 +309,7 @@ tests:
     ########### KeySetWrite Epoch2 field validations
     - label: "KeySetWrite with EpochStartTime2 null fails INVALID_COMMAND"
       command: "KeySetWrite"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       arguments:
           values:
               - name: "GroupKeySet"
@@ -350,7 +349,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey2 with length 1 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -372,7 +371,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey2 with length 0 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:

src/app/tests/suites/certification/Test_TC_SMOKECO_2_3.yaml

@@ -55,7 +55,7 @@ tests:
       minInterval: 3
       maxInterval: 30
       response:
-          Value: 0
+          value: 0
           constraints:
               type: enum8
 

src/app/tests/suites/ciTests.json

@@ -297,7 +297,6 @@
     "Groups": [
         "TestGroupMessaging",
         "TestGroupsCluster",
-        "TestGroupKeyManagementCluster",
         "Test_TC_G_1_1",
         "Test_TC_G_2_1"
     ],

src/app/tests/suites/pythonRunnerOnlyTests.json

@@ -0,0 +1,5 @@
+{
+    "Groups": ["TestGroupKeyManagementCluster"],
+    "Others": ["TestEqualities"],
+    "collection": ["Groups", "Others"]
+}