Define LastReceivedEventNumber as a global variable.

scripts/py_matter_yamltests/matter_yamltests/parser.py

@@ -14,8 +14,10 @@
 #    limitations under the License.
 
 import copy
+import logging
 from dataclasses import dataclass, field
 from enum import Enum, auto
+from typing import Optional
 
 from . import fixes
 from .constraints import get_constraints, is_typed_constraint
@@ -698,6 +700,28 @@ def event_number(self, value):
     def pics(self):
         return self._test.pics
 
+    def _get_last_event_number(self, responses) -> Optional[int]:
+        if not self.is_event:
+            return None
+
+        # find the largest event number in all responses
+        # This iterates over everything (not just last element) since some commands like
+        # `chip-tool any read-all` may return multiple replies
+        event_number = None
+
+        for response in responses:
+            if not isinstance(response, dict):
+                continue
+            received_event_number = response.get('eventNumber')
+
+            if not isinstance(received_event_number, int):
+                continue
+
+            if (event_number is None) or (event_number < received_event_number):
+                event_number = received_event_number
+
+        return event_number
+
     def post_process_response(self, received_responses):
         result = PostProcessResponseResult()
 
@@ -710,6 +734,18 @@ def post_process_response(self, received_responses):
         if self._test.save_response_as:
             self._runtime_config_variable_storage[self._test.save_response_as] = received_responses
 
+        if self.is_event:
+            last_event_number = self._get_last_event_number(received_responses)
+            if last_event_number:
+                if 'LastReceivedEventNumber' in self._runtime_config_variable_storage:
+                    if self._runtime_config_variable_storage['LastReceivedEventNumber'] > last_event_number:
+                        logging.warning(
+                            "Received an older event than expected: received %r < %r",
+                            last_event_number,
+                            self._runtime_config_variable_storage['LastReceivedEventNumber']
+                        )
+                self._runtime_config_variable_storage['LastReceivedEventNumber'] = last_event_number
+
         if self.wait_for is not None:
             self._response_cluster_wait_validation(received_responses, result)
             return result
@@ -1162,6 +1198,10 @@ def __apply_legacy_config(self, config):
         self.__apply_legacy_config_if_missing(config, 'cluster', '')
         self.__apply_legacy_config_if_missing(config, 'timeout', 90)
 
+        # These values are default runtime values (non-legacy)
+        self.__apply_legacy_config_if_missing(
+            config, 'LastReceivedEventNumber', 0)
+
     def __apply_legacy_config_if_missing(self, config, key, value):
         if key not in config:
             config[key] = value

scripts/py_matter_yamltests/matter_yamltests/runner.py

@@ -122,14 +122,6 @@ def run(self, config: TestRunnerConfig) -> bool:
 
 
 class TestRunner(TestRunnerBase):
-    """
-    TestRunner is a default runner implementation.
-
-    last_event_number: The latest event number value after the readEvent command.
-    """
-
-    last_event_number: int = 0
-
     async def start(self):
         return
 
@@ -180,9 +172,6 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig):
 
             test_duration = 0
             for idx, request in enumerate(parser.tests):
-                if request.is_event and request.event_number == 'newEventsOnly':
-                    request.event_number = self.last_event_number + 1
-
                 if not request.is_pics_enabled:
                     hooks.step_skipped(request.label, request.pics)
                     continue
@@ -207,13 +196,6 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig):
                 duration = round((time.time() - start) * 1000, 2)
                 test_duration += duration
 
-                if request.is_event:
-                    last_event = responses[-1]
-                    if isinstance(last_event, dict):
-                        received_event_number = last_event.get('eventNumber')
-                        if isinstance(received_event_number, int) and self.last_event_number < received_event_number:
-                            self.last_event_number = received_event_number
-
                 logger = request.post_process_response(responses)
 
                 if logger.is_failure():

src/app/tests/suites/certification/Test_TC_SMOKECO_2_2.yaml

@@ -123,7 +123,7 @@ tests:
       PICS: SMOKECO.S.E00
       command: "readEvent"
       event: "SmokeAlarm"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: 1 }
 
@@ -203,7 +203,7 @@ tests:
       PICS: SMOKECO.S.E00
       command: "readEvent"
       event: "SmokeAlarm"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: 2 }
 
@@ -248,6 +248,6 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}

src/app/tests/suites/certification/Test_TC_SMOKECO_2_3.yaml

@@ -122,7 +122,7 @@ tests:
       PICS: SMOKECO.S.E01
       command: "readEvent"
       event: "COAlarm"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: 1 }
 
@@ -202,7 +202,7 @@ tests:
       PICS: SMOKECO.S.E01
       command: "readEvent"
       event: "COAlarm"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: 2 }
 
@@ -247,6 +247,6 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}

src/app/tests/suites/certification/Test_TC_SMOKECO_2_4.yaml

@@ -135,7 +135,7 @@ tests:
       PICS: SMOKECO.S.E02
       command: "readEvent"
       event: "LowBattery"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: 1 }
 
@@ -180,7 +180,7 @@ tests:
       PICS: SMOKECO.S.E02
       command: "readEvent"
       event: "LowBattery"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: 2 }
 
@@ -225,7 +225,7 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -281,7 +281,7 @@ tests:
       PICS: SMOKECO.S.E03
       command: "readEvent"
       event: "HardwareFault"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -327,7 +327,7 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -383,7 +383,7 @@ tests:
       PICS: SMOKECO.S.E04
       command: "readEvent"
       event: "EndOfService"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -429,7 +429,7 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -503,7 +503,7 @@ tests:
       PICS: SMOKECO.S.E05
       command: "readEvent"
       event: "SelfTestComplete"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -520,7 +520,7 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -565,7 +565,7 @@ tests:
       PICS: SMOKECO.S.E05
       command: "readEvent"
       event: "SelfTestComplete"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -582,6 +582,6 @@ tests:
       PICS: SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}

src/app/tests/suites/certification/Test_TC_SMOKECO_2_5.yaml

@@ -163,7 +163,7 @@ tests:
       PICS: SMOKECO.S.A0008 && SMOKECO.S.E08
       command: "readEvent"
       event: "InterconnectSmokeAlarm"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: interconnectSmokeAlarmSeverityLevel }
 
@@ -218,7 +218,7 @@ tests:
       PICS: SMOKECO.S.A0008 && SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -276,7 +276,7 @@ tests:
       PICS: SMOKECO.S.A0009 && SMOKECO.S.E09
       command: "readEvent"
       event: "InterconnectCOAlarm"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: { AlarmSeverityLevel: interconnectCOAlarmSeverityLevel }
 
@@ -331,7 +331,7 @@ tests:
       PICS: SMOKECO.S.A0009 && SMOKECO.S.E0a
       command: "readEvent"
       event: "AllClear"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -689,7 +689,7 @@ tests:
       PICS: SMOKECO.S.A0004 && SMOKECO.S.F00 && SMOKECO.S.E06
       command: "readEvent"
       event: "AlarmMuted"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -726,7 +726,7 @@ tests:
       PICS: SMOKECO.S.A0004 && SMOKECO.S.F00 && SMOKECO.S.E07
       command: "readEvent"
       event: "MuteEnded"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -921,7 +921,7 @@ tests:
       PICS: SMOKECO.S.A0004 && SMOKECO.S.F01 && SMOKECO.S.E06
       command: "readEvent"
       event: "AlarmMuted"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}
 
@@ -958,7 +958,7 @@ tests:
       PICS: SMOKECO.S.A0004 && SMOKECO.S.F01 && SMOKECO.S.E07
       command: "readEvent"
       event: "MuteEnded"
-      eventNumber: "newEventsOnly"
+      eventNumber: "LastReceivedEventNumber + 1"
       response:
           value: {}