From 1d90bbe6f1ac994a9ba953c9356d3fd10d776e6e Mon Sep 17 00:00:00 2001 From: Joshua Villasenor Date: Tue, 30 May 2023 14:48:39 -0700 Subject: [PATCH 1/5] Resolve issue with UserPrompts in test harness. --- .../py_matter_yamltests/matter_yamltests/hooks.py | 2 +- .../matter_yamltests/parser_config.py | 2 +- .../pseudo_clusters/clusters/log_commands.py | 7 ++++++- .../py_matter_yamltests/matter_yamltests/runner.py | 4 ++-- scripts/tests/yaml/tests_logger.py | 12 ++++++------ 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/py_matter_yamltests/matter_yamltests/hooks.py b/scripts/py_matter_yamltests/matter_yamltests/hooks.py index 7e779420759e6c..3ecb8a1babd4ed 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/hooks.py +++ b/scripts/py_matter_yamltests/matter_yamltests/hooks.py @@ -143,7 +143,7 @@ def step_skipped(self, name: str, expression: str): """ pass - def step_start(self, name: str): + def step_start(self, request): """ This method is called when the runner starts running a step from the test. diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser_config.py b/scripts/py_matter_yamltests/matter_yamltests/parser_config.py index 251aa5ebb17266..17b19f2b4588ef 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/parser_config.py +++ b/scripts/py_matter_yamltests/matter_yamltests/parser_config.py @@ -29,7 +29,7 @@ def get_config(test_file: str): config_options = {} yaml_loader = YamlLoader() - _, _, config, _ = yaml_loader.load(test_file) + _, _, _, config, _ = yaml_loader.load(test_file) config_options = {key: value if not isinstance( value, dict) else value['defaultValue'] for key, value in config.items()} return config_options diff --git a/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py b/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py index 00a98e285c84e6..e46f2184fad3b4 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py +++ b/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py @@ -39,7 +39,12 @@ class LogCommands(PseudoCluster): definition = _DEFINITION async def UserPrompt(self, request): - pass + input_result = input("") + for value in request.arguments.get("values", []): + if value.get('name') and 'expectedValue' in value['name']: + request.responses = [{"values": [{"name": "expectedValue", "value": value['value']}]}] + return {"value": {"expectedValue": input_result}} + return {} async def Log(self, request): pass diff --git a/scripts/py_matter_yamltests/matter_yamltests/runner.py b/scripts/py_matter_yamltests/matter_yamltests/runner.py index 6f201c8ea6ad74..9d560216321494 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/runner.py +++ b/scripts/py_matter_yamltests/matter_yamltests/runner.py @@ -180,11 +180,11 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig): hooks.step_skipped(request.label, request.pics) continue elif not config.adapter: - hooks.step_start(request.label) + hooks.step_start(request) hooks.step_unknown() continue else: - hooks.step_start(request.label) + hooks.step_start(request) start = time.time() if config.pseudo_clusters.supports(request): diff --git a/scripts/tests/yaml/tests_logger.py b/scripts/tests/yaml/tests_logger.py index 0f718e243dbbb8..8cb74f025cfe6f 100755 --- a/scripts/tests/yaml/tests_logger.py +++ b/scripts/tests/yaml/tests_logger.py @@ -193,11 +193,14 @@ def step_skipped(self, name: str, expression: str): self.__index += 1 self.__skipped += 1 - def step_start(self, name: str): + def step_start(self, request): if self.__use_test_harness_log_format: - print(self.__strings.test_harness_step_start.format(index=self.__index, name=name)) + print(self.__strings.test_harness_step_start.format(index=self.__index, name=request.label)) - print(self.__strings.step_start.format(index=self.__index, name=click.style(name, bold=True)), end='') + print(self.__strings.step_start.format(index=self.__index, name=click.style(request.label, bold=True)), end='') + if request.command == 'UserPrompt': + message = request.arguments['values'][0]['value'] + print("\n" + self.__strings.user_prompt.format(message=f'{message}')) # flushing stdout such that the previous print statement is visible on the screen for long running tasks. sys.stdout.flush() @@ -218,9 +221,6 @@ def step_success(self, logger, logs, duration: int, request): elif request.command == 'Log': message = request.arguments['values'][0]['value'] print(self.__strings.log.format(message=f'{message}')) - elif request.command == 'UserPrompt': - message = request.arguments['values'][0]['value'] - print(self.__strings.user_prompt.format(message=f'{message}')) if self.__show_adapter_logs: self.__log_printer.print(logs) From 30d92f2d143d9da8078f37633bc0a86100e12224 Mon Sep 17 00:00:00 2001 From: Joshua Villasenor Date: Wed, 31 May 2023 07:49:22 -0700 Subject: [PATCH 2/5] Update UserPrompt to match legacy behavior if not expected value is set in YAML --- .../pseudo_clusters/clusters/log_commands.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py b/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py index e46f2184fad3b4..16bb5ebe2e370a 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py +++ b/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/clusters/log_commands.py @@ -39,11 +39,16 @@ class LogCommands(PseudoCluster): definition = _DEFINITION async def UserPrompt(self, request): - input_result = input("") + expected_value = None for value in request.arguments.get("values", []): if value.get('name') and 'expectedValue' in value['name']: - request.responses = [{"values": [{"name": "expectedValue", "value": value['value']}]}] - return {"value": {"expectedValue": input_result}} + expected_value = value['value'] + request.responses = [{"values": [{"name": "expectedValue", "value": expected_value}]}] + + if expected_value is not None: + input_result = input("") + return {"value": {"expectedValue": input_result}} + return {} async def Log(self, request): From 20f90c561f4b582a28c75265473f1e3b618c188a Mon Sep 17 00:00:00 2001 From: Joshua Villasenor Date: Wed, 31 May 2023 11:08:07 -0700 Subject: [PATCH 3/5] Update documentation and typing for request parameter. --- .../py_matter_yamltests/matter_yamltests/hooks.py | 15 ++++++++------- scripts/tests/yaml/tests_logger.py | 7 ++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/py_matter_yamltests/matter_yamltests/hooks.py b/scripts/py_matter_yamltests/matter_yamltests/hooks.py index 3ecb8a1babd4ed..6017dd68ad8f2f 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/hooks.py +++ b/scripts/py_matter_yamltests/matter_yamltests/hooks.py @@ -14,6 +14,7 @@ # limitations under the License. from .errors import TestStepError +from .parser import TestStep class TestParserHooks(): @@ -143,18 +144,18 @@ def step_skipped(self, name: str, expression: str): """ pass - def step_start(self, request): + def step_start(self, request: TestStep): """ This method is called when the runner starts running a step from the test. Parameters ---------- - name: str - The name of the test step that is starting. + request: TestStep + The original request as defined by the test step. """ pass - def step_success(self, logger, logs, duration: int, request): + def step_success(self, logger, logs, duration: int, request: TestStep): """ This method is called when running a step succeeds. @@ -169,12 +170,12 @@ def step_success(self, logger, logs, duration: int, request): duration: int How long it took to run the test step, in milliseconds. - request: + request: TestStep The original request as defined by the test step. """ pass - def step_failure(self, logger, logs, duration: int, request, received): + def step_failure(self, logger, logs, duration: int, request: TestStep, received): """ This method is called when running a step fails. @@ -189,7 +190,7 @@ def step_failure(self, logger, logs, duration: int, request, received): duration: int How long it took to run the test step, in milliseconds. - request: + request: TestStep The original request as defined by the test step. received: diff --git a/scripts/tests/yaml/tests_logger.py b/scripts/tests/yaml/tests_logger.py index 8cb74f025cfe6f..4815a9815288d6 100755 --- a/scripts/tests/yaml/tests_logger.py +++ b/scripts/tests/yaml/tests_logger.py @@ -23,6 +23,7 @@ import click from matter_yamltests.errors import TestStepError, TestStepKeyError from matter_yamltests.hooks import TestParserHooks, TestRunnerHooks, WebSocketRunnerHooks +from matter_yamltests.parser import TestStep def _strikethrough(str): @@ -193,7 +194,7 @@ def step_skipped(self, name: str, expression: str): self.__index += 1 self.__skipped += 1 - def step_start(self, request): + def step_start(self, request: TestStep): if self.__use_test_harness_log_format: print(self.__strings.test_harness_step_start.format(index=self.__index, name=request.label)) @@ -211,7 +212,7 @@ def step_unknown(self): self.__runned += 1 - def step_success(self, logger, logs, duration: int, request): + def step_success(self, logger, logs, duration: int, request: TestStep): print(self.__strings.step_result.format(state=_SUCCESS, duration=duration)) self.__print_results(logger) @@ -230,7 +231,7 @@ def step_success(self, logger, logs, duration: int, request): self.__errors += logger.errors self.__runned += 1 - def step_failure(self, logger, logs, duration: int, request, received): + def step_failure(self, logger, logs, duration: int, request: TestStep, received): print(self.__strings.step_result.format(state=_FAILURE, duration=duration)) self.__print_results(logger) From 28ff5fcb305d87b948be9064807412cef50547e3 Mon Sep 17 00:00:00 2001 From: Joshua Villasenor Date: Wed, 31 May 2023 11:59:20 -0700 Subject: [PATCH 4/5] Add documentation of UserPrompt logging in step_start --- scripts/tests/yaml/tests_logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/tests/yaml/tests_logger.py b/scripts/tests/yaml/tests_logger.py index 4815a9815288d6..ce124dee821ce1 100755 --- a/scripts/tests/yaml/tests_logger.py +++ b/scripts/tests/yaml/tests_logger.py @@ -199,6 +199,8 @@ def step_start(self, request: TestStep): print(self.__strings.test_harness_step_start.format(index=self.__index, name=request.label)) print(self.__strings.step_start.format(index=self.__index, name=click.style(request.label, bold=True)), end='') + # This is to keep previous behavior of UserPrompt. Where it logs USER_PROMPT prior to user input. See link below: + # https://github.com/project-chip/connectedhomeip/blob/master/src/app/tests/suites/commands/log/LogCommands.cpp#L31 if request.command == 'UserPrompt': message = request.arguments['values'][0]['value'] print("\n" + self.__strings.user_prompt.format(message=f'{message}')) From 359256099cf96823f49693029b354b7b808e837e Mon Sep 17 00:00:00 2001 From: Joshua Villasenor Date: Thu, 1 Jun 2023 08:53:13 -0700 Subject: [PATCH 5/5] Make reference link a permalink --- scripts/tests/yaml/tests_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/yaml/tests_logger.py b/scripts/tests/yaml/tests_logger.py index ce124dee821ce1..f01b0d28ec26a4 100755 --- a/scripts/tests/yaml/tests_logger.py +++ b/scripts/tests/yaml/tests_logger.py @@ -200,7 +200,7 @@ def step_start(self, request: TestStep): print(self.__strings.step_start.format(index=self.__index, name=click.style(request.label, bold=True)), end='') # This is to keep previous behavior of UserPrompt. Where it logs USER_PROMPT prior to user input. See link below: - # https://github.com/project-chip/connectedhomeip/blob/master/src/app/tests/suites/commands/log/LogCommands.cpp#L31 + # https://github.com/project-chip/connectedhomeip/blob/6644a4b0b0d1272ae325c651b27bd0e7068f3a8a/src/app/tests/suites/commands/log/LogCommands.cpp#L31 if request.command == 'UserPrompt': message = request.arguments['values'][0]['value'] print("\n" + self.__strings.user_prompt.format(message=f'{message}'))