From a59bac690275aae4992ec338e20d3cf4dd133e2e Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 12 Jun 2024 17:04:56 -0400 Subject: [PATCH 01/18] Automated implementation of 1 python test to use Metadata script as an intermediate argument holder --- scripts/tests/py/metadata.py | 186 +++++++++++-------------------- scripts/tests/run_python_test.py | 33 +++++- src/python_testing/TC_SC_3_6.py | 7 ++ 3 files changed, 105 insertions(+), 121 deletions(-) diff --git a/scripts/tests/py/metadata.py b/scripts/tests/py/metadata.py index 0445b3a31b4a6e..2f758c629a313e 100644 --- a/scripts/tests/py/metadata.py +++ b/scripts/tests/py/metadata.py @@ -15,51 +15,63 @@ import re from dataclasses import dataclass -from typing import Dict, List, Optional - +from typing import Dict, List, Optional, Any import yaml @dataclass class Metadata: - py_script_path: Optional[str] = None - run: Optional[str] = None - app: Optional[str] = None - discriminator: Optional[str] = None - passcode: Optional[str] = None - - def copy_from_dict(self, attr_dict: Dict[str, str]) -> None: + py_script_path: str + run: str + app: str + app_args: str + script_args: str + factoryreset: bool = False + factoryreset_app_only: bool = False + script_gdb: bool = False + quiet: bool = True + + def copy_from_dict(self, attr_dict: Dict[str, Any]) -> None: """ Sets the value of the attributes from a dictionary. Attributes: attr_dict: - Dictionary that stores attributes value that should - be transferred to this class. + Dictionary that stores attributes value that should + be transferred to this class. """ - if "app" in attr_dict: self.app = attr_dict["app"] if "run" in attr_dict: self.run = attr_dict["run"] - if "discriminator" in attr_dict: - self.discriminator = attr_dict["discriminator"] + if "app-args" in attr_dict: + self.app_args = attr_dict["app-args"] - if "passcode" in attr_dict: - self.passcode = attr_dict["passcode"] + if "script-args" in attr_dict: + self.script_args = attr_dict["script-args"] if "py_script_path" in attr_dict: self.py_script_path = attr_dict["py_script_path"] - # TODO - set other attributes as well + if "factoryreset" in attr_dict: + self.factoryreset = bool(attr_dict["factoryreset"]) + + if "factoryreset_app_only" in attr_dict: + self.factoryreset_app_only = bool(attr_dict["factoryreset_app_only"]) + + if "script_gdb" in attr_dict: + self.script_gdb = bool(attr_dict["script_gdb"]) + + if "quiet" in attr_dict: + self.quiet = bool(attr_dict["quiet"]) class MetadataReader: """ - A class to parse run arguments from the test scripts and + A class to parse run arguments from the test scripts and resolve them to environment specific values. """ @@ -70,97 +82,31 @@ def __init__(self, env_yaml_file_path: str): Parameters: env_yaml_file_path: - Path to the environment file that contains the YAML configuration. + Path to the environment file that contains the YAML configuration. """ with open(env_yaml_file_path) as stream: - self.env = yaml.safe_load(stream) + self.env: Dict[str, str] = yaml.safe_load(stream) def __resolve_env_vals__(self, metadata_dict: Dict[str, str]) -> None: """ Resolves the argument defined in the test script to environment values. For example, if a test script defines "all_clusters" as the value for app name, we will check the environment configuration to see what raw value is - assocaited with the "all_cluster" variable and set the value for "app" option + associated with the "all_cluster" variable and set the value for "app" option to this raw value. Parameter: metadata_dict: - Dictionary where each key represent a particular argument and its value represent - the value for that argument defined in the test script. + Dictionary where each key represent a particular argument and its value represent + the value for that argument defined in the test script. """ + for arg, arg_val in metadata_dict.items(): + # We do not expect to recurse (like ${FOO_${BAR}}) so just expand once + for name, value in self.env.items(): + arg_val = arg_val.replace(f'${{{name}}}', value) + metadata_dict[arg] = arg_val - for run_arg, run_arg_val in metadata_dict.items(): - - if not type(run_arg_val) == str or run_arg == "run": - metadata_dict[run_arg] = run_arg_val - continue - - if run_arg_val is None: - continue - - sub_args = run_arg_val.split('/') - - if len(sub_args) not in [1, 2]: - err = """The argument is not in the correct format. - The argument must follow the format of arg1 or arg1/arg2. - For example, arg1 represents the argument type and optionally arg2 - represents a specific variable defined in the environment file whose - value should be used as the argument value. If arg2 is not specified, - we will just use the first value associated with arg1 in the environment file.""" - raise Exception(err) - - if len(sub_args) == 1: - run_arg_val = self.env.get(sub_args[0]) - - elif len(sub_args) == 2: - run_arg_val = self.env.get(sub_args[0]).get(sub_args[1]) - - # if a argument has been specified in the comment header - # but can't be found in the env file, consider it to be - # boolean value. - if run_arg_val is None: - run_arg_val = True - - metadata_dict[run_arg] = run_arg_val - - def __read_args__(self, run_args_lines: List[str]) -> Dict[str, str]: - """ - Parses a list of lines and extracts argument - values from it. - - Parameters: - - run_args_lines: - Line in test script header that contains run argument definition. - Each line will contain a list of run arguments separated by a space. - Line below is one example of what the run argument line will look like: - "app/all-clusters discriminator KVS storage-path" - - In this case the line defines that app, discriminator, KVS, and storage-path - are the arguments that should be used with this run. - - An argument can be defined multiple times in the same line or in different lines. - The last definition will override any previous definition. For example, - "KVS/kvs1 KVS/kvs2 KVS/kvs3" line will lead to KVS value of kvs3. - """ - metadata_dict = {} - - for run_line in run_args_lines: - for run_arg_word in run_line.strip().split(): - ''' - We expect the run arg to be defined in one of the - following two formats: - 1. run_arg - 2. run_arg/run_arg_val - - Examples: "discriminator" and "app/all_clusters" - - ''' - run_arg = run_arg_word.split('/', 1)[0] - metadata_dict[run_arg] = run_arg_word - - return metadata_dict def parse_script(self, py_script_path: str) -> List[Metadata]: """ @@ -171,47 +117,51 @@ def parse_script(self, py_script_path: str) -> List[Metadata]: Parameter: py_script_path: - path to the python test script + path to the python test script Return: List[Metadata] - List of Metadata object where each Metadata element represents - the run arguments associated with a particular run defined in - the script file. + List of Metadata object where each Metadata element represents + the run arguments associated with a particular run defined in + the script file. """ runs_def_ptrn = re.compile(r'^\s*#\s*test-runner-runs:\s*(.*)$') - args_def_ptrn = re.compile(r'^\s*#\s*test-runner-run/([a-zA-Z0-9_]+):\s*(.*)$') + arg_def_ptrn = re.compile(r'^\s*#\s*test-runner-run/([a-zA-Z0-9_]+)/([a-zA-Z0-9_\-]+):\s*(.*)$') - runs_arg_lines: Dict[str, List[str]] = {} - runs_metadata = [] + runs_arg_lines: Dict[str, Dict[str, str]] = {} + runs_metadata: List[Metadata] = [] with open(py_script_path, 'r', encoding='utf8') as py_script: for line in py_script.readlines(): - runs_match = runs_def_ptrn.match(line.strip()) - args_match = args_def_ptrn.match(line.strip()) + args_match = arg_def_ptrn.match(line.strip()) if runs_match: for run in runs_match.group(1).strip().split(): - runs_arg_lines[run] = [] + runs_arg_lines[run] = {} + runs_arg_lines[run]['run'] = run + runs_arg_lines[run]['py_script_path'] = py_script_path elif args_match: - runs_arg_lines[args_match.group(1)].append(args_match.group(2)) - - for run, lines in runs_arg_lines.items(): - metadata_dict = self.__read_args__(lines) - self.__resolve_env_vals__(metadata_dict) - - # store the run value and script location in the - # metadata object - metadata_dict['py_script_path'] = py_script_path - metadata_dict['run'] = run - - metadata = Metadata() - - metadata.copy_from_dict(metadata_dict) + runs_arg_lines[args_match.group(1)][args_match.group(2)] = args_match.group(3) + + for run, attr in runs_arg_lines.items(): + self.__resolve_env_vals__(attr) + + metadata = Metadata( + py_script_path=attr.get("py_script_path", ""), + run=attr.get("run", ""), + app=attr.get("app", ""), + app_args=attr.get("app_args", ""), + script_args=attr.get("script_args", ""), + factoryreset=bool(attr.get("factoryreset", False)), + factoryreset_app_only=bool(attr.get("factoryreset_app_only", False)), + script_gdb=bool(attr.get("script_gdb", False)), + quiet=bool(attr.get("quiet", True)) + ) + metadata.copy_from_dict(attr) runs_metadata.append(metadata) return runs_metadata diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index 5d2afc31ae21b7..496287df572a0b 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -32,6 +32,7 @@ import click import coloredlogs from colorama import Fore, Style +from py.metadata import MetadataReader, Metadata DEFAULT_CHIP_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) @@ -89,7 +90,33 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st @click.option("--script-gdb", is_flag=True, help='Run script through gdb') @click.option("--quiet", is_flag=True, help="Do not print output from passing tests. Use this flag in CI to keep github log sizes manageable.") -def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str, script: str, script_args: str, script_gdb: bool, quiet: bool): +@click.option("--load-from-env", default=None, help="YAML file that contains values for environment variables.") +def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str, script: str, script_args: str, script_gdb: bool, quiet: bool, load_from_env): + if load_from_env: + reader = MetadataReader(load_from_env) + runs = reader.parse_script(script) + else: + runs = [ + Metadata( + py_script_path=script, + run="cmd-run", + app=app, + app_args=app_args, + script_args=script_args, + factoryreset=factoryreset, + factoryreset_app_only=factoryreset_app_only, + script_gdb=script_gdb, + quiet=quiet + ) + ] + + for run in runs: + print(f"Executing run: {run.py_script_path}") + main_impl(run.app, run.factoryreset, run.factoryreset_app_only, run.app_args, run.py_script_path, run.script_args, run.script_gdb, run.quiet) + + +def main_impl(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str, script: str, script_args: str, script_gdb: bool, quiet: bool): + app_args = app_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) script_args = script_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) @@ -192,8 +219,8 @@ def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: st else: logging.info("Test completed successfully") - sys.exit(exit_code) - + if exit_code != 0: + sys.exit(exit_code) if __name__ == '__main__': main(auto_envvar_prefix='CHIP') diff --git a/src/python_testing/TC_SC_3_6.py b/src/python_testing/TC_SC_3_6.py index ec09d4bb8e815e..2363afdc1e3c0a 100644 --- a/src/python_testing/TC_SC_3_6.py +++ b/src/python_testing/TC_SC_3_6.py @@ -15,6 +15,13 @@ # limitations under the License. # +# test-runner-runs: run1 +# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} +# test-runner-run/run1/factoryreset: True +# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto + + import asyncio import logging import queue From b0de6c4b9e03343a77ed510d57d6aced221f084e Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 12 Jun 2024 17:07:20 -0400 Subject: [PATCH 02/18] Added modified tests.yaml --- .github/workflows/tests.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dccd846affddc1..61e34704ee7cb4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -474,6 +474,16 @@ jobs: build \ --copy-artifacts-to objdir-clone \ " + - name: Generate an argument environment file + run: | + touch /tmp/test_env.yaml + echo "ALL_CLUSTERS_APP: out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app" >> /tmp/test_env.yaml + echo "CHIP_LOCK_APP: out/linux-x64-lock-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-lock-app" >> /tmp/test_env.yaml + echo "ENERGY_MANAGEMENT_APP: out/linux-x64-energy-management-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-energy-management-app" >> /tmp/test_env.yaml + echo "TRACE_APP: out/trace_data/app-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml + echo "TRACE_TEST_JSON: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml + echo "TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME}" >> /tmp/test_env.yaml + - name: Run Tests run: | mkdir -p out/trace_data @@ -515,7 +525,7 @@ jobs: scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json --enable-key 000102030405060708090a0b0c0d0e0f" --script "src/python_testing/TC_IDM_1_4.py" --script-args "--hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_PWRTL_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_SC_3_6.py scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_10.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_11.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' From 676282eaa2dd505c3810dcd1664a4c6e594c5a25 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 12 Jun 2024 21:19:48 +0000 Subject: [PATCH 03/18] Restyled by autopep8 --- scripts/tests/py/metadata.py | 1 - scripts/tests/run_python_test.py | 12 +++++++----- src/python_testing/TC_SC_3_6.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/tests/py/metadata.py b/scripts/tests/py/metadata.py index 2f758c629a313e..5b847c6f6b4217 100644 --- a/scripts/tests/py/metadata.py +++ b/scripts/tests/py/metadata.py @@ -107,7 +107,6 @@ def __resolve_env_vals__(self, metadata_dict: Dict[str, str]) -> None: arg_val = arg_val.replace(f'${{{name}}}', value) metadata_dict[arg] = arg_val - def parse_script(self, py_script_path: str) -> List[Metadata]: """ Parses a script and returns a list of metadata object where diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index 496287df572a0b..d4261ad168bf05 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -107,14 +107,15 @@ def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: st factoryreset_app_only=factoryreset_app_only, script_gdb=script_gdb, quiet=quiet - ) - ] + ) + ] for run in runs: print(f"Executing run: {run.py_script_path}") - main_impl(run.app, run.factoryreset, run.factoryreset_app_only, run.app_args, run.py_script_path, run.script_args, run.script_gdb, run.quiet) - - + main_impl(run.app, run.factoryreset, run.factoryreset_app_only, run.app_args, + run.py_script_path, run.script_args, run.script_gdb, run.quiet) + + def main_impl(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: str, script: str, script_args: str, script_gdb: bool, quiet: bool): app_args = app_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) @@ -222,5 +223,6 @@ def main_impl(app: str, factoryreset: bool, factoryreset_app_only: bool, app_arg if exit_code != 0: sys.exit(exit_code) + if __name__ == '__main__': main(auto_envvar_prefix='CHIP') diff --git a/src/python_testing/TC_SC_3_6.py b/src/python_testing/TC_SC_3_6.py index 2363afdc1e3c0a..cdcf3b9bde7388 100644 --- a/src/python_testing/TC_SC_3_6.py +++ b/src/python_testing/TC_SC_3_6.py @@ -15,7 +15,7 @@ # limitations under the License. # -# test-runner-runs: run1 +# test-runner-runs: run1 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP} # test-runner-run/run1/factoryreset: True # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json From 9080c26d359382eb02701479d97c54e9312dd143 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 12 Jun 2024 21:19:57 +0000 Subject: [PATCH 04/18] Restyled by isort --- scripts/tests/py/metadata.py | 3 ++- scripts/tests/run_python_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tests/py/metadata.py b/scripts/tests/py/metadata.py index 5b847c6f6b4217..2de3d9965d4335 100644 --- a/scripts/tests/py/metadata.py +++ b/scripts/tests/py/metadata.py @@ -15,7 +15,8 @@ import re from dataclasses import dataclass -from typing import Dict, List, Optional, Any +from typing import Any, Dict, List, Optional + import yaml diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index d4261ad168bf05..068d735ec101e2 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -32,7 +32,7 @@ import click import coloredlogs from colorama import Fore, Style -from py.metadata import MetadataReader, Metadata +from py.metadata import Metadata, MetadataReader DEFAULT_CHIP_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) From 5608abe83a56f081be5c9e35b78965f61e94322c Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 13 Jun 2024 10:15:24 -0400 Subject: [PATCH 05/18] Fixed the unite test for metadata --- scripts/tests/py/test_metadata.py | 80 ++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index 8707d483026bfb..5d5b32d76dd49c 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import tempfile +import os import unittest from os import path +import tempfile from typing import List from metadata import Metadata, MetadataReader @@ -22,29 +23,64 @@ class TestMetadataReader(unittest.TestCase): - def setUp(self): - # build the reader object - self.reader = MetadataReader(path.join(path.dirname(__file__), "env_test.yaml")) + test_file_content = ''' + # test-runner-runs: run1 + # test-runner-run/run1/app: ${ALL_CLUSTERS_APP} + # test-runner-run/run1/app-args: --discriminator 1234 --trace-to json:${TRACE_APP}.json + # test-runner-run/run1/script-args: --commissioning-method on-network --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto + # test-runner-run/run1/factoryreset: True + ''' - def assertMetadataParse(self, file_content: str, expected: List[Metadata]): - with tempfile.NamedTemporaryFile(mode='w', delete=False) as fp: + env_file_content = ''' + ALL_CLUSTERS_APP: out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app + CHIP_LOCK_APP: out/linux-x64-lock-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-lock-app + ENERGY_MANAGEMENT_APP: out/linux-x64-energy-management-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-energy-management-app + TRACE_APP: out/trace_data/app-{SCRIPT_BASE_NAME} + TRACE_TEST_JSON: out/trace_data/test-{SCRIPT_BASE_NAME} + TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME} + ''' + + def generate_temp_file(self, directory: str, file_content: str) -> str: + fd, temp_file_path = tempfile.mkstemp(dir=directory) + with os.fdopen(fd, 'w') as fp: fp.write(file_content) - fp.close() - for e in expected: - e.py_script_path = fp.name - actual = self.reader.parse_script(fp.name) - self.assertEqual(actual, expected) - - def test_parse_single_run(self): - self.assertMetadataParse(''' - # test-runner-runs: run1 - # test-runner-run/run1: app/all-clusters discriminator passcode - ''', - [ - Metadata(app="out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app", - discriminator=1234, run="run1", passcode=20202021) - ] - ) + return temp_file_path + + def generate_run_commands(self, reader: MetadataReader, script_path: str) -> List[str]: + metadata_list = reader.parse_script(script_path) + commands = [] + + for metadata in metadata_list: + cmd = ( + "scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py " + f"--app {metadata.app} " + f"{'--factoryreset' if metadata.factoryreset else ''} " + f"--app-args \"{metadata.app_args}\" " + f"--script \"{metadata.py_script_path}\" " + f"--script-args \"{metadata.script_args}\"'" + ) + commands.append(cmd.strip()) + + return commands + + def test_run_arg_generation(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_file = self.generate_temp_file(temp_dir, self.test_file_content) + env_file = self.generate_temp_file(temp_dir, self.env_file_content) + + reader = MetadataReader(env_file) + self.maxDiff = None + + test_file_expected_arg_string = ( + "scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py " + "--app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app " + "--factoryreset --app-args \"--discriminator 1234 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json\" " + "--script \"" + temp_file + "\" --script-args \"--commissioning-method on-network " + "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto\"'" + ) + + actual = self.generate_run_commands(reader, temp_file)[0] + self.assertEqual(test_file_expected_arg_string, actual) if __name__ == "__main__": From 6d1adda539a34d7f1488eeda4de0e3f569a467cf Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 13 Jun 2024 11:25:04 -0400 Subject: [PATCH 06/18] Fixed the Syntax error Unterminated quoted string --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 61e34704ee7cb4..078328db955ddc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -525,7 +525,7 @@ jobs: scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json --enable-key 000102030405060708090a0b0c0d0e0f" --script "src/python_testing/TC_IDM_1_4.py" --script-args "--hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_PWRTL_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_SC_3_6.py + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script "src/python_testing/TC_SC_3_6.py" scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_10.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_11.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' From e93f9df347b0b2d627d334a33051d42b20773401 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 13 Jun 2024 15:25:48 +0000 Subject: [PATCH 07/18] Restyled by autopep8 --- scripts/tests/py/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index 5d5b32d76dd49c..9b0ba0dee17d1a 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -67,10 +67,10 @@ def test_run_arg_generation(self): with tempfile.TemporaryDirectory() as temp_dir: temp_file = self.generate_temp_file(temp_dir, self.test_file_content) env_file = self.generate_temp_file(temp_dir, self.env_file_content) - + reader = MetadataReader(env_file) self.maxDiff = None - + test_file_expected_arg_string = ( "scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py " "--app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app " From ec4f6922e1e067ae93bd984a6dd47c1109f52e57 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 13 Jun 2024 15:25:50 +0000 Subject: [PATCH 08/18] Restyled by isort --- scripts/tests/py/test_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index 9b0ba0dee17d1a..f38a5ac4beb15b 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -13,9 +13,9 @@ # limitations under the License. import os +import tempfile import unittest from os import path -import tempfile from typing import List from metadata import Metadata, MetadataReader From daf8a57fb4a8f5091be163d387223eb57d917357 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 13 Jun 2024 13:07:07 -0400 Subject: [PATCH 09/18] Fixed the quoted string error --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 078328db955ddc..b7b1dbd4a76e4f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -525,7 +525,7 @@ jobs: scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json --enable-key 000102030405060708090a0b0c0d0e0f" --script "src/python_testing/TC_IDM_1_4.py" --script-args "--hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_PWRTL_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script "src/python_testing/TC_SC_3_6.py" + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_SC_3_6.py' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_10.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --quiet --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_2_11.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' From 889aded3a6a7a326bf504d97d3aa899be20d1643 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 13 Jun 2024 15:47:44 -0400 Subject: [PATCH 10/18] did ruff clean --- scripts/tests/py/metadata.py | 2 +- scripts/tests/py/test_metadata.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/tests/py/metadata.py b/scripts/tests/py/metadata.py index 2de3d9965d4335..46ac1fe9ec8ab8 100644 --- a/scripts/tests/py/metadata.py +++ b/scripts/tests/py/metadata.py @@ -15,7 +15,7 @@ import re from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List import yaml diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index f38a5ac4beb15b..d878220e3b85e7 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -15,10 +15,9 @@ import os import tempfile import unittest -from os import path from typing import List -from metadata import Metadata, MetadataReader +from metadata import MetadataReader class TestMetadataReader(unittest.TestCase): From 940ac3fd5367a4e93df4ea91c0ea2d28d6be26d9 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 13 Jun 2024 15:59:48 -0400 Subject: [PATCH 11/18] rename scripts/tests/yaml to scripts/tests/chipyaml and replace the imports for yaml.paths_finder to chipyaml.paths_finder --- scripts/tests/{yaml => chipyaml}/__init__.py | 0 scripts/tests/{yaml => chipyaml}/chiptool.py | 0 scripts/tests/{yaml => chipyaml}/darwinframeworktool.py | 0 scripts/tests/{yaml => chipyaml}/extensions/__init__.py | 0 scripts/tests/{yaml => chipyaml}/extensions/example_cluster.py | 0 .../extensions/wildcard_response_extractor_cluster.py | 0 scripts/tests/{yaml => chipyaml}/paths_finder.py | 0 scripts/tests/{yaml => chipyaml}/relative_importer.py | 0 scripts/tests/{yaml => chipyaml}/runner.py | 0 scripts/tests/{yaml => chipyaml}/tests_finder.py | 0 scripts/tests/{yaml => chipyaml}/tests_logger.py | 0 scripts/tests/{yaml => chipyaml}/tests_tool.py | 0 scripts/tests/run_darwin_framework_ota_test.py | 2 +- scripts/tests/run_test_suite.py | 2 +- 14 files changed, 2 insertions(+), 2 deletions(-) rename scripts/tests/{yaml => chipyaml}/__init__.py (100%) rename scripts/tests/{yaml => chipyaml}/chiptool.py (100%) rename scripts/tests/{yaml => chipyaml}/darwinframeworktool.py (100%) rename scripts/tests/{yaml => chipyaml}/extensions/__init__.py (100%) rename scripts/tests/{yaml => chipyaml}/extensions/example_cluster.py (100%) rename scripts/tests/{yaml => chipyaml}/extensions/wildcard_response_extractor_cluster.py (100%) rename scripts/tests/{yaml => chipyaml}/paths_finder.py (100%) rename scripts/tests/{yaml => chipyaml}/relative_importer.py (100%) rename scripts/tests/{yaml => chipyaml}/runner.py (100%) rename scripts/tests/{yaml => chipyaml}/tests_finder.py (100%) rename scripts/tests/{yaml => chipyaml}/tests_logger.py (100%) rename scripts/tests/{yaml => chipyaml}/tests_tool.py (100%) diff --git a/scripts/tests/yaml/__init__.py b/scripts/tests/chipyaml/__init__.py similarity index 100% rename from scripts/tests/yaml/__init__.py rename to scripts/tests/chipyaml/__init__.py diff --git a/scripts/tests/yaml/chiptool.py b/scripts/tests/chipyaml/chiptool.py similarity index 100% rename from scripts/tests/yaml/chiptool.py rename to scripts/tests/chipyaml/chiptool.py diff --git a/scripts/tests/yaml/darwinframeworktool.py b/scripts/tests/chipyaml/darwinframeworktool.py similarity index 100% rename from scripts/tests/yaml/darwinframeworktool.py rename to scripts/tests/chipyaml/darwinframeworktool.py diff --git a/scripts/tests/yaml/extensions/__init__.py b/scripts/tests/chipyaml/extensions/__init__.py similarity index 100% rename from scripts/tests/yaml/extensions/__init__.py rename to scripts/tests/chipyaml/extensions/__init__.py diff --git a/scripts/tests/yaml/extensions/example_cluster.py b/scripts/tests/chipyaml/extensions/example_cluster.py similarity index 100% rename from scripts/tests/yaml/extensions/example_cluster.py rename to scripts/tests/chipyaml/extensions/example_cluster.py diff --git a/scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py b/scripts/tests/chipyaml/extensions/wildcard_response_extractor_cluster.py similarity index 100% rename from scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py rename to scripts/tests/chipyaml/extensions/wildcard_response_extractor_cluster.py diff --git a/scripts/tests/yaml/paths_finder.py b/scripts/tests/chipyaml/paths_finder.py similarity index 100% rename from scripts/tests/yaml/paths_finder.py rename to scripts/tests/chipyaml/paths_finder.py diff --git a/scripts/tests/yaml/relative_importer.py b/scripts/tests/chipyaml/relative_importer.py similarity index 100% rename from scripts/tests/yaml/relative_importer.py rename to scripts/tests/chipyaml/relative_importer.py diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/chipyaml/runner.py similarity index 100% rename from scripts/tests/yaml/runner.py rename to scripts/tests/chipyaml/runner.py diff --git a/scripts/tests/yaml/tests_finder.py b/scripts/tests/chipyaml/tests_finder.py similarity index 100% rename from scripts/tests/yaml/tests_finder.py rename to scripts/tests/chipyaml/tests_finder.py diff --git a/scripts/tests/yaml/tests_logger.py b/scripts/tests/chipyaml/tests_logger.py similarity index 100% rename from scripts/tests/yaml/tests_logger.py rename to scripts/tests/chipyaml/tests_logger.py diff --git a/scripts/tests/yaml/tests_tool.py b/scripts/tests/chipyaml/tests_tool.py similarity index 100% rename from scripts/tests/yaml/tests_tool.py rename to scripts/tests/chipyaml/tests_tool.py diff --git a/scripts/tests/run_darwin_framework_ota_test.py b/scripts/tests/run_darwin_framework_ota_test.py index 672b26369d3c3e..534a688cbed544 100755 --- a/scripts/tests/run_darwin_framework_ota_test.py +++ b/scripts/tests/run_darwin_framework_ota_test.py @@ -10,7 +10,7 @@ from chiptest.accessories import AppsRegister from chiptest.runner import Runner from chiptest.test_definition import App, ExecutionCapture -from yaml.paths_finder import PathsFinder +from chipyaml.paths_finder import PathsFinder TEST_NODE_ID = '0x12344321' TEST_VID = '0xFFF1' diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 2df88cd6740653..c22b4abff92f09 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -28,7 +28,7 @@ from chiptest.accessories import AppsRegister from chiptest.glob_matcher import GlobMatcher from chiptest.test_definition import TestRunTime, TestTag -from yaml.paths_finder import PathsFinder +from chipyaml.paths_finder import PathsFinder DEFAULT_CHIP_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) From 7f29f1d2ce176e7e47a1cfb08074c1e7d52b0709 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 13 Jun 2024 17:45:40 -0400 Subject: [PATCH 12/18] Adding the rename changes I had in the new PR to make it clean and independent --- scripts/tests/run_darwin_framework_ota_test.py | 2 +- scripts/tests/run_test_suite.py | 2 +- scripts/tests/{chipyaml => yaml}/__init__.py | 0 scripts/tests/{chipyaml => yaml}/chiptool.py | 0 scripts/tests/{chipyaml => yaml}/darwinframeworktool.py | 0 scripts/tests/{chipyaml => yaml}/extensions/__init__.py | 0 scripts/tests/{chipyaml => yaml}/extensions/example_cluster.py | 0 .../extensions/wildcard_response_extractor_cluster.py | 0 scripts/tests/{chipyaml => yaml}/paths_finder.py | 0 scripts/tests/{chipyaml => yaml}/relative_importer.py | 0 scripts/tests/{chipyaml => yaml}/runner.py | 0 scripts/tests/{chipyaml => yaml}/tests_finder.py | 0 scripts/tests/{chipyaml => yaml}/tests_logger.py | 0 scripts/tests/{chipyaml => yaml}/tests_tool.py | 0 14 files changed, 2 insertions(+), 2 deletions(-) rename scripts/tests/{chipyaml => yaml}/__init__.py (100%) rename scripts/tests/{chipyaml => yaml}/chiptool.py (100%) rename scripts/tests/{chipyaml => yaml}/darwinframeworktool.py (100%) rename scripts/tests/{chipyaml => yaml}/extensions/__init__.py (100%) rename scripts/tests/{chipyaml => yaml}/extensions/example_cluster.py (100%) rename scripts/tests/{chipyaml => yaml}/extensions/wildcard_response_extractor_cluster.py (100%) rename scripts/tests/{chipyaml => yaml}/paths_finder.py (100%) rename scripts/tests/{chipyaml => yaml}/relative_importer.py (100%) rename scripts/tests/{chipyaml => yaml}/runner.py (100%) rename scripts/tests/{chipyaml => yaml}/tests_finder.py (100%) rename scripts/tests/{chipyaml => yaml}/tests_logger.py (100%) rename scripts/tests/{chipyaml => yaml}/tests_tool.py (100%) diff --git a/scripts/tests/run_darwin_framework_ota_test.py b/scripts/tests/run_darwin_framework_ota_test.py index 534a688cbed544..672b26369d3c3e 100755 --- a/scripts/tests/run_darwin_framework_ota_test.py +++ b/scripts/tests/run_darwin_framework_ota_test.py @@ -10,7 +10,7 @@ from chiptest.accessories import AppsRegister from chiptest.runner import Runner from chiptest.test_definition import App, ExecutionCapture -from chipyaml.paths_finder import PathsFinder +from yaml.paths_finder import PathsFinder TEST_NODE_ID = '0x12344321' TEST_VID = '0xFFF1' diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index c22b4abff92f09..2df88cd6740653 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -28,7 +28,7 @@ from chiptest.accessories import AppsRegister from chiptest.glob_matcher import GlobMatcher from chiptest.test_definition import TestRunTime, TestTag -from chipyaml.paths_finder import PathsFinder +from yaml.paths_finder import PathsFinder DEFAULT_CHIP_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) diff --git a/scripts/tests/chipyaml/__init__.py b/scripts/tests/yaml/__init__.py similarity index 100% rename from scripts/tests/chipyaml/__init__.py rename to scripts/tests/yaml/__init__.py diff --git a/scripts/tests/chipyaml/chiptool.py b/scripts/tests/yaml/chiptool.py similarity index 100% rename from scripts/tests/chipyaml/chiptool.py rename to scripts/tests/yaml/chiptool.py diff --git a/scripts/tests/chipyaml/darwinframeworktool.py b/scripts/tests/yaml/darwinframeworktool.py similarity index 100% rename from scripts/tests/chipyaml/darwinframeworktool.py rename to scripts/tests/yaml/darwinframeworktool.py diff --git a/scripts/tests/chipyaml/extensions/__init__.py b/scripts/tests/yaml/extensions/__init__.py similarity index 100% rename from scripts/tests/chipyaml/extensions/__init__.py rename to scripts/tests/yaml/extensions/__init__.py diff --git a/scripts/tests/chipyaml/extensions/example_cluster.py b/scripts/tests/yaml/extensions/example_cluster.py similarity index 100% rename from scripts/tests/chipyaml/extensions/example_cluster.py rename to scripts/tests/yaml/extensions/example_cluster.py diff --git a/scripts/tests/chipyaml/extensions/wildcard_response_extractor_cluster.py b/scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py similarity index 100% rename from scripts/tests/chipyaml/extensions/wildcard_response_extractor_cluster.py rename to scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py diff --git a/scripts/tests/chipyaml/paths_finder.py b/scripts/tests/yaml/paths_finder.py similarity index 100% rename from scripts/tests/chipyaml/paths_finder.py rename to scripts/tests/yaml/paths_finder.py diff --git a/scripts/tests/chipyaml/relative_importer.py b/scripts/tests/yaml/relative_importer.py similarity index 100% rename from scripts/tests/chipyaml/relative_importer.py rename to scripts/tests/yaml/relative_importer.py diff --git a/scripts/tests/chipyaml/runner.py b/scripts/tests/yaml/runner.py similarity index 100% rename from scripts/tests/chipyaml/runner.py rename to scripts/tests/yaml/runner.py diff --git a/scripts/tests/chipyaml/tests_finder.py b/scripts/tests/yaml/tests_finder.py similarity index 100% rename from scripts/tests/chipyaml/tests_finder.py rename to scripts/tests/yaml/tests_finder.py diff --git a/scripts/tests/chipyaml/tests_logger.py b/scripts/tests/yaml/tests_logger.py similarity index 100% rename from scripts/tests/chipyaml/tests_logger.py rename to scripts/tests/yaml/tests_logger.py diff --git a/scripts/tests/chipyaml/tests_tool.py b/scripts/tests/yaml/tests_tool.py similarity index 100% rename from scripts/tests/chipyaml/tests_tool.py rename to scripts/tests/yaml/tests_tool.py From 70c82525d2eb8ef30e8723f491a1f722c178c635 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Mon, 17 Jun 2024 15:30:30 -0400 Subject: [PATCH 13/18] Updates and Fixes to the Script --- scripts/tests/py/test_metadata.py | 42 +++++++++++++------------------ scripts/tests/run_python_test.py | 2 +- src/python_testing/TC_SC_3_6.py | 1 + 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index d878220e3b85e7..c7a8df35bc7cce 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -13,12 +13,11 @@ # limitations under the License. import os -import tempfile import unittest +import tempfile from typing import List -from metadata import MetadataReader - +from metadata import MetadataReader, Metadata class TestMetadataReader(unittest.TestCase): @@ -39,37 +38,29 @@ class TestMetadataReader(unittest.TestCase): TRACE_TEST_PERFETTO: out/trace_data/test-{SCRIPT_BASE_NAME} ''' + expected_metadata = Metadata( + script_args="--commissioning-method on-network --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto", + py_script_path="", + app_args="--discriminator 1234 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json", + run="run1", + app="out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app", + factoryreset=True + ) + def generate_temp_file(self, directory: str, file_content: str) -> str: fd, temp_file_path = tempfile.mkstemp(dir=directory) with os.fdopen(fd, 'w') as fp: fp.write(file_content) return temp_file_path - def generate_run_commands(self, reader: MetadataReader, script_path: str) -> List[str]: - metadata_list = reader.parse_script(script_path) - commands = [] - - for metadata in metadata_list: - cmd = ( - "scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py " - f"--app {metadata.app} " - f"{'--factoryreset' if metadata.factoryreset else ''} " - f"--app-args \"{metadata.app_args}\" " - f"--script \"{metadata.py_script_path}\" " - f"--script-args \"{metadata.script_args}\"'" - ) - commands.append(cmd.strip()) - - return commands - def test_run_arg_generation(self): with tempfile.TemporaryDirectory() as temp_dir: temp_file = self.generate_temp_file(temp_dir, self.test_file_content) env_file = self.generate_temp_file(temp_dir, self.env_file_content) - + reader = MetadataReader(env_file) self.maxDiff = None - + test_file_expected_arg_string = ( "scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py " "--app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app " @@ -77,9 +68,10 @@ def test_run_arg_generation(self): "--script \"" + temp_file + "\" --script-args \"--commissioning-method on-network " "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto\"'" ) - - actual = self.generate_run_commands(reader, temp_file)[0] - self.assertEqual(test_file_expected_arg_string, actual) + + self.expected_metadata.py_script_path = temp_file + actual = reader.parse_script(temp_file)[0] + self.assertEqual(self.expected_metadata, actual) if __name__ == "__main__": diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index 836c1e0ac6bbf1..5ccd67a987a401 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -111,7 +111,7 @@ def main(app: str, factoryreset: bool, factoryreset_app_only: bool, app_args: st ] for run in runs: - print(f"Executing run: {run.py_script_path}") + print(f"Executing {run.py_script_path.split('/')[-1]} {run.run}") main_impl(run.app, run.factoryreset, run.factoryreset_app_only, run.app_args, run.py_script_path, run.script_args, run.script_gdb, run.quiet) diff --git a/src/python_testing/TC_SC_3_6.py b/src/python_testing/TC_SC_3_6.py index cdcf3b9bde7388..9247cb546e19c6 100644 --- a/src/python_testing/TC_SC_3_6.py +++ b/src/python_testing/TC_SC_3_6.py @@ -18,6 +18,7 @@ # test-runner-runs: run1 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP} # test-runner-run/run1/factoryreset: True +# test-runner-run/run1/quiet: True # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json # test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto From 7f15f52e82bd42ec7f1c15cc3bbd0ba0e42b17b2 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Tue, 18 Jun 2024 07:36:14 -0400 Subject: [PATCH 14/18] Fixed latest tests.yaml and test_metadata.py changes --- .github/workflows/tests.yaml | 2 +- scripts/tests/py/test_metadata.py | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 932635ec69b4ad..1c422a472c6bf2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -477,7 +477,7 @@ jobs: " - name: Generate an argument environment file run: | - touch /tmp/test_env.yaml + echo -n "" >/tmp/test_env.yaml echo "ALL_CLUSTERS_APP: out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app" >> /tmp/test_env.yaml echo "CHIP_LOCK_APP: out/linux-x64-lock-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-lock-app" >> /tmp/test_env.yaml echo "ENERGY_MANAGEMENT_APP: out/linux-x64-energy-management-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-energy-management-app" >> /tmp/test_env.yaml diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index c7a8df35bc7cce..025b35893f85fd 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -27,6 +27,7 @@ class TestMetadataReader(unittest.TestCase): # test-runner-run/run1/app-args: --discriminator 1234 --trace-to json:${TRACE_APP}.json # test-runner-run/run1/script-args: --commissioning-method on-network --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # test-runner-run/run1/factoryreset: True + # test-runner-run/run1/quiet: True ''' env_file_content = ''' @@ -44,7 +45,8 @@ class TestMetadataReader(unittest.TestCase): app_args="--discriminator 1234 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json", run="run1", app="out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app", - factoryreset=True + factoryreset=True, + quiet=True ) def generate_temp_file(self, directory: str, file_content: str) -> str: @@ -61,14 +63,6 @@ def test_run_arg_generation(self): reader = MetadataReader(env_file) self.maxDiff = None - test_file_expected_arg_string = ( - "scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py " - "--app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app " - "--factoryreset --app-args \"--discriminator 1234 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json\" " - "--script \"" + temp_file + "\" --script-args \"--commissioning-method on-network " - "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto\"'" - ) - self.expected_metadata.py_script_path = temp_file actual = reader.parse_script(temp_file)[0] self.assertEqual(self.expected_metadata, actual) From a76c5daa467bb1834e621ec0ee4dc121a79d8180 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 18 Jun 2024 11:36:37 +0000 Subject: [PATCH 15/18] Restyled by autopep8 --- scripts/tests/py/test_metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index 025b35893f85fd..5bf9761aa9c4c7 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -19,6 +19,7 @@ from metadata import MetadataReader, Metadata + class TestMetadataReader(unittest.TestCase): test_file_content = ''' @@ -47,7 +48,7 @@ class TestMetadataReader(unittest.TestCase): app="out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app", factoryreset=True, quiet=True - ) + ) def generate_temp_file(self, directory: str, file_content: str) -> str: fd, temp_file_path = tempfile.mkstemp(dir=directory) @@ -59,10 +60,10 @@ def test_run_arg_generation(self): with tempfile.TemporaryDirectory() as temp_dir: temp_file = self.generate_temp_file(temp_dir, self.test_file_content) env_file = self.generate_temp_file(temp_dir, self.env_file_content) - + reader = MetadataReader(env_file) self.maxDiff = None - + self.expected_metadata.py_script_path = temp_file actual = reader.parse_script(temp_file)[0] self.assertEqual(self.expected_metadata, actual) From 118245e696848ffa364403102743e355416e3b9e Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 18 Jun 2024 11:36:37 +0000 Subject: [PATCH 16/18] Restyled by isort --- scripts/tests/py/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index 5bf9761aa9c4c7..a817d8cb9bca3e 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -13,11 +13,11 @@ # limitations under the License. import os -import unittest import tempfile +import unittest from typing import List -from metadata import MetadataReader, Metadata +from metadata import Metadata, MetadataReader class TestMetadataReader(unittest.TestCase): From 72e3527a05c5acf4ccc0dc4770f65241934a5dd0 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Tue, 18 Jun 2024 08:52:41 -0400 Subject: [PATCH 17/18] Removed extra import typing.list as per code linter --- scripts/tests/py/test_metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index a817d8cb9bca3e..e72b06885933c9 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -15,11 +15,9 @@ import os import tempfile import unittest -from typing import List from metadata import Metadata, MetadataReader - class TestMetadataReader(unittest.TestCase): test_file_content = ''' From ece8fe414aeb4840234e27a0db6e1d1dee5dd348 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 18 Jun 2024 12:53:00 +0000 Subject: [PATCH 18/18] Restyled by autopep8 --- scripts/tests/py/test_metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/py/test_metadata.py b/scripts/tests/py/test_metadata.py index e72b06885933c9..a0c12a0ab0ed5a 100644 --- a/scripts/tests/py/test_metadata.py +++ b/scripts/tests/py/test_metadata.py @@ -18,6 +18,7 @@ from metadata import Metadata, MetadataReader + class TestMetadataReader(unittest.TestCase): test_file_content = '''