diff --git a/examples/chef/chef.py b/examples/chef/chef.py index 5fdb6bc42610d2..ece284f34f4e7f 100755 --- a/examples/chef/chef.py +++ b/examples/chef/chef.py @@ -27,6 +27,7 @@ import constants import stateful_shell +import chef_util from typing import Sequence @@ -172,6 +173,7 @@ def main(argv: Sequence[str]) -> None: dest="generate_zzz", action="store_true") parser.add_option("", "--validate_zzz", help="Checks if cached ZAP output needs to be regenrated, for use in CI. If this flag is set, all other arguments are ignored.", dest="validate_zzz", action="store_true") parser.add_option("", "--use_zzz", help="Use pre generated output from the ZAP tool found in the zzz_generated folder. Used to decrease execution time of CI jobs", dest="use_zzz", action="store_true") + parser.add_option("", "--build_all", help="Builds all chef examples across all platforms and collates artifacts. Chef exits after completion.", dest="build_all", action="store_true") options, _ = parser.parse_args(argv) @@ -180,32 +182,13 @@ def main(argv: Sequence[str]) -> None: chef_zzz_root = os.path.join(_CHEF_SCRIPT_PATH, "zzz_generated") ci_manifest_file_name = os.path.join(_CHEF_SCRIPT_PATH, "cimanifest.json") chef_devices_dir = os.path.join(_CHEF_SCRIPT_PATH, "devices") - ci_allowlist = ['lighting-app.zap'] # # Validate zzz_generated # if options.validate_zzz: - ci_manifest = {} - print(f"Validating {chef_zzz_root}") - for device_dir_item in os.listdir(chef_devices_dir): - target_file_ext = ".zap" - if device_dir_item.endswith(target_file_ext) and device_dir_item in ci_allowlist: - device_name = device_dir_item[:-len(target_file_ext)] - device_file_path = os.path.join(chef_devices_dir, device_dir_item) - with open(device_file_path, "rb") as device_file: - device_file_data = device_file.read() - device_file_md5 = hashlib.md5(device_file_data).hexdigest() - ci_manifest[device_name] = device_file_md5 - print(f"Manifest for {device_name} : {device_file_md5}") - # git ls-tree fails in CI - # git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"] - # zap_commit = str(subprocess.check_output(git_cmd, cwd=_REPO_BASE_PATH)) - # zap_commit = zap_commit.split(" ")[2] - # zap_commit = zap_commit[:zap_commit.index("\\")] - # print(f"zap commit: {zap_commit}") - # ci_manifest["zap_commit"] = zap_commit + ci_manifest = chef_util.generate_device_manifest(chef_devices_dir) with open(ci_manifest_file_name, "r", encoding="utf-8") as ci_manifest_file: cached_manifest = json.loads(ci_manifest_file.read()) for device in ci_manifest: @@ -218,6 +201,7 @@ def main(argv: Sequence[str]) -> None: if cached_manifest[device] != ci_manifest[device]: print("ZAP updated in master, please pull master and run chef with the flag --generate_zzz and commit") exit(1) + print("Cached ZAP output is up to date!") exit(0) # @@ -246,7 +230,6 @@ def main(argv: Sequence[str]) -> None: # if options.generate_zzz: - ci_manifest = {} print(f"Cleaning {chef_zzz_root}") if not os.path.exists(chef_zzz_root): print(f"{chef_zzz_root} doesn't exist; creating") @@ -258,15 +241,8 @@ def main(argv: Sequence[str]) -> None: print(f"Generating files in {chef_zzz_root} for all devices") for device_dir_item in os.listdir(chef_devices_dir): target_file_ext = ".zap" - if device_dir_item.endswith(target_file_ext) and device_dir_item in ci_allowlist: - # Filter to one example + if device_dir_item.endswith(target_file_ext) and device_dir_item in chef_util.ci_allowlist: device_name = device_dir_item[:-len(target_file_ext)] - device_file_path = os.path.join(chef_devices_dir, device_dir_item) - with open(device_file_path, "rb") as device_file: - device_file_data = device_file.read() - device_file_md5 = hashlib.md5(device_file_data).hexdigest() - ci_manifest[device_name] = device_file_md5 - print(f"Manifest for {device_name} : {device_file_md5}") print(f"Generating files for {device_name}") device_out_dir = os.path.join(chef_zzz_root, device_name) os.mkdir(device_out_dir) @@ -275,14 +251,13 @@ def main(argv: Sequence[str]) -> None: shell.run_cmd(f"{_REPO_BASE_PATH}/scripts/tools/zap/generate.py\ {_CHEF_SCRIPT_PATH}/devices/{device_name}.zap -o {device_out_dir}") shell.run_cmd(f"touch {device_out_dir}/af-gen-event.h") - with open(ci_manifest_file_name, "w+", encoding="utf-8") as ci_manifest_file: - git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"] - zap_commit = str(subprocess.check_output(git_cmd, cwd=_REPO_BASE_PATH)) - zap_commit = zap_commit.split(" ")[2] - zap_commit = zap_commit[:zap_commit.index("\\")] - print(f"zap commit: {zap_commit}") - ci_manifest["zap_commit"] = zap_commit - ci_manifest_file.write(json.dumps(ci_manifest, indent=4)) + chef_util.generate_device_manifest(chef_devices_dir, include_zap_submod=True, write_manifest_file = True, ci_manifest_file_name = ci_manifest_file_name, repo_base_path=_REPO_BASE_PATH) + exit(0) + # + # Build all + # + + if options.build_all: exit(0) # diff --git a/examples/chef/chef_util.py b/examples/chef/chef_util.py new file mode 100644 index 00000000000000..b675db8d802598 --- /dev/null +++ b/examples/chef/chef_util.py @@ -0,0 +1,40 @@ +import subprocess +import json +import os +import hashlib + +ci_allowlist = ['lighting-app.zap'] +cd_output_dirs = { + 'linux': 'linux/out', + 'esp32': 'esp32/build', + 'nrfconnect': 'nrfconnect/build', +} + +def check_zap_master(repo_base_path: str) -> str: + """Produces hash of ZAP submodule in branch master""" + git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"] + zap_commit = str(subprocess.check_output(git_cmd, cwd=repo_base_path)) + zap_commit = zap_commit.split(" ")[2] + zap_commit = zap_commit[:zap_commit.index("\\")] + print(f"zap commit: {zap_commit}") + return zap_commit + +def generate_device_manifest(chef_devices_dir: str, include_zap_submod: bool = False, write_manifest_file: bool = False, ci_manifest_file_name: str = '', repo_base_path: str = '') -> dict: + """Produces dictionary containing md5 of device dir zap files""" + ci_manifest = {} + for device_dir_item in os.listdir(chef_devices_dir): + target_file_ext = ".zap" + if device_dir_item.endswith(target_file_ext) and device_dir_item in ci_allowlist: + device_name = device_dir_item[:-len(target_file_ext)] + device_file_path = os.path.join(chef_devices_dir, device_dir_item) + with open(device_file_path, "rb") as device_file: + device_file_data = device_file.read() + device_file_md5 = hashlib.md5(device_file_data).hexdigest() + ci_manifest[device_name] = device_file_md5 + print(f"Manifest for {device_name} : {device_file_md5}") + if include_zap_submod: + ci_manifest["zap_commit"] = check_zap_master(repo_base_path) + if write_manifest_file: + with open(ci_manifest_file_name, "w+", encoding="utf-8") as ci_manifest_file: + ci_manifest_file.write(json.dumps(ci_manifest, indent=4)) + return ci_manifest