From 3f826ff28c41944d8895f0c7cc7a886d37b82263 Mon Sep 17 00:00:00 2001 From: Austin Bozowski Date: Fri, 27 May 2022 19:07:38 +0000 Subject: [PATCH] Begin refactor --- examples/chef/chef.py | 117 +++++++++++++++++++++++------------ examples/chef/chef_util.py | 70 --------------------- examples/chef/cicd_meta.json | 14 +++++ 3 files changed, 92 insertions(+), 109 deletions(-) delete mode 100644 examples/chef/chef_util.py create mode 100644 examples/chef/cicd_meta.json diff --git a/examples/chef/chef.py b/examples/chef/chef.py index 1341a577cfde3a..b6b08af34ddd35 100755 --- a/examples/chef/chef.py +++ b/examples/chef/chef.py @@ -23,11 +23,11 @@ import sys import tarfile import textwrap +from typing import Any, Dict from typing import Sequence import yaml -import chef_util import constants import stateful_shell @@ -41,7 +41,7 @@ _DEVICE_LIST = [file[:-4] for file in os.listdir(_DEVICE_FOLDER) if file.endswith(".zap")] _CHEF_ZZZ_ROOT = os.path.join(_CHEF_SCRIPT_PATH, "zzz_generated") _CI_MANIFEST_FILE_NAME = os.path.join(_CHEF_SCRIPT_PATH, "cimanifest.json") -_CHEF_DEVICES_DIR = os.path.join(_CHEF_SCRIPT_PATH, "devices") +_CI_ALLOW_LIST = ["lighting-app"] gen_dir = "" # Filled in after sample app type is read from args. @@ -97,6 +97,52 @@ def check_python_version() -> None: exit(1) +def check_zap_master() -> str: + """Produces hash of ZAP submodule in branch master""" + git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"] + zap_commit = str(subprocess.check_output(git_cmd, cwd=_REPO_BASE_PATH)) + zap_commit = zap_commit.split(" ")[2] + zap_commit = zap_commit[:zap_commit.index("\\")] + print(f"zap commit: {zap_commit}") + return zap_commit + + +def generate_device_manifest( + include_zap_submod: bool = False, + write_manifest_file: bool = False) -> Dict[str, Any]: + """Produces dictionary containing md5 of device dir zap files. + + Args: + include_zap_submod: Include ZAP commit in manifest + write_manifest_file: Serialize manifest + Returns: + Dict containing MD5 of device dir zap files. + """ + ci_manifest = {} + for device_name in _DEVICE_LIST: + device_file_path = os.path.join(_CHEF_DEVICES_DIR, device_dir_item) + with open(device_file_path, "rb") as device_file: + device_file_data = device_file.read() + device_file_md5 = hashlib.md5(device_file_data).hexdigest() + ci_manifest[device_name] = device_file_md5 + print(f"Manifest for {device_name} : {device_file_md5}") + if write_manifest_file: + device_zzz_dir_root = os.path.join(_CHEF_ZZZ_ROOT, device_name) + device_zzz_md5_file = os.path.join(device_zzz_dir_root, 'INPUTMD5.txt') + with open(device_zzz_md5_file, "w+", encoding="utf-8") as md5file: + md5file.write(device_file_md5) + if include_zap_submod: + ci_manifest["zap_commit"] = check_zap_master(_REPO_BASE_PATH) + if write_manifest_file: + with open(_CI_MANIFEST_FILE_NAME, "w+", encoding="utf-8") as ci_manifest_file: + ci_manifest_file.write(json.dumps(ci_manifest, indent=4)+"\n") + return ci_manifest + + +def load_cicd_config() -> dict: + pass + + def main(argv: Sequence[str]) -> None: check_python_version() config = load_config() @@ -191,11 +237,11 @@ def main(argv: Sequence[str]) -> None: # if options.validate_zzz: - fix_instructions = "Cached files out of date. Please bootstrap, activate, install zap, run chef with the flag --generate_zzz and commit /examples/chef/zzz_generated and /examples/chef/cimanifes.json" + fix_instructions = "Cached files out of date. Please bootstrap, activate, install zap, run chef with the flag --generate_zzz and commit /examples/chef/zzz_generated and /examples/chef/cimanifes.json. Ensure you are running with the latest version of ZAP from master" ci_manifest = chef_util.generate_device_manifest(_CHEF_DEVICES_DIR) with open(_CI_MANIFEST_FILE_NAME, "r", encoding="utf-8") as ci_manifest_file: cached_manifest = json.loads(ci_manifest_file.read()) - for device in ci_manifest: + for device in _CI_MANIFEST: if device != "zap_commit": try: if cached_manifest[device] != ci_manifest[device]: @@ -260,22 +306,19 @@ def main(argv: Sequence[str]) -> None: shutil.rmtree(_CHEF_ZZZ_ROOT) os.mkdir(_CHEF_ZZZ_ROOT) print(f"Generating files in {_CHEF_ZZZ_ROOT} for all devices", flush=True) - for device_dir_item in os.listdir(_CHEF_DEVICES_DIR): - target_file_ext = ".zap" - if device_dir_item.endswith(target_file_ext): - device_name = device_dir_item[:-len(target_file_ext)] - print(f"Generating files for {device_name}", flush=True) - device_out_dir = os.path.join(_CHEF_ZZZ_ROOT, device_name) - os.mkdir(device_out_dir) - device_out_dir = os.path.join(device_out_dir, "zap-generated") - os.mkdir(device_out_dir) - shell.run_cmd(textwrap.dedent(f"""\ - {_REPO_BASE_PATH}/scripts/tools/zap/generate.py \ - {_CHEF_SCRIPT_PATH}/devices/{device_name}.zap -o {device_out_dir} - """)) - shell.run_cmd(f"touch {device_out_dir}/af-gen-event.h") - chef_util.generate_device_manifest(_CHEF_DEVICES_DIR, include_zap_submod=True, write_manifest_file=True, - ci_manifest_file_name=_CI_MANIFEST_FILE_NAME, repo_base_path=_REPO_BASE_PATH, chef_zzz_root=_CHEF_ZZZ_ROOT) + for device_name in _DEVICE_LIST: + print(f"Generating files for {device_name}", flush=True) + device_out_dir = os.path.join(_CHEF_ZZZ_ROOT, device_name) + os.mkdir(device_out_dir) + device_out_dir = os.path.join(device_out_dir, "zap-generated") + os.mkdir(device_out_dir) + shell.run_cmd(textwrap.dedent(f"""\ + {_REPO_BASE_PATH}/scripts/tools/zap/generate.py \ + {_CHEF_SCRIPT_PATH}/devices/{device_name}.zap -o {device_out_dir} + """)) + shell.run_cmd(f"touch {device_out_dir}/af-gen-event.h") + chef_util.generate_device_manifest(include_zap_submod=True, + write_manifest_file=True) exit(0) # @@ -302,29 +345,25 @@ def main(argv: Sequence[str]) -> None: os.environ['GNUARMEMB_TOOLCHAIN_PATH'] = os.environ['PW_ARM_CIPD_INSTALL_DIR'] archive_prefix = "/workspace/artifacts/" if not os.path.exists(archive_prefix): - # shutil.rmtree(archive_prefix) os.mkdir(archive_prefix) archive_suffix = ".tar.gz" cd_platforms_meta = chef_util.cd_platforms_meta - for device in os.listdir(_CHEF_DEVICES_DIR): - target_file_ext = ".zap" - if device.endswith(target_file_ext): - device_name = device[:-len(target_file_ext)] - for platform, platform_meta in cd_platforms_meta.items(): - directory = platform_meta['build_dir'] - label = platform_meta['platform_label'] - output_dir = os.path.join(_CHEF_SCRIPT_PATH, directory) - command = f"./chef.py -cbr --use_zzz -d {device_name} -t {platform}" - print('-' * 64, flush=True) - print(f"Building {command}", flush=True) - print('-' * 64, flush=True) - subprocess.check_call(command, cwd=_CHEF_SCRIPT_PATH, shell=True) - archive_name = f"{label}-chef-{device_name}-wifi-rpc" - archive_full_name = archive_prefix + archive_name + archive_suffix - print(f"Adding build output to archive {archive_full_name}", flush=True) - with tarfile.open(archive_full_name, "w:gz") as tar: - tar.add(output_dir, arcname=".") + for device_name in _DEVICE_LIST: + for platform, platform_meta in cd_platforms_meta.items(): + directory = platform_meta['build_dir'] + label = platform_meta['platform_label'] + output_dir = os.path.join(_CHEF_SCRIPT_PATH, directory) + command = f"./chef.py -cbr --use_zzz -d {device_name} -t {platform}" + print('-' * 64, flush=True) + print(f"Building {command}", flush=True) + print('-' * 64, flush=True) + subprocess.check_call(command, cwd=_CHEF_SCRIPT_PATH, shell=True) + archive_name = f"{label}-chef-{device_name}-wifi-rpc" + archive_full_name = archive_prefix + archive_name + archive_suffix + print(f"Adding build output to archive {archive_full_name}", flush=True) + with tarfile.open(archive_full_name, "w:gz") as tar: + tar.add(output_dir, arcname=".") exit(0) # diff --git a/examples/chef/chef_util.py b/examples/chef/chef_util.py deleted file mode 100644 index 476f39f402772a..00000000000000 --- a/examples/chef/chef_util.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2022 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json -import os -import subprocess - -ci_allowlist = ['lighting-app.zap'] -cd_platforms_meta = { - 'linux': { - 'build_dir': 'linux/out', - 'platform_label': 'linux_x86', - }, - 'esp32': { - 'build_dir': 'esp32/build', - 'platform_label': 'esp32-m5stack', - }, - 'nrfconnect': { - 'build_dir': 'nrfconnect/build', - 'platform_label': 'nrf-nrf52840dk', - }, -} - - -def check_zap_master(repo_base_path: str) -> str: - """Produces hash of ZAP submodule in branch master""" - git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"] - zap_commit = str(subprocess.check_output(git_cmd, cwd=repo_base_path)) - zap_commit = zap_commit.split(" ")[2] - zap_commit = zap_commit[:zap_commit.index("\\")] - print(f"zap commit: {zap_commit}") - return zap_commit - - -def generate_device_manifest(chef_devices_dir: str, include_zap_submod: bool = False, write_manifest_file: bool = False, ci_manifest_file_name: str = '', repo_base_path: str = '', chef_zzz_root: str = '') -> dict: - """Produces dictionary containing md5 of device dir zap files""" - ci_manifest = {} - for device_dir_item in os.listdir(chef_devices_dir): - target_file_ext = ".zap" - if device_dir_item.endswith(target_file_ext): - device_name = device_dir_item[:-len(target_file_ext)] - device_file_path = os.path.join(chef_devices_dir, device_dir_item) - with open(device_file_path, "rb") as device_file: - device_file_data = device_file.read() - device_file_md5 = hashlib.md5(device_file_data).hexdigest() - ci_manifest[device_name] = device_file_md5 - print(f"Manifest for {device_name} : {device_file_md5}") - if write_manifest_file: - device_zzz_dir_root = os.path.join(chef_zzz_root, device_name) - device_zzz_md5_file = os.path.join(device_zzz_dir_root, 'INPUTMD5.txt') - with open(device_zzz_md5_file, "w+", encoding="utf-8") as md5file: - md5file.write(device_file_md5) - if include_zap_submod: - ci_manifest["zap_commit"] = check_zap_master(repo_base_path) - if write_manifest_file: - with open(ci_manifest_file_name, "w+", encoding="utf-8") as ci_manifest_file: - ci_manifest_file.write(json.dumps(ci_manifest, indent=4)+"\n") - return ci_manifest diff --git a/examples/chef/cicd_meta.json b/examples/chef/cicd_meta.json new file mode 100644 index 00000000000000..afd9fa3541490f --- /dev/null +++ b/examples/chef/cicd_meta.json @@ -0,0 +1,14 @@ +{ + 'linux': { + 'build_dir': 'linux/out', + 'platform_label': 'linux_x86', + }, + 'esp32': { + 'build_dir': 'esp32/build', + 'platform_label': 'esp32-m5stack', + }, + 'nrfconnect': { + 'build_dir': 'nrfconnect/build', + 'platform_label': 'nrf-nrf52840dk', + }, +}