Begin refactor

examples/chef/chef.py

@@ -23,11 +23,11 @@
 import sys
 import tarfile
 import textwrap
+from typing import Any, Dict
 from typing import Sequence
 
 import yaml
 
-import chef_util
 import constants
 import stateful_shell
 
@@ -41,7 +41,7 @@
 _DEVICE_LIST = [file[:-4] for file in os.listdir(_DEVICE_FOLDER) if file.endswith(".zap")]
 _CHEF_ZZZ_ROOT = os.path.join(_CHEF_SCRIPT_PATH, "zzz_generated")
 _CI_MANIFEST_FILE_NAME = os.path.join(_CHEF_SCRIPT_PATH, "cimanifest.json")
-_CHEF_DEVICES_DIR = os.path.join(_CHEF_SCRIPT_PATH, "devices")
+_CI_ALLOW_LIST = ["lighting-app"]
 
 gen_dir = ""  # Filled in after sample app type is read from args.
 
@@ -97,6 +97,52 @@ def check_python_version() -> None:
         exit(1)
 
 
+def check_zap_master() -> str:
+    """Produces hash of ZAP submodule in branch master"""
+    git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"]
+    zap_commit = str(subprocess.check_output(git_cmd, cwd=_REPO_BASE_PATH))
+    zap_commit = zap_commit.split(" ")[2]
+    zap_commit = zap_commit[:zap_commit.index("\\")]
+    print(f"zap commit: {zap_commit}")
+    return zap_commit
+
+
+def generate_device_manifest(
+    include_zap_submod: bool = False,
+    write_manifest_file: bool = False) -> Dict[str, Any]:
+    """Produces dictionary containing md5 of device dir zap files.
+
+    Args:
+        include_zap_submod: Include ZAP commit in manifest
+        write_manifest_file: Serialize manifest
+    Returns:
+        Dict containing MD5 of device dir zap files.
+    """
+    ci_manifest = {}
+    for device_name in _DEVICE_LIST:
+        device_file_path = os.path.join(_CHEF_DEVICES_DIR, device_dir_item)
+        with open(device_file_path, "rb") as device_file:
+            device_file_data = device_file.read()
+            device_file_md5 = hashlib.md5(device_file_data).hexdigest()
+            ci_manifest[device_name] = device_file_md5
+            print(f"Manifest for {device_name} : {device_file_md5}")
+            if write_manifest_file:
+                device_zzz_dir_root = os.path.join(_CHEF_ZZZ_ROOT, device_name)
+                device_zzz_md5_file = os.path.join(device_zzz_dir_root, 'INPUTMD5.txt')
+                with open(device_zzz_md5_file, "w+", encoding="utf-8") as md5file:
+                    md5file.write(device_file_md5)
+    if include_zap_submod:
+        ci_manifest["zap_commit"] = check_zap_master(_REPO_BASE_PATH)
+    if write_manifest_file:
+        with open(_CI_MANIFEST_FILE_NAME, "w+", encoding="utf-8") as ci_manifest_file:
+            ci_manifest_file.write(json.dumps(ci_manifest, indent=4)+"\n")
+    return ci_manifest
+
+
+def load_cicd_config() -> dict:
+    pass
+
+
 def main(argv: Sequence[str]) -> None:
     check_python_version()
     config = load_config()
@@ -191,11 +237,11 @@ def main(argv: Sequence[str]) -> None:
     #
 
     if options.validate_zzz:
-        fix_instructions = "Cached files out of date. Please bootstrap, activate, install zap, run chef with the flag --generate_zzz and commit /examples/chef/zzz_generated and /examples/chef/cimanifes.json"
+        fix_instructions = "Cached files out of date. Please bootstrap, activate, install zap, run chef with the flag --generate_zzz and commit /examples/chef/zzz_generated and /examples/chef/cimanifes.json. Ensure you are running with the latest version of ZAP from master"
         ci_manifest = chef_util.generate_device_manifest(_CHEF_DEVICES_DIR)
         with open(_CI_MANIFEST_FILE_NAME, "r", encoding="utf-8") as ci_manifest_file:
             cached_manifest = json.loads(ci_manifest_file.read())
-            for device in ci_manifest:
+            for device in _CI_MANIFEST:
                 if device != "zap_commit":
                     try:
                         if cached_manifest[device] != ci_manifest[device]:
@@ -260,22 +306,19 @@ def main(argv: Sequence[str]) -> None:
             shutil.rmtree(_CHEF_ZZZ_ROOT)
             os.mkdir(_CHEF_ZZZ_ROOT)
         print(f"Generating files in {_CHEF_ZZZ_ROOT} for all devices", flush=True)
-        for device_dir_item in os.listdir(_CHEF_DEVICES_DIR):
-            target_file_ext = ".zap"
-            if device_dir_item.endswith(target_file_ext):
-                device_name = device_dir_item[:-len(target_file_ext)]
-                print(f"Generating files for {device_name}", flush=True)
-                device_out_dir = os.path.join(_CHEF_ZZZ_ROOT, device_name)
-                os.mkdir(device_out_dir)
-                device_out_dir = os.path.join(device_out_dir, "zap-generated")
-                os.mkdir(device_out_dir)
-                shell.run_cmd(textwrap.dedent(f"""\
-                {_REPO_BASE_PATH}/scripts/tools/zap/generate.py \
-                {_CHEF_SCRIPT_PATH}/devices/{device_name}.zap -o {device_out_dir}
-                """))
-                shell.run_cmd(f"touch {device_out_dir}/af-gen-event.h")
-        chef_util.generate_device_manifest(_CHEF_DEVICES_DIR, include_zap_submod=True, write_manifest_file=True,
-                                           ci_manifest_file_name=_CI_MANIFEST_FILE_NAME, repo_base_path=_REPO_BASE_PATH, chef_zzz_root=_CHEF_ZZZ_ROOT)
+        for device_name in _DEVICE_LIST:
+            print(f"Generating files for {device_name}", flush=True)
+            device_out_dir = os.path.join(_CHEF_ZZZ_ROOT, device_name)
+            os.mkdir(device_out_dir)
+            device_out_dir = os.path.join(device_out_dir, "zap-generated")
+            os.mkdir(device_out_dir)
+            shell.run_cmd(textwrap.dedent(f"""\
+            {_REPO_BASE_PATH}/scripts/tools/zap/generate.py \
+            {_CHEF_SCRIPT_PATH}/devices/{device_name}.zap -o {device_out_dir}
+            """))
+            shell.run_cmd(f"touch {device_out_dir}/af-gen-event.h")
+        chef_util.generate_device_manifest(include_zap_submod=True,
+                                           write_manifest_file=True)
         exit(0)
 
     #
@@ -302,29 +345,25 @@ def main(argv: Sequence[str]) -> None:
         os.environ['GNUARMEMB_TOOLCHAIN_PATH'] = os.environ['PW_ARM_CIPD_INSTALL_DIR']
         archive_prefix = "/workspace/artifacts/"
         if not os.path.exists(archive_prefix):
-            # shutil.rmtree(archive_prefix)
             os.mkdir(archive_prefix)
         archive_suffix = ".tar.gz"
 
         cd_platforms_meta = chef_util.cd_platforms_meta
-        for device in os.listdir(_CHEF_DEVICES_DIR):
-            target_file_ext = ".zap"
-            if device.endswith(target_file_ext):
-                device_name = device[:-len(target_file_ext)]
-                for platform, platform_meta in cd_platforms_meta.items():
-                    directory = platform_meta['build_dir']
-                    label = platform_meta['platform_label']
-                    output_dir = os.path.join(_CHEF_SCRIPT_PATH, directory)
-                    command = f"./chef.py -cbr --use_zzz -d {device_name} -t {platform}"
-                    print('-' * 64, flush=True)
-                    print(f"Building {command}", flush=True)
-                    print('-' * 64, flush=True)
-                    subprocess.check_call(command, cwd=_CHEF_SCRIPT_PATH, shell=True)
-                    archive_name = f"{label}-chef-{device_name}-wifi-rpc"
-                    archive_full_name = archive_prefix + archive_name + archive_suffix
-                    print(f"Adding build output to archive {archive_full_name}", flush=True)
-                    with tarfile.open(archive_full_name, "w:gz") as tar:
-                        tar.add(output_dir, arcname=".")
+        for device_name in _DEVICE_LIST:
+            for platform, platform_meta in cd_platforms_meta.items():
+                directory = platform_meta['build_dir']
+                label = platform_meta['platform_label']
+                output_dir = os.path.join(_CHEF_SCRIPT_PATH, directory)
+                command = f"./chef.py -cbr --use_zzz -d {device_name} -t {platform}"
+                print('-' * 64, flush=True)
+                print(f"Building {command}", flush=True)
+                print('-' * 64, flush=True)
+                subprocess.check_call(command, cwd=_CHEF_SCRIPT_PATH, shell=True)
+                archive_name = f"{label}-chef-{device_name}-wifi-rpc"
+                archive_full_name = archive_prefix + archive_name + archive_suffix
+                print(f"Adding build output to archive {archive_full_name}", flush=True)
+                with tarfile.open(archive_full_name, "w:gz") as tar:
+                    tar.add(output_dir, arcname=".")
         exit(0)
 
     #

examples/chef/cicd_meta.json

@@ -0,0 +1,14 @@
+{
+    'linux': {
+        'build_dir': 'linux/out',
+        'platform_label': 'linux_x86',
+    },
+    'esp32': {
+        'build_dir': 'esp32/build',
+        'platform_label': 'esp32-m5stack',
+    },
+    'nrfconnect': {
+        'build_dir': 'nrfconnect/build',
+        'platform_label': 'nrf-nrf52840dk',
+    },
+}