CLI refactoring for common build target APIs

lib/python/qmk/build_targets.py

@@ -0,0 +1,211 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+import json
+import shutil
+from typing import List, Union
+from pathlib import Path
+from dotty_dict import dotty, Dotty
+from milc import cli
+from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
+from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
+from qmk.keyboard import keyboard_folder
+from qmk.info import keymap_json
+from qmk.cli.generate.compilation_database import write_compilation_database
+
+
+class BuildTarget:
+    def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
+        self._keyboard = keyboard_folder(keyboard)
+        self._keyboard_safe = self._keyboard.replace('/', '_')
+        self._keymap = keymap
+        self._parallel = 1
+        self._clean = False
+        self._compiledb = False
+        self._target = f'{self._keyboard_safe}_{self.keymap}'
+        self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
+        self._generated_files_path = self._intermediate_output / 'src'
+        self._json = json.to_dict() if isinstance(json, Dotty) else json
+
+    def __str__(self):
+        return f'{self.keyboard}:{self.keymap}'
+
+    def __repr__(self):
+        return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+    def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None:
+        if parallel is not None:
+            self._parallel = parallel
+        if clean is not None:
+            self._clean = clean
+        if compiledb is not None:
+            self._compiledb = compiledb
+
+    @property
+    def keyboard(self) -> str:
+        return self._keyboard
+
+    @property
+    def keymap(self) -> str:
+        return self._keymap
+
+    @property
+    def json(self) -> dict:
+        if not self._json:
+            self._load_json()
+        if not self._json:
+            return {}
+        return self._json
+
+    @property
+    def dotty(self) -> Dotty:
+        return dotty(self.json)
+
+    def _common_make_args(self, dry_run: bool = False, build_target: str = None):
+        compile_args = [
+            find_make(),
+            *get_make_parallel_args(self._parallel),
+            '-r',
+            '-R',
+            '-f',
+            'builddefs/build_keyboard.mk',
+        ]
+
+        if not cli.config.general.verbose:
+            compile_args.append('-s')
+
+        verbose = 'true' if cli.config.general.verbose else 'false'
+        color = 'true' if cli.config.general.color else 'false'
+
+        if dry_run:
+            compile_args.append('-n')
+
+        if build_target:
+            compile_args.append(build_target)
+
+        compile_args.extend([
+            f'KEYBOARD={self.keyboard}',
+            f'KEYMAP={self.keymap}',
+            f'KEYBOARD_FILESAFE={self._keyboard_safe}',
+            f'TARGET={self._target}',
+            f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
+            f'VERBOSE={verbose}',
+            f'COLOR={color}',
+            'SILENT=false',
+            'QMK_BIN="qmk"',
+        ])
+
+        return compile_args
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        raise NotImplementedError("prepare_build() not implemented in base class")
+
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        raise NotImplementedError("compile_command() not implemented in base class")
+
+    def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None:
+        self.prepare_build(build_target=build_target, **env_vars)
+        command = self.compile_command(build_target=build_target, dry_run=True, **env_vars)
+        write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars)
+
+    def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        if self._clean or self._compiledb:
+            command = [find_make(), "clean"]
+            if dry_run:
+                command.append('-n')
+            cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command))
+            cli.run(command, capture_output=False)
+
+        if self._compiledb and not dry_run:
+            self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars)
+
+        self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars)
+        command = self.compile_command(build_target=build_target, **env_vars)
+        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
+        if not dry_run:
+            cli.echo('\n')
+            ret = cli.run(command, capture_output=False)
+            if ret.returncode:
+                return ret.returncode
+
+
+class KeyboardKeymapBuildTarget(BuildTarget):
+    def __init__(self, keyboard: str, keymap: str, json: dict = None):
+        super().__init__(keyboard=keyboard, keymap=keymap, json=json)
+
+    def __repr__(self):
+        return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
+
+    def _load_json(self):
+        self._json = keymap_json(self.keyboard, self.keymap)
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        pass
+
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+
+        for key, value in env_vars.items():
+            compile_args.append(f'{key}={value}')
+
+        return compile_args
+
+
+class JsonKeymapBuildTarget(BuildTarget):
+    def __init__(self, json_path):
+        if isinstance(json_path, Path):
+            self.json_path = json_path
+        else:
+            self.json_path = None
+
+        json = parse_configurator_json(json_path)  # Will load from stdin if provided
+
+        # In case the user passes a keymap.json from a keymap directory directly to the CLI.
+        # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json
+        json["keymap"] = json.get("keymap", "default_json")
+
+        super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
+
+        self._keymap_json = self._generated_files_path / 'keymap.json'
+
+    def __repr__(self):
+        return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
+
+    def _load_json(self):
+        pass  # Already loaded in constructor
+
+    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
+        if self._clean:
+            if self._intermediate_output.exists():
+                shutil.rmtree(self._intermediate_output)
+
+        # begin with making the deepest folder in the tree
+        self._generated_files_path.mkdir(exist_ok=True, parents=True)
+
+        # Compare minified to ensure consistent comparison
+        new_content = json.dumps(self.json, separators=(',', ':'))
+        if self._keymap_json.exists():
+            old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
+            if old_content == new_content:
+                new_content = None
+
+        # Write the keymap.json file if different so timestamps are only updated
+        # if the content changes -- running `make` won't treat it as modified.
+        if new_content:
+            self._keymap_json.write_text(new_content, encoding='utf-8')
+
+    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
+        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
+        compile_args.extend([
+            f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
+            f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
+            f'KEYMAP_JSON={self._keymap_json}',
+            f'KEYMAP_PATH={self._generated_files_path}',
+        ])
+
+        for key, value in env_vars.items():
+            compile_args.append(f'{key}={value}')
+
+        return compile_args

lib/python/qmk/cli/clean.py

@@ -2,7 +2,7 @@
 """
 from subprocess import DEVNULL
 
-from qmk.commands import create_make_target
+from qmk.commands import find_make
 from milc import cli
 
 
@@ -11,4 +11,4 @@
 def clean(cli):
     """Runs `make clean` (or `make distclean` if --all is passed)
     """
-    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL)
+    cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL)

lib/python/qmk/cli/compile.py

@@ -7,22 +7,11 @@
 from milc import cli
 
 import qmk.path
-from qmk.constants import QMK_FIRMWARE
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment
+from qmk.commands import build_environment
 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards
 from qmk.keymap import keymap_completer, locate_keymap
-from qmk.cli.generate.compilation_database import write_compilation_database
-
-
-def _is_keymap_target(keyboard, keymap):
-    if keymap == 'all':
-        return True
-
-    if locate_keymap(keyboard, keymap):
-        return True
-
-    return False
+from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget
 
 
 @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile')
@@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")
 @cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")
 @cli.subcommand('Compile a QMK Firmware.')
 @automagic_keyboard
@@ -53,47 +43,27 @@ def compile(cli):
     # Build the environment vars
     envs = build_environment(cli.args.env)
 
-    # Determine the compile command
-    commands = []
-
-    current_keyboard = None
-    current_keymap = None
+    # Handler for the build target
+    target = None
 
     if cli.args.filename:
-        # If a configurator JSON was provided generate a keymap and compile it
-        user_keymap = parse_configurator_json(cli.args.filename)
-        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)]
+        # if we were given a filename, assume we have a json build target
+        target = JsonKeymapBuildTarget(cli.args.filename)
 
     elif cli.config.compile.keyboard and cli.config.compile.keymap:
-        # Generate the make command for a specific keyboard/keymap.
-        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap):
+        # if we got a keyboard and keymap, attempt to find it
+        if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):
             cli.log.error('Invalid keymap argument.')
             cli.print_help()
             return False
 
-        if cli.args.clean:
-            commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))
-        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs))
+        # If we got here, then we have a valid keyboard and keymap for a build target
+        target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap)
 
-        current_keyboard = cli.config.compile.keyboard
-        current_keymap = cli.config.compile.keymap
-
-    if not commands:
+    if not target:
         cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
         cli.print_help()
         return False
 
-    if cli.args.compiledb:
-        if current_keyboard is None or current_keymap is None:
-            cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.')
-            cli.print_help()
-            return False
-        write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json')
-
-    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1]))
-    if not cli.args.dry_run:
-        cli.echo('\n')
-        for command in commands:
-            ret = cli.run(command, capture_output=False)
-            if ret.returncode:
-                return ret.returncode
+    target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb)
+    target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs)

lib/python/qmk/cli/find.py

@@ -19,13 +19,9 @@
 def find(cli):
     """Search through all keyboards and keymaps for a given search criteria.
     """
+    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter)
+    for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
+        print(f'{target}')
 
-    if len(cli.args.filter) == 0 and len(cli.args.print) > 0:
-        cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.')
-
-    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print)
-    for keyboard, keymap, print_vals in targets:
-        print(f'{keyboard}:{keymap}')
-
-        for key, val in print_vals:
-            print(f'    {key}={val}')
+        for key in cli.args.print:
+            print(f'    {key}={target.dotty.get(key, None)}')