From 2188512fca8887d31815c3be4a21e7ff07f064ce Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 30 Nov 2022 10:11:41 -0500 Subject: [PATCH] Code pre-generation support (#23763) * Start implementing a pregenerator * Start moving pregenerate logic into a separate directory * Start adding some ability to figure out pregeneration output locations * Pregeneration for bridge seems to work * Add missing files * Restyle * Better log level logic * Pregeneration of java also exists * Allow pregeneration of cpp app data. Full codegen usage is now enabled * Move sdk root around * Make things run * Restyle * Parallel code pregen - can do pregen in 1.08 seconds on my machine * Restyle * Make sure pregen folders are split * Pregeneration compile when using GN works * Restyle * Direct pregen usage works now * Restyle * Minor sort * Add support for both pregen and no pregen * Restyle * Fix pregen dir output logic * Support pregenerated directory in build_examples.py * Fix gn build logic * Somewhat simpler code for parallel vs serial codegen * Use imap_unordered to not care about actual parallel generation order * Also fix java jni codegen with pregenerated data * Fix java compilation deps: java codegen uses data model files * NRF now can use pregen folder * Allow telink to also use a pregen dir * Better shell escape, make mbedos cmake flags work with pregen dir * Restyle * Add pregen support for esp32 as well * Add a test for esp32 pregeneration support * Also test pregen support in linux builds (to check gn builds) * Remove unused file * Fix spelling * Fix esp32 compilation - gn arguments need to be passed from cmake * Fix some define forwarding logic in codegen * Make sure java build config (which includes header paths) is set as a config and applies to generated sources * java build config should apply to all sources, not just transitively * Restyle * Replace codege with codegen. * Fix naming typo * Update text for build steps * Update spacing logic to make the code cleaner. The nospace args is odd --- .github/workflows/build.yaml | 20 ++- .github/workflows/examples-esp32.yaml | 18 +++ build/chip/chip_codegen.cmake | 91 ++++++++---- build/chip/chip_codegen.gni | 152 +++++++++++++++----- config/esp32/components/chip/CMakeLists.txt | 4 + docs/code_generation.md | 26 +++- scripts/build/build_examples.py | 8 +- scripts/build/builders/builder.py | 4 + scripts/build/builders/esp32.py | 18 ++- scripts/build/builders/gn.py | 5 + scripts/build/builders/mbed.py | 18 +-- scripts/build/builders/nrf.py | 3 + scripts/build/builders/telink.py | 16 ++- scripts/codepregen.py | 114 +++++++++++++++ scripts/pregenerate/__init__.py | 69 +++++++++ scripts/pregenerate/pregenerators.py | 102 +++++++++++++ scripts/pregenerate/types.py | 44 ++++++ src/app/chip_data_model.gni | 24 +++- src/controller/data_model/BUILD.gn | 20 ++- 19 files changed, 661 insertions(+), 95 deletions(-) create mode 100755 scripts/codepregen.py create mode 100644 scripts/pregenerate/__init__.py create mode 100644 scripts/pregenerate/pregenerators.py create mode 100644 scripts/pregenerate/types.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 276b1f264e0cee..4a3ce8550c2e7e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -229,12 +229,28 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py --no-log-timestamps \ - --target linux-x64-all-clusters-ipv6only-clang \ - --target linux-x64-chip-tool-ipv6only-clang \ --target linux-x64-minmdns-ipv6only-clang \ --target linux-x64-rpc-console \ build \ " + - name: Create a pre-generate directory and ensure compile-time codegen would fail + run: | + ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated" + mv scripts/codegen.py scripts/codegen.py.renamed + - name: Build using build_examples.py (pregen) + timeout-minutes: 60 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/build/build_examples.py --no-log-timestamps \ + --target linux-x64-all-clusters-ipv6only-clang \ + --target linux-x64-chip-tool-ipv6only-clang \ + --pregen-dir ./zzz_pregenerated \ + build \ + " + - name: Undo code pre-generation changes (make compile time codegen work again) + run: | + rm -rf ./zzz_pregenerated + mv scripts/codegen.py.renamed scripts/codegen.py - name: Run fake linux tests with build_examples timeout-minutes: 15 run: | diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml index 6755e3f64b7bbe..9299ac3822c684 100644 --- a/.github/workflows/examples-esp32.yaml +++ b/.github/workflows/examples-esp32.yaml @@ -71,11 +71,29 @@ jobs: "./scripts/build/build_examples.py \ --enable-flashbundle \ --target esp32-m5stack-all-clusters \ + build \ + --copy-artifacts-to out/artifacts \ + " + - name: Prepare code pregen and ensure compile time pregen not possible + run: | + ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated" + mv scripts/codegen.py scripts/codegen.py.renamed + - name: Build some M5Stack variations with pregen + timeout-minutes: 60 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/build/build_examples.py \ + --enable-flashbundle \ --target esp32-m5stack-all-clusters-minimal \ --target esp32-m5stack-all-clusters-rpc-ipv6only \ + --pregen-dir ./zzz_pregenerated \ build \ --copy-artifacts-to out/artifacts \ " + - name: Undo code pregeneration changes + run: | + rm -rf ./zzz_pregenerated + mv scripts/codegen.py.renamed scripts/codegen.py - name: Prepare bloat report run: | .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \ diff --git a/build/chip/chip_codegen.cmake b/build/chip/chip_codegen.cmake index 17e3844fb45ba9..885d189f43f6de 100644 --- a/build/chip/chip_codegen.cmake +++ b/build/chip/chip_codegen.cmake @@ -31,40 +31,71 @@ function(chip_codegen TARGET_NAME) ${ARGN} ) - set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}") + set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.") - string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}") + if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "") + set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}") - file(MAKE_DIRECTORY "${GEN_FOLDER}") - file(GENERATE - OUTPUT "${GEN_FOLDER}/expected.outputs" - CONTENT "${OUTPUT_AS_NEWLINES}" - ) + string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}") + file(MAKE_DIRECTORY "${GEN_FOLDER}") + file(GENERATE + OUTPUT "${GEN_FOLDER}/expected.outputs" + CONTENT "${OUTPUT_AS_NEWLINES}" + ) + + + set(OUT_NAMES) + foreach(NAME IN LISTS ARG_OUTPUTS) + list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") + endforeach() + + # Python is expected to be in the path + # + # find_package(Python3 REQUIRED) + add_custom_command( + OUTPUT ${OUT_NAMES} + COMMAND "${CHIP_ROOT}/scripts/codegen.py" + ARGS "--generator" "${ARG_GENERATOR}" + "--output-dir" "${GEN_FOLDER}" + "--expected-outputs" "${GEN_FOLDER}/expected.outputs" + "${ARG_INPUT}" + DEPENDS + "${ARG_INPUT}" + VERBATIM + ) + + add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}") + + # Forward outputs to the parent + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) + set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) + else() + # Gets a path such as: + # examples/lock-app/lock-common/lock-app.matter + file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT}) + + # Removes the trailing file extension to get something like: + # examples/lock-app/lock-common/lock-app + string(REGEX REPLACE "\.matter$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}") + + + # Build the final location within the pregen directory + set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/codegen/${ARG_GENERATOR}") + + # TODO: build a fake target of ${TARGET_NAME} + + # Here we have ${CHIP_CODEGEN_PREGEN_DIR} + set(OUT_NAMES) + foreach(NAME IN LISTS ARG_OUTPUTS) + list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") + endforeach() - set(OUT_NAMES) - foreach(NAME IN LISTS ARG_OUTPUTS) - list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") - endforeach() - - # Python is expected to be in the path - # - # find_package(Python3 REQUIRED) - add_custom_command( - OUTPUT ${OUT_NAMES} - COMMAND "${CHIP_ROOT}/scripts/codegen.py" - ARGS "--generator" "${ARG_GENERATOR}" - "--output-dir" "${GEN_FOLDER}" - "--expected-outputs" "${GEN_FOLDER}/expected.outputs" - "${ARG_INPUT}" - DEPENDS - "${ARG_INPUT}" - VERBATIM - ) - add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}") + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) + set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) - # Forward outputs to the parent - set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) - set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) + # allow adding dependencies to a phony target since no codegen is done + add_custom_target(${TARGET_NAME}) + endif() endfunction() diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index bab79934bd37f5..fd5d69c40c513f 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -18,6 +18,75 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/python.gni") +declare_args() { + # Location where code has been pre-generated + chip_code_pre_generated_directory = "" +} + +# Code generation that will happen at build time. +# +# +template("_chip_build_time_codegen") { + _name = target_name + _generator = invoker.generator + + config("${_name}_config") { + include_dirs = [ target_gen_dir ] + } + + pw_python_action("${_name}_codegen") { + script = "${chip_root}/scripts/codegen.py" + + _idl_file = invoker.input + _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs" + + write_file(_expected_outputs, invoker.outputs, "list lines") + + args = [ + "--generator", + _generator, + "--output-dir", + rebase_path(target_gen_dir, root_build_dir), + "--expected-outputs", + rebase_path(_expected_outputs, root_build_dir), + rebase_path(_idl_file, root_build_dir), + ] + + deps = [ "${chip_root}/scripts/idl" ] + + inputs = [ + _idl_file, + _expected_outputs, + ] + sources = [ _idl_file ] + + outputs = [] + foreach(name, invoker.outputs) { + outputs += [ "${target_gen_dir}/${name}" ] + } + } + + source_set(_name) { + sources = [] + foreach(name, invoker.outputs) { + sources += [ "${target_gen_dir}/${name}" ] + } + + public_configs = [ ":${_name}_config" ] + + if (defined(invoker.public_configs)) { + public_configs += invoker.public_configs + } + + forward_variables_from(invoker, [ "deps" ]) + + if (!defined(deps)) { + deps = [] + } + deps += [ ":${_name}_codegen" ] + } +} + # Defines a target that runs code generation based on # scripts/codegen.py # @@ -32,13 +101,29 @@ import("$dir_pw_build/python.gni") # Explicit names of the expected outputs. Enforced to validate that # expected outputs are generated when processing input files. # +# deps, public_configs +# Forwarded to the resulting source set +# +# Command line parameters: +# +# chip_code_pre_generated_directory: +# - If this is set, generation will NOT happen at compile time but rather +# the code generation is assumed to have already happened and reside in +# the given location. +# - The TOP LEVEL directory is assumed to be given. Actual location for +# individual generators is expected to be of the form +# // +# # NOTE: content of "outputs" is verified to match the output of codegen.py # exactly. It is not inferred on purpose, to make build-rules explicit -# and verifiable (even though codege.py can at runtime report its outputs) +# and verifiable (even though codegen.py can at runtime report its outputs) # # To find the list of generated files, you can run codegen.py with the # "--name-only" argument # +# NOTE: +# the result of the target_name WILL BE a `source_set`. Treat it as such. +# # Example usage: # # chip_codegen("java-jni-generate") { @@ -53,43 +138,46 @@ import("$dir_pw_build/python.gni") # } # template("chip_codegen") { - _name = target_name - _generator = invoker.generator - - config("${_name}_config") { - include_dirs = [ target_gen_dir ] - } + if (chip_code_pre_generated_directory == "") { + _chip_build_time_codegen(target_name) { + forward_variables_from(invoker, + [ + "deps", + "generator", + "input", + "outputs", + "public_configs", + ]) + } + } else { + _name = target_name - pw_python_action(_name) { - script = "${chip_root}/scripts/codegen.py" + # This contstructs a path like: + # FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/) + # USING "cpp-app" for generator: + # => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app + _generation_dir = + chip_code_pre_generated_directory + "/" + + string_replace(rebase_path(invoker.input, chip_root), ".matter", "") + + "/codegen/" + invoker.generator - _idl_file = invoker.input - _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs" - - write_file(_expected_outputs, invoker.outputs, "list lines") + config("${_name}_config") { + include_dirs = [ "${_generation_dir}" ] + } - args = [ - "--generator", - _generator, - "--output-dir", - rebase_path(target_gen_dir, root_build_dir), - "--expected-outputs", - rebase_path(_expected_outputs, root_build_dir), - rebase_path(_idl_file, root_build_dir), - ] + source_set(_name) { + public_configs = [ ":${_name}_config" ] - deps = [ "${chip_root}/scripts/idl" ] - public_configs = [ ":${_name}_config" ] + if (defined(invoker.public_configs)) { + public_configs += invoker.public_configs + } - inputs = [ - _idl_file, - _expected_outputs, - ] - sources = [ _idl_file ] + forward_variables_from(invoker, [ "deps" ]) - outputs = [] - foreach(name, invoker.outputs) { - outputs += [ "${target_gen_dir}/${name}" ] + sources = [] + foreach(name, invoker.outputs) { + sources += [ "${_generation_dir}/${name}" ] + } } } } diff --git a/config/esp32/components/chip/CMakeLists.txt b/config/esp32/components/chip/CMakeLists.txt index 44f923808e17e6..d7cb69a6a8fa34 100644 --- a/config/esp32/components/chip/CMakeLists.txt +++ b/config/esp32/components/chip/CMakeLists.txt @@ -101,6 +101,10 @@ if(CONFIG_DISABLE_IPV4) chip_gn_arg_append("chip_inet_config_enable_ipv4" "false") endif() +if(CHIP_CODEGEN_PREGEN_DIR) + chip_gn_arg_append("chip_code_pre_generated_directory" "\"${CHIP_CODEGEN_PREGEN_DIR}\"") +endif() + if(CONFIG_ENABLE_PW_RPC) string(APPEND chip_gn_args "import(\"//build_overrides/pigweed.gni\")\n") chip_gn_arg_append("remove_default_configs" "[\"//third_party/connectedhomeip/third_party/pigweed/repo/pw_build:toolchain_cpp_standard\"]") diff --git a/docs/code_generation.md b/docs/code_generation.md index 51a615f9872a9e..051a2152fb36a3 100644 --- a/docs/code_generation.md +++ b/docs/code_generation.md @@ -176,12 +176,32 @@ enerated ### `*.matter` code generation -Currently `*.matter` code generation is done at compile time. +`*.matter` code generation can be done either at compile time or it can use +pre-generated output. Rules for how `codegen.py` is invoked and how includes/sources are set are defined at: - `src/app/chip_data_model.cmake` -- `build/chip/esp32/esp32_codegen.cmake` (support for 2-pass cmake builds used - by the Espressif `idf.py` build system) - `src/app/chip_data_model.gni` + +Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for +the 2-pass cmake builds used by the Espressif `idf.py` build system. + +## Pre-generation + +Code pre-generation can be used: + +- when compile-time code generation is not desirable. This may be for + importing into build systems that do not have the pre-requisites to run code + generation at build time or to save the code generation time at the expense + of running code generation for every possible zap/generation type +- To check changes in generated code across versions, beyond the comparisons + of golden image tests in `scripts/idl/tests` + +The script to trigger code pre-generation is `scripts/code_pregenerate.py` and +requires the pre-generation output directory as an argument + +```bash +scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} +``` diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py index 5792bfaec64274..780241f089a35f 100755 --- a/scripts/build/build_examples.py +++ b/scripts/build/build_examples.py @@ -87,6 +87,11 @@ def ValidateRepoPath(context, parameter, value): default='./out', type=click.Path(file_okay=False, resolve_path=True), help='Prefix for the generated file output.') +@click.option( + '--pregen-dir', + default=None, + type=click.Path(file_okay=False, resolve_path=True), + help='Directory where generated files have been pre-generated.') @click.option( '--clean', default=False, @@ -114,7 +119,7 @@ def ValidateRepoPath(context, parameter, value): 'for using ccache when building examples.')) @click.pass_context def main(context, log_level, target, repo, - out_prefix, clean, dry_run, dry_run_output, enable_flashbundle, + out_prefix, pregen_dir, clean, dry_run, dry_run_output, enable_flashbundle, no_log_timestamps, pw_command_launcher): # Ensures somewhat pretty logging of what is going on log_fmt = '%(asctime)s %(levelname)-7s %(message)s' @@ -143,6 +148,7 @@ def main(context, log_level, target, repo, context.obj.SetupBuilders(targets=requested_targets, options=BuilderOptions( enable_flashbundle=enable_flashbundle, pw_command_launcher=pw_command_launcher, + pregen_dir=pregen_dir, )) if clean: diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py index 99bc9255ee5ef9..dc420fce5e6b41 100644 --- a/scripts/build/builders/builder.py +++ b/scripts/build/builders/builder.py @@ -24,9 +24,13 @@ class BuilderOptions: # Enable flashbundle generation stage enable_flashbundle: bool = False + # Allow to wrap default build command pw_command_launcher: str = None + # Locations where files are pre-generated + pregen_dir: str = None + class Builder(ABC): """Generic builder base class for CHIP. diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index d84e8f56e8edbc..54c47a48cb557f 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -176,11 +176,19 @@ def generate(self): self._Execute( ['bash', '-c', 'echo -e "\\nCONFIG_DISABLE_IPV4=y\\n" >>%s' % shlex.quote(defaults_out)]) - cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out} reconfigure".format( - defaults=shlex.quote(defaults_out), - example_path=self.ExamplePath, - out=shlex.quote(self.output_dir) - ) + cmake_flags = [] + + if self.options.pregen_dir: + cmake_flags.append( + f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + + cmake_args = ['-C', self.ExamplePath, '-B', + shlex.quote(self.output_dir)] + cmake_flags + + cmake_args = " ".join(cmake_args) + defaults = shlex.quote(defaults_out) + + cmd = f"\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py {cmake_args} reconfigure" # This will do a 'cmake reconfigure' which will create ninja files without rebuilding self._IdfEnvExecute(cmd) diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py index bacd87d2399e02..9e859f6025a5a7 100644 --- a/scripts/build/builders/gn.py +++ b/scripts/build/builders/gn.py @@ -61,8 +61,13 @@ def generate(self): ] extra_args = [] + if self.options.pw_command_launcher: extra_args.append('pw_command_launcher="%s"' % self.options.pw_command_launcher) + + if self.options.pregen_dir: + extra_args.append('chip_code_pre_generated_directory="%s"' % self.options.pregen_dir) + extra_args.extend(self.GnBuildArgs() or []) if extra_args: cmd += ['--args=%s' % ' '.join(extra_args)] diff --git a/scripts/build/builders/mbed.py b/scripts/build/builders/mbed.py index 73fc802386207f..46b3ead610bb72 100644 --- a/scripts/build/builders/mbed.py +++ b/scripts/build/builders/mbed.py @@ -121,14 +121,16 @@ def generate(self): '--mbed-os-path', self.mbed_os_path, ], title='Generating config ' + self.identifier) - self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), '-GNinja', - '-DCMAKE_BUILD_TYPE={}'.format( - self.profile.ProfileName.lower()), - '-DMBED_OS_PATH={}'.format( - shlex.quote(self.mbed_os_path)), - '-DMBED_OS_POSIX_SOCKET_PATH={}'.format( - shlex.quote(self.mbed_os_posix_socket_path)), - ], title='Generating ' + self.identifier) + flags = [] + flags.append(f"-DMBED_OS_PATH={shlex.quote(self.mbed_os_path)}") + flags.append(f"-DMBED_OS_PATH={shlex.quote(self.mbed_os_path)}") + flags.append(f"-DMBED_OS_POSIX_SOCKET_PATH={shlex.quote(self.mbed_os_posix_socket_path)}") + + if self.options.pregen_dir: + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + + self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), + '-GNinja'] + flags, title='Generating ' + self.identifier) def _build(self): # Remove old artifacts to force linking diff --git a/scripts/build/builders/nrf.py b/scripts/build/builders/nrf.py index 03298cc1f0ac94..0037a38da21679 100644 --- a/scripts/build/builders/nrf.py +++ b/scripts/build/builders/nrf.py @@ -179,6 +179,9 @@ def generate(self): if self.board == NrfBoard.NRF52840DONGLE and self.app != NrfApp.ALL_CLUSTERS and self.app != NrfApp.ALL_CLUSTERS_MINIMAL: flags.append("-DCONF_FILE=prj_no_dfu.conf") + if self.options.pregen_dir: + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + build_flags = " -- " + " ".join(flags) if len(flags) > 0 else "" cmd = ''' diff --git a/scripts/build/builders/telink.py b/scripts/build/builders/telink.py index a85ec7c9fe58d6..32a9bf85c3187e 100644 --- a/scripts/build/builders/telink.py +++ b/scripts/build/builders/telink.py @@ -100,15 +100,21 @@ def generate(self): if os.path.exists(self.output_dir): return + flags = [] + if self.options.pregen_dir: + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + + build_flags = " -- " + " ".join(flags) if len(flags) > 0 else "" + cmd = self.get_cmd_prefixes() cmd += ''' source "$ZEPHYR_BASE/zephyr-env.sh"; -west build --cmake-only -d {outdir} -b {board} {sourcedir} +west build --cmake-only -d {outdir} -b {board} {sourcedir}{build_flags} '''.format( - outdir=shlex.quote( - self.output_dir), board=self.board.GnArgName(), sourcedir=shlex.quote( - os.path.join( - self.root, 'examples', self.app.ExampleName(), 'telink'))).strip() + outdir=shlex.quote(self.output_dir), + board=self.board.GnArgName(), + sourcedir=shlex.quote(os.path.join(self.root, 'examples', self.app.ExampleName(), 'telink')), + build_flags=build_flags).strip() self._Execute(['bash', '-c', cmd], title='Generating ' + self.identifier) diff --git a/scripts/codepregen.py b/scripts/codepregen.py new file mode 100755 index 00000000000000..13ca9a60d972c7 --- /dev/null +++ b/scripts/codepregen.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import logging +import multiprocessing +import itertools +import enum +import os +import sys + + +try: + from pregenerate import FindPregenerationTargets +except: + import os + sys.path.append(os.path.abspath(os.path.dirname(__file__))) + from pregenerate import FindPregenerationTargets + +try: + import coloredlogs + _has_coloredlogs = True +except: + _has_coloredlogs = False + +# Supported log levels, mapping string values required for argument +# parsing into logging constants +__LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, +} + + +def _ParallelGenerateOne(arg): + """ + Helper method to be passed to multiprocessing parallel generation of + items. + """ + arg[0].Generate(arg[1]) + + +@click.command() +@click.option( + '--log-level', + default='INFO', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output') +@click.option( + '--parallel/--no-parallel', + default=True, + help='Do parallel/multiprocessing codegen.') +@click.option( + '--sdk-root', + default=None, + help='Path to the SDK root (where .zap/.matter files exist).') +@click.argument('output_dir') +def main(log_level, parallel, sdk_root, output_dir): + if _has_coloredlogs: + coloredlogs.install(level=__LOG_LEVELS__[ + log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') + else: + logging.basicConfig( + level=__LOG_LEVELS__[log_level], + format='%(asctime)s %(levelname)-7s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + if not sdk_root: + sdk_root = os.path.join(os.path.dirname( + os.path.realpath(__file__)), '..') + + sdk_root = os.path.abspath(sdk_root) + + if not output_dir: + raise Exception("Missing output directory") + + output_dir = os.path.abspath(output_dir) + + logging.info(f"Pre-generating {sdk_root} data into {output_dir}") + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + targets = FindPregenerationTargets(sdk_root) + + if parallel: + target_and_dir = zip(targets, itertools.repeat(output_dir)) + with multiprocessing.Pool() as pool: + for _ in pool.imap_unordered(_ParallelGenerateOne, target_and_dir): + pass + else: + for target in targets: + target.Generate(output_dir) + + logging.info("Done") + + +if __name__ == '__main__': + main() diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py new file mode 100644 index 00000000000000..eab3205dc9d981 --- /dev/null +++ b/scripts/pregenerate/__init__.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os + +from typing import Iterator + +from .types import InputIdlFile, IdlFileType + + +from .pregenerators import CodegenJavaPregenerator +from .pregenerators import CodegenBridgePregenerator +from .pregenerators import CodegenCppAppPregenerator + + +def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: + relevant_subdirs = [ + 'examples', # all example apps + 'src', # realistically only controller/data_model + ] + + while sdk_root.endswith('/'): + sdk_root = sdk_root[:-1] + sdk_root_length = len(sdk_root) + + for subdir_name in relevant_subdirs: + top_directory_name = os.path.join(sdk_root, subdir_name) + logging.debug(f"Searching {top_directory_name}") + for root, dirs, files in os.walk(top_directory_name): + for file in files: + if file.endswith('.zap'): + yield InputIdlFile(file_type=IdlFileType.ZAP, + relative_path=os.path.join(root[sdk_root_length+1:], file)) + if file.endswith('.matter'): + yield InputIdlFile(file_type=IdlFileType.MATTER, + relative_path=os.path.join(root[sdk_root_length+1:], file)) + + +def FindPregenerationTargets(sdk_root: str): + """Finds all relevand pre-generation targets in the given + SDK root. + + Pre-generation targets are based on zap and matter files with options + on what rules to pregenerate and how. + """ + + generators = [ + CodegenBridgePregenerator(sdk_root), + CodegenJavaPregenerator(sdk_root), + CodegenCppAppPregenerator(sdk_root), + ] + + for idl in FindAllIdls(sdk_root): + for generator in generators: + if generator.Accept(idl): + yield generator.CreateTarget(idl) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py new file mode 100644 index 00000000000000..3d6658dd3bb7b1 --- /dev/null +++ b/scripts/pregenerate/pregenerators.py @@ -0,0 +1,102 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import shlex +import subprocess + +from .types import InputIdlFile, IdlFileType + +CODEGEN_PY_PATH = os.path.join(os.path.dirname(__file__), '..', 'codegen.py') + + +class CodegenTarget: + """A target that uses `scripts/codegen.py` to generate files.""" + + def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str): + self.idl = idl + self.generator = generator + self.sdk_root = sdk_root + + if idl.file_type != IdlFileType.MATTER: + raise Exception(f"Can only code generate for `*.matter` input files, not for {idl}") + + def Generate(self, output_root: str): + '''Runs codegen.py to generate in the specified directory''' + + output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generator) + + logging.info(f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}") + + cmd = [ + CODEGEN_PY_PATH, + '--log-level', 'fatal', + '--generator', self.generator, + '--output-dir', output_dir, + os.path.join(self.sdk_root, self.idl.relative_path) + ] + + logging.debug(f"Executing {cmd}") + subprocess.check_call(cmd) + + +class CodegenBridgePregenerator: + """Pregeneration logic for "bridge" codegen.py outputs""" + + def __init__(self, sdk_root): + self.sdk_root = sdk_root + + def Accept(self, idl: InputIdlFile): + # Bridge is highly specific, a single path is acceptable for dynamic + # bridge codegen + return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter" + + def CreateTarget(self, idl: InputIdlFile): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge") + + +class CodegenJavaPregenerator: + """Pregeneration logic for "java" codegen.py outputs""" + + def __init__(self, sdk_root): + self.sdk_root = sdk_root + + def Accept(self, idl: InputIdlFile): + # Java is highly specific, a single path is acceptable for dynamic + # bridge codegen + return idl.relative_path == "src/controller/data_model/controller-clusters.matter" + + def CreateTarget(self, idl: InputIdlFile): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java") + + +class CodegenCppAppPregenerator: + """Pregeneration logic for "cpp-app" codegen.py outputs""" + + def __init__(self, sdk_root): + self.sdk_root = sdk_root + + def Accept(self, idl: InputIdlFile): + if idl.file_type != IdlFileType.MATTER: + return False + + # we should not be checked for these, but verify just in case + if '/tests/' in idl.relative_path: + return False + + return True + + def CreateTarget(self, idl: InputIdlFile): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app") diff --git a/scripts/pregenerate/types.py b/scripts/pregenerate/types.py new file mode 100644 index 00000000000000..3d8c0bca3034d8 --- /dev/null +++ b/scripts/pregenerate/types.py @@ -0,0 +1,44 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from dataclasses import dataclass +from enum import Enum, auto + + +class IdlFileType(Enum): + ZAP = auto() + MATTER = auto() + + +@dataclass +class InputIdlFile: + file_type: IdlFileType + relative_path: str + + @property + def pregen_subdir(self): + ''' + Returns the relative path inside the pregenerate directory where + data for this IDL file should be pregenerated. + ''' + top_dir = os.path.splitext(self.relative_path)[0] + + if self.file_type == IdlFileType.MATTER: + return os.path.join(top_dir, "codegen") + elif self.file_type == IdlFileType.ZAP: + return os.path.join(top_dir, "zap") + else: + raise Exception("Unknown file type for self") diff --git a/src/app/chip_data_model.gni b/src/app/chip_data_model.gni index 68aabd7463dda8..ecab5a9d02a465 100644 --- a/src/app/chip_data_model.gni +++ b/src/app/chip_data_model.gni @@ -51,21 +51,30 @@ template("chip_data_model") { _idl = string_replace(invoker.zap_file, ".zap", ".matter") } + config("${_data_model_name}_config") { + include_dirs = [] + + if (defined(invoker.zap_pregenerated_dir)) { + include_dirs += [ "${invoker.zap_pregenerated_dir}/.." ] + } + } + chip_codegen("${_data_model_name}_codegen") { input = _idl generator = "cpp-app" + outputs = [ "app/PluginApplicationCallbacks.h", "app/callback-stub.cpp", ] - } - config("${_data_model_name}_config") { - include_dirs = [] + public_configs = [ ":${_data_model_name}_config" ] - if (defined(invoker.zap_pregenerated_dir)) { - include_dirs += [ "${invoker.zap_pregenerated_dir}/.." ] + if (!defined(deps)) { + deps = [] } + + deps += [ "${chip_root}/src/app/common:cluster-objects" ] } _use_default_im_dispatch = !defined(invoker.use_default_im_dispatch) || @@ -84,7 +93,10 @@ template("chip_data_model") { sources = [] } - sources += get_target_outputs(":${_data_model_name}_codegen") + if (!defined(deps)) { + deps = [] + } + deps += [ ":${_data_model_name}_codegen" ] sources += [ "${_app_root}/clusters/barrier-control-server/barrier-control-server.h", diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn index 011669db09524a..95b0e6cf02072a 100644 --- a/src/controller/data_model/BUILD.gn +++ b/src/controller/data_model/BUILD.gn @@ -31,6 +31,12 @@ chip_data_model("data_model") { } if (current_os == "android" || build_java_matter_controller) { + config("java-build-config") { + if (build_java_matter_controller) { + include_dirs = java_matter_controller_dependent_paths + } + } + chip_codegen("java-jni-generate") { input = "controller-clusters.matter" generator = "java" @@ -167,12 +173,21 @@ if (current_os == "android" || build_java_matter_controller) { "jni/UnitTestingClient-ReadImpl.cpp", "jni/UnitTestingClient-InvokeSubscribeImpl.cpp", ] + + deps = [ + ":data_model", + "${chip_root}/src/platform:platform_buildconfig", + ] + + public_configs = [ ":java-build-config" ] } source_set("java-jni-sources") { - sources = get_target_outputs(":java-jni-generate") + public_configs = [ + ":java-build-config", + "${chip_root}/src:includes", + ] - public_configs = [ "${chip_root}/src:includes" ] deps = [ ":data_model", ":java-jni-generate", @@ -182,7 +197,6 @@ if (current_os == "android" || build_java_matter_controller) { ] if (build_java_matter_controller) { - include_dirs = java_matter_controller_dependent_paths deps += [ "${chip_root}/src/platform/Linux" ] } else { deps += [ "${chip_root}/src/platform/android" ]