Code pre-generation support (#23763)

* Start implementing a pregenerator

* Start moving pregenerate logic into a separate directory

* Start adding some ability to figure out pregeneration output locations

* Pregeneration for bridge seems to work

* Add missing files

* Restyle

* Better log level logic

* Pregeneration of java also exists

* Allow pregeneration of cpp app data. Full codegen usage is now enabled

* Move sdk root around

* Make things run

* Restyle

* Parallel code pregen - can do pregen in 1.08 seconds on my machine

* Restyle

* Make sure pregen folders are split

* Pregeneration compile when using GN works

* Restyle

* Direct pregen usage works now

* Restyle

* Minor sort

* Add support for both pregen and no pregen

* Restyle

* Fix pregen dir output logic

* Support pregenerated directory in build_examples.py

* Fix gn build logic

* Somewhat simpler code for parallel vs serial codegen

* Use imap_unordered to not care about actual parallel generation order

* Also fix java jni codegen with pregenerated data

* Fix java compilation deps: java codegen uses data model files

* NRF now can use pregen folder

* Allow telink to also use a pregen dir

* Better shell escape, make mbedos cmake flags work with pregen dir

* Restyle

* Add pregen support for esp32 as well

* Add a test for esp32 pregeneration support

* Also test pregen support in linux builds (to check gn builds)

* Remove unused file

* Fix spelling

* Fix esp32 compilation - gn arguments need to be passed from cmake

* Fix some define forwarding logic in codegen

* Make sure java build config (which includes header paths) is set as a config and applies to generated sources

* java build config should apply to all sources, not just transitively

* Restyle

* Replace codege with codegen.

* Fix naming typo

* Update text for build steps

* Update spacing logic to make the code cleaner. The nospace args is odd

.github/workflows/build.yaml

@@ -229,12 +229,28 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                     "./scripts/build/build_examples.py --no-log-timestamps \
-                       --target linux-x64-all-clusters-ipv6only-clang \
-                       --target linux-x64-chip-tool-ipv6only-clang \
                        --target linux-x64-minmdns-ipv6only-clang \
                        --target linux-x64-rpc-console \
                        build \
                     "
+            - name: Create a pre-generate directory and ensure compile-time codegen would fail
+              run: |
+                  ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated"
+                  mv scripts/codegen.py scripts/codegen.py.renamed
+            - name: Build using build_examples.py (pregen)
+              timeout-minutes: 60
+              run: |
+                  ./scripts/run_in_build_env.sh \
+                    "./scripts/build/build_examples.py --no-log-timestamps \
+                       --target linux-x64-all-clusters-ipv6only-clang \
+                       --target linux-x64-chip-tool-ipv6only-clang \
+                       --pregen-dir ./zzz_pregenerated \
+                       build \
+                    "
+            - name: Undo code pre-generation changes (make compile time codegen work again)
+              run: |
+                  rm -rf ./zzz_pregenerated
+                  mv scripts/codegen.py.renamed scripts/codegen.py
             - name: Run fake linux tests with build_examples
               timeout-minutes: 15
               run: |

.github/workflows/examples-esp32.yaml

@@ -71,11 +71,29 @@ jobs:
                      "./scripts/build/build_examples.py \
                         --enable-flashbundle \
                         --target esp32-m5stack-all-clusters \
+                        build \
+                        --copy-artifacts-to out/artifacts \
+                     "
+            - name: Prepare code pregen and ensure compile time pregen not possible
+              run: |
+                  ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated"
+                  mv scripts/codegen.py scripts/codegen.py.renamed
+            - name: Build some M5Stack variations with pregen
+              timeout-minutes: 60
+              run: |
+                  ./scripts/run_in_build_env.sh \
+                     "./scripts/build/build_examples.py \
+                        --enable-flashbundle \
                         --target esp32-m5stack-all-clusters-minimal \
                         --target esp32-m5stack-all-clusters-rpc-ipv6only \
+                        --pregen-dir ./zzz_pregenerated \
                         build \
                         --copy-artifacts-to out/artifacts \
                      "
+            - name: Undo code pregeneration changes
+              run: |
+                  rm -rf ./zzz_pregenerated
+                  mv scripts/codegen.py.renamed scripts/codegen.py
             - name: Prepare bloat report
               run: |
                   .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \

build/chip/chip_codegen.cmake

@@ -31,40 +31,71 @@ function(chip_codegen TARGET_NAME)
          ${ARGN}
     )
 
-    set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
+    set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.")
 
-    string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
+    if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "")
+        set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
 
-    file(MAKE_DIRECTORY "${GEN_FOLDER}")
-    file(GENERATE
-        OUTPUT "${GEN_FOLDER}/expected.outputs"
-        CONTENT "${OUTPUT_AS_NEWLINES}"
-    )
+        string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
 
+        file(MAKE_DIRECTORY "${GEN_FOLDER}")
+        file(GENERATE
+            OUTPUT "${GEN_FOLDER}/expected.outputs"
+            CONTENT "${OUTPUT_AS_NEWLINES}"
+        )
+
+
+        set(OUT_NAMES)
+        foreach(NAME IN LISTS ARG_OUTPUTS)
+            list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+        endforeach()
+
+        # Python is expected to be in the path
+        #
+        # find_package(Python3 REQUIRED)
+        add_custom_command(
+            OUTPUT ${OUT_NAMES}
+            COMMAND "${CHIP_ROOT}/scripts/codegen.py"
+            ARGS "--generator" "${ARG_GENERATOR}"
+                 "--output-dir" "${GEN_FOLDER}"
+                 "--expected-outputs" "${GEN_FOLDER}/expected.outputs"
+                 "${ARG_INPUT}"
+            DEPENDS
+                "${ARG_INPUT}"
+            VERBATIM
+        )
+
+        add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
+
+        # Forward outputs to the parent
+        set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+        set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+    else()
+        # Gets a path such as:
+        #    examples/lock-app/lock-common/lock-app.matter
+        file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT})
+
+        # Removes the trailing file extension to get something like:
+        #    examples/lock-app/lock-common/lock-app
+        string(REGEX REPLACE "\.matter$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}")
+
+
+        # Build the final location within the pregen directory
+        set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/codegen/${ARG_GENERATOR}")
+
+        # TODO: build a fake target of ${TARGET_NAME}
+
+        # Here we have ${CHIP_CODEGEN_PREGEN_DIR}
+        set(OUT_NAMES)
+        foreach(NAME IN LISTS ARG_OUTPUTS)
+            list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+        endforeach()
 
-    set(OUT_NAMES)
-    foreach(NAME IN LISTS ARG_OUTPUTS)
-        list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
-    endforeach()
-
-    # Python is expected to be in the path
-    #
-    # find_package(Python3 REQUIRED)
-    add_custom_command(
-        OUTPUT ${OUT_NAMES}
-        COMMAND "${CHIP_ROOT}/scripts/codegen.py"
-        ARGS "--generator" "${ARG_GENERATOR}"
-             "--output-dir" "${GEN_FOLDER}"
-             "--expected-outputs" "${GEN_FOLDER}/expected.outputs"
-             "${ARG_INPUT}"
-        DEPENDS
-            "${ARG_INPUT}"
-        VERBATIM
-    )
 
-    add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
+        set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+        set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
 
-    # Forward outputs to the parent
-    set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
-    set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+        # allow adding dependencies to a phony target since no codegen is done
+        add_custom_target(${TARGET_NAME})
+    endif()
 endfunction()

build/chip/chip_codegen.gni

@@ -18,6 +18,75 @@ import("//build_overrides/pigweed.gni")
 
 import("$dir_pw_build/python.gni")
 
+declare_args() {
+  # Location where code has been pre-generated
+  chip_code_pre_generated_directory = ""
+}
+
+# Code generation that will happen at build time.
+#
+#
+template("_chip_build_time_codegen") {
+  _name = target_name
+  _generator = invoker.generator
+
+  config("${_name}_config") {
+    include_dirs = [ target_gen_dir ]
+  }
+
+  pw_python_action("${_name}_codegen") {
+    script = "${chip_root}/scripts/codegen.py"
+
+    _idl_file = invoker.input
+    _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs"
+
+    write_file(_expected_outputs, invoker.outputs, "list lines")
+
+    args = [
+      "--generator",
+      _generator,
+      "--output-dir",
+      rebase_path(target_gen_dir, root_build_dir),
+      "--expected-outputs",
+      rebase_path(_expected_outputs, root_build_dir),
+      rebase_path(_idl_file, root_build_dir),
+    ]
+
+    deps = [ "${chip_root}/scripts/idl" ]
+
+    inputs = [
+      _idl_file,
+      _expected_outputs,
+    ]
+    sources = [ _idl_file ]
+
+    outputs = []
+    foreach(name, invoker.outputs) {
+      outputs += [ "${target_gen_dir}/${name}" ]
+    }
+  }
+
+  source_set(_name) {
+    sources = []
+    foreach(name, invoker.outputs) {
+      sources += [ "${target_gen_dir}/${name}" ]
+    }
+
+    public_configs = [ ":${_name}_config" ]
+
+    if (defined(invoker.public_configs)) {
+      public_configs += invoker.public_configs
+    }
+
+    forward_variables_from(invoker, [ "deps" ])
+
+    if (!defined(deps)) {
+      deps = []
+    }
+    deps += [ ":${_name}_codegen" ]
+  }
+}
+
 # Defines a target that runs code generation based on
 # scripts/codegen.py
 #
@@ -32,13 +101,29 @@ import("$dir_pw_build/python.gni")
 #     Explicit names of the expected outputs. Enforced to validate that
 #     expected outputs are generated when processing input files.
 #
+#   deps, public_configs
+#     Forwarded to the resulting source set
+#
+# Command line parameters:
+#
+#  chip_code_pre_generated_directory:
+#     - If this is set, generation will NOT happen at compile time but rather
+#       the code generation is assumed to have already happened and reside in
+#       the given location.
+#     - The TOP LEVEL directory is assumed to be given. Actual location for
+#       individual generators is expected to be of the form
+#       <top_dir>/<matter_path>/<generator>
+#
 # NOTE: content of "outputs" is verified to match the output of codegen.py
 #       exactly. It is not inferred on purpose, to make build-rules explicit
-#       and verifiable (even though codege.py can at runtime report its outputs)
+#       and verifiable (even though codegen.py can at runtime report its outputs)
 #
 #       To find the list of generated files, you can run codegen.py with the
 #       "--name-only" argument
 #
+# NOTE:
+#   the result of the target_name WILL BE a `source_set`. Treat it as such.
+#
 # Example usage:
 #
 #  chip_codegen("java-jni-generate") {
@@ -53,43 +138,46 @@ import("$dir_pw_build/python.gni")
 #  }
 #
 template("chip_codegen") {
-  _name = target_name
-  _generator = invoker.generator
-
-  config("${_name}_config") {
-    include_dirs = [ target_gen_dir ]
-  }
+  if (chip_code_pre_generated_directory == "") {
+    _chip_build_time_codegen(target_name) {
+      forward_variables_from(invoker,
+                             [
+                               "deps",
+                               "generator",
+                               "input",
+                               "outputs",
+                               "public_configs",
+                             ])
+    }
+  } else {
+    _name = target_name
 
-  pw_python_action(_name) {
-    script = "${chip_root}/scripts/codegen.py"
+    # This contstructs a path like:
+    #  FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/)
+    #  USING "cpp-app" for generator:
+    #    => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app
+    _generation_dir =
+        chip_code_pre_generated_directory + "/" +
+        string_replace(rebase_path(invoker.input, chip_root), ".matter", "") +
+        "/codegen/" + invoker.generator
 
-    _idl_file = invoker.input
-    _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs"
-
-    write_file(_expected_outputs, invoker.outputs, "list lines")
+    config("${_name}_config") {
+      include_dirs = [ "${_generation_dir}" ]
+    }
 
-    args = [
-      "--generator",
-      _generator,
-      "--output-dir",
-      rebase_path(target_gen_dir, root_build_dir),
-      "--expected-outputs",
-      rebase_path(_expected_outputs, root_build_dir),
-      rebase_path(_idl_file, root_build_dir),
-    ]
+    source_set(_name) {
+      public_configs = [ ":${_name}_config" ]
 
-    deps = [ "${chip_root}/scripts/idl" ]
-    public_configs = [ ":${_name}_config" ]
+      if (defined(invoker.public_configs)) {
+        public_configs += invoker.public_configs
+      }
 
-    inputs = [
-      _idl_file,
-      _expected_outputs,
-    ]
-    sources = [ _idl_file ]
+      forward_variables_from(invoker, [ "deps" ])
 
-    outputs = []
-    foreach(name, invoker.outputs) {
-      outputs += [ "${target_gen_dir}/${name}" ]
+      sources = []
+      foreach(name, invoker.outputs) {
+        sources += [ "${_generation_dir}/${name}" ]
+      }
     }
   }
 }

config/esp32/components/chip/CMakeLists.txt

@@ -101,6 +101,10 @@ if(CONFIG_DISABLE_IPV4)
     chip_gn_arg_append("chip_inet_config_enable_ipv4"        "false")
 endif()
 
+if(CHIP_CODEGEN_PREGEN_DIR)
+    chip_gn_arg_append("chip_code_pre_generated_directory"  "\"${CHIP_CODEGEN_PREGEN_DIR}\"")
+endif()
+
 if(CONFIG_ENABLE_PW_RPC)
     string(APPEND chip_gn_args "import(\"//build_overrides/pigweed.gni\")\n")
     chip_gn_arg_append("remove_default_configs"             "[\"//third_party/connectedhomeip/third_party/pigweed/repo/pw_build:toolchain_cpp_standard\"]")

docs/code_generation.md

@@ -176,12 +176,32 @@ enerated
 
 ### `*.matter` code generation
 
-Currently `*.matter` code generation is done at compile time.
+`*.matter` code generation can be done either at compile time or it can use
+pre-generated output.
 
 Rules for how `codegen.py` is invoked and how includes/sources are set are
 defined at:
 
 -   `src/app/chip_data_model.cmake`
--   `build/chip/esp32/esp32_codegen.cmake` (support for 2-pass cmake builds used
-    by the Espressif `idf.py` build system)
 -   `src/app/chip_data_model.gni`
+
+Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for
+the 2-pass cmake builds used by the Espressif `idf.py` build system.
+
+## Pre-generation
+
+Code pre-generation can be used:
+
+-   when compile-time code generation is not desirable. This may be for
+    importing into build systems that do not have the pre-requisites to run code
+    generation at build time or to save the code generation time at the expense
+    of running code generation for every possible zap/generation type
+-   To check changes in generated code across versions, beyond the comparisons
+    of golden image tests in `scripts/idl/tests`
+
+The script to trigger code pre-generation is `scripts/code_pregenerate.py` and
+requires the pre-generation output directory as an argument
+
+```bash
+scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
+```