From 203125096f65ba1637f16dcfd7af926733dd109d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 1 Dec 2022 17:31:07 -0500 Subject: [PATCH] Avoid usage of a separate build environment for compile time codegen (#23861) * Try to disable virtual env usage for codegen * Fix IDL paths. also add missing callbackstubsource.jinja from the list for change detection * Restyle * Update chip script: need to ensure build environment is set up * Restyle * Fix typo * Fix cirque typo * Switch bootstrap with activate --- build/chip/chip_codegen.gni | 10 +++++++-- scripts/build/gn_gen_cirque.sh | 14 +++++++++---- scripts/idl/BUILD.gn | 33 +++++------------------------- scripts/idl/files.gni | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 scripts/idl/files.gni diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index fd5d69c40c513f..533cd02b27ea76 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -17,6 +17,7 @@ import("//build_overrides/chip.gni") import("//build_overrides/pigweed.gni") import("$dir_pw_build/python.gni") +import("${chip_root}/scripts/idl/files.gni") declare_args() { # Location where code has been pre-generated @@ -37,6 +38,9 @@ template("_chip_build_time_codegen") { pw_python_action("${_name}_codegen") { script = "${chip_root}/scripts/codegen.py" + # TODO: this seems to touch internals. Is this ok? speeds up builds! + _pw_internal_run_in_venv = false + _idl_file = invoker.input _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs" @@ -52,12 +56,14 @@ template("_chip_build_time_codegen") { rebase_path(_idl_file, root_build_dir), ] - deps = [ "${chip_root}/scripts/idl" ] - inputs = [ _idl_file, _expected_outputs, ] + + # ensure any change in codegen files will result in a rebuild + inputs += matter_idl_generator_files + sources = [ _idl_file ] outputs = [] diff --git a/scripts/build/gn_gen_cirque.sh b/scripts/build/gn_gen_cirque.sh index f65525bd6dfb7b..85da0c232745c5 100755 --- a/scripts/build/gn_gen_cirque.sh +++ b/scripts/build/gn_gen_cirque.sh @@ -27,10 +27,16 @@ set -x env -echo "Build: GN configure" +cd "$ROOT_PATH" -gn --root="$CHIP_ROOT" gen --check --fail-on-unused-args "$CHIP_ROOT/out/debug" --args='target_os="all"'"chip_build_tests=false chip_enable_wifi=false chip_im_force_fabric_quota_check=true enable_default_builds=false enable_host_gcc_build=true enable_standalone_chip_tool_build=true enable_linux_all_clusters_app_build=true enable_linux_lighting_app_build=true" +echo "Ensure submodules for Linux builds are checked out" +./scripts/checkout_submodules.py --shallow --platform linux -echo "Build: Ninja build" +echo "Setup build environment" +source "./scripts/activate.sh" -time ninja -C "$CHIP_ROOT/out/debug" all check +echo "Build: GN configure" +gn --root="$CHIP_ROOT" gen --check --fail-on-unused-args out/debug --args='target_os="all"'"chip_build_tests=false chip_enable_wifi=false chip_im_force_fabric_quota_check=true enable_default_builds=false enable_host_gcc_build=true enable_standalone_chip_tool_build=true enable_linux_all_clusters_app_build=true enable_linux_lighting_app_build=true" + +echo "Build: Ninja build" +time ninja -C out/debug all check diff --git a/scripts/idl/BUILD.gn b/scripts/idl/BUILD.gn index 359f50e89d4159..324ed3a737bbf0 100644 --- a/scripts/idl/BUILD.gn +++ b/scripts/idl/BUILD.gn @@ -18,20 +18,15 @@ import("//build_overrides/chip.gni") import("//build_overrides/pigweed.gni") import("$dir_pw_build/python.gni") +import("${chip_root}/scripts/idl/files.gni") + pw_python_package("idl") { setup = [ "setup.py" ] - inputs = [ + inputs = matter_idl_generator_templates + inputs += [ # Dependency grammar "matter_grammar.lark", - # Templates used for generation - "generators/bridge/BridgeClustersCpp.jinja", - "generators/bridge/BridgeClustersCommon.jinja", - "generators/bridge/BridgeClustersGlobalStructs.jinja", - "generators/java/ChipClustersCpp.jinja", - "generators/java/ChipClustersRead.jinja", - "generators/cpp/application/PluginApplicationCallbacksHeader.jinja", - # Unit test data "tests/available_tests.yaml", "tests/inputs/cluster_struct_attribute.matter", @@ -70,25 +65,7 @@ pw_python_package("idl") { "tests/outputs/simple_attribute/jni/MyClusterClient-InvokeSubscribeImpl.cpp", ] - sources = [ - "__init__.py", - "generators/__init__.py", - "generators/bridge/__init__.py", - "generators/cpp/__init__.py", - "generators/cpp/application/__init__.py", - "generators/filters.py", - "generators/java/__init__.py", - "generators/types.py", - "matter_idl_parser.py", - "matter_idl_types.py", - "xml_parser.py", - "zapxml/__init__.py", - "zapxml/handlers/__init__.py", - "zapxml/handlers/base.py", - "zapxml/handlers/context.py", - "zapxml/handlers/handlers.py", - "zapxml/handlers/parsing.py", - ] + sources = matter_idl_generator_sources tests = [ "test_matter_idl_parser.py", diff --git a/scripts/idl/files.gni b/scripts/idl/files.gni new file mode 100644 index 00000000000000..997f1eab8b06c5 --- /dev/null +++ b/scripts/idl/files.gni @@ -0,0 +1,37 @@ +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +# Templates used for generation +matter_idl_generator_templates = [ + "${chip_root}/scripts/idl/generators/bridge/BridgeClustersCpp.jinja", + "${chip_root}/scripts/idl/generators/bridge/BridgeClustersCommon.jinja", + "${chip_root}/scripts/idl/generators/bridge/BridgeClustersGlobalStructs.jinja", + "${chip_root}/scripts/idl/generators/java/ChipClustersCpp.jinja", + "${chip_root}/scripts/idl/generators/java/ChipClustersRead.jinja", + "${chip_root}/scripts/idl/generators/cpp/application/CallbackStubSource.jinja", + "${chip_root}/scripts/idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja", +] + +matter_idl_generator_sources = [ + "${chip_root}/scripts/idl/__init__.py", + "${chip_root}/scripts/idl/generators/__init__.py", + "${chip_root}/scripts/idl/generators/bridge/__init__.py", + "${chip_root}/scripts/idl/generators/cpp/__init__.py", + "${chip_root}/scripts/idl/generators/cpp/application/__init__.py", + "${chip_root}/scripts/idl/generators/filters.py", + "${chip_root}/scripts/idl/generators/java/__init__.py", + "${chip_root}/scripts/idl/generators/types.py", + "${chip_root}/scripts/idl/matter_idl_parser.py", + "${chip_root}/scripts/idl/matter_idl_types.py", + "${chip_root}/scripts/idl/xml_parser.py", + "${chip_root}/scripts/idl/zapxml/__init__.py", + "${chip_root}/scripts/idl/zapxml/handlers/__init__.py", + "${chip_root}/scripts/idl/zapxml/handlers/base.py", + "${chip_root}/scripts/idl/zapxml/handlers/context.py", + "${chip_root}/scripts/idl/zapxml/handlers/handlers.py", + "${chip_root}/scripts/idl/zapxml/handlers/parsing.py", +] + +# All the files that the matter idl infrastructure will use +matter_idl_generator_files = + matter_idl_generator_templates + matter_idl_generator_sources