From 64a901e144fb00bdf785f8fb34f7fa749a2002d8 Mon Sep 17 00:00:00 2001 From: Anthony DiGirolamo Date: Mon, 13 Feb 2023 11:48:10 -0800 Subject: [PATCH] python: New //:python_wheels target - //BUILD.gn - Cleanup some existing python related groups - Add group("python_wheels") which is used by //scripts/build_python.sh - //scripts/build_python.sh - Install matter_yamltests wheel into the venv if --include_yamltests --- .github/workflows/tests.yaml | 2 + BUILD.gn | 69 +++++-------------- examples/common/pigweed/rpc_console/BUILD.gn | 5 +- .../pigweed/rpc_console/common_setup.cfg | 19 ----- scripts/BUILD.gn | 3 +- scripts/build_python.sh | 41 ++++++++--- scripts/common_setup.cfg | 19 ----- 7 files changed, 58 insertions(+), 100 deletions(-) delete mode 100644 examples/common/pigweed/rpc_console/common_setup.cfg delete mode 100644 scripts/common_setup.cfg diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ae86fe217ee5cf..acc6a8ca760671 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -234,6 +234,7 @@ jobs: timeout-minutes: 45 if: github.event_name == 'pull_request' run: | + scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env --include-yamltests' ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ --run-yamltests-with-chip-repl \ @@ -255,6 +256,7 @@ jobs: timeout-minutes: 45 if: github.event_name == 'push' run: | + scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env --include-yamltests' ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ --run-yamltests-with-chip-repl \ diff --git a/BUILD.gn b/BUILD.gn index 5bb1f218fe581c..96ab7addb4a9ce 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -58,56 +58,12 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { } } - # Pigweed Python packages expected to be used in the :matter_build_venv - # target. If all packages are needed this list should match - # _pigweed_python_deps in: - # https://cs.opensource.google/pigweed/pigweed/+/master:pw_env_setup/BUILD.gn?q=_pigweed_python_deps - _pigweed_python_packages = [ - "$dir_pw_allocator/py", - "$dir_pw_arduino_build/py", - "$dir_pw_bloat/py", - "$dir_pw_build/py", - "$dir_pw_build_info/py", - "$dir_pw_build_mcuxpresso/py", - "$dir_pw_cli/py", - "$dir_pw_console/py", - "$dir_pw_cpu_exception_cortex_m/py", - "$dir_pw_docgen/py", - "$dir_pw_doctor/py", - "$dir_pw_env_setup/py", - "$dir_pw_hdlc/py", - "$dir_pw_log:protos.python", - "$dir_pw_log_tokenized/py", - "$dir_pw_metric/py", - "$dir_pw_module/py", - "$dir_pw_package/py", - "$dir_pw_presubmit/py", - "$dir_pw_protobuf/py", - "$dir_pw_protobuf_compiler/py", - "$dir_pw_rpc/py", - "$dir_pw_snapshot/py:pw_snapshot", - "$dir_pw_snapshot/py:pw_snapshot_metadata", - "$dir_pw_software_update/py", - "$dir_pw_status/py", - "$dir_pw_stm32cube_build/py", - "$dir_pw_symbolizer/py", - "$dir_pw_system/py", - "$dir_pw_thread/py", - "$dir_pw_tls_client/py", - "$dir_pw_tokenizer/py", - "$dir_pw_toolchain/py", - "$dir_pw_trace/py", - "$dir_pw_trace_tokenized/py", - "$dir_pw_transfer/py", - "$dir_pw_unit_test/py", - "$dir_pw_watch/py", - ] - # Matter's in-tree pw_python_package or pw_python_distribution targets. _matter_python_packages = [ - "//integrations/mobly:chip_mobly", "//examples/chef", "//examples/common/pigweed/rpc_console/py:chip_rpc", + "//integrations/mobly:chip_mobly", + "//scripts/py_matter_yamltests:matter_yamltests", ] pw_python_venv("matter_build_venv") { @@ -116,7 +72,8 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { requirements = pw_build_PIP_REQUIREMENTS # Packages available to import within GN's build venv. - source_packages = _matter_python_packages + _pigweed_python_packages + source_packages = _matter_python_packages + + [ "$dir_pw_env_setup:core_pigweed_python_packages" ] } pw_python_pip_install("pip_install_matter_packages") { @@ -141,6 +98,19 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { editable = true } + # Python wheel targets, used by //scripts/build_python.sh + group("python_wheels") { + if (chip_enable_python_modules) { + deps = [ + "${chip_root}/scripts:matter_yamltests_distribution.wheel", + "${chip_root}/src/controller/python:chip-repl", + ] + if (enable_pylib) { + deps += [ "${chip_root}/src/pybindings/pycontroller" ] + } + } + } + # This is a real toolchain. Build CHIP. group("default") { deps = [ @@ -201,10 +171,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { deps += [ "${chip_root}/src/tools/chip-cert" ] } if (chip_enable_python_modules) { - if (enable_pylib) { - deps += [ "${chip_root}/src/pybindings/pycontroller" ] - } - deps += [ "${chip_root}/src/controller/python:chip-repl" ] + deps += [ ":python_wheels" ] } } diff --git a/examples/common/pigweed/rpc_console/BUILD.gn b/examples/common/pigweed/rpc_console/BUILD.gn index 2976f5688d7445..14bac8e686b65f 100644 --- a/examples/common/pigweed/rpc_console/BUILD.gn +++ b/examples/common/pigweed/rpc_console/BUILD.gn @@ -32,10 +32,11 @@ group("default") { pw_python_distribution("chip_rpc_distribution") { packages = [ "py:chip_rpc" ] generate_setup_cfg = { - common_config_file = "common_setup.cfg" + name = "chip_rpc" + version = "0.0.1" + include_default_pyproject_file = true append_date_to_version = true } - extra_files = [ "py/pyproject.toml > pyproject.toml" ] } pw_internal_pip_install("pip_install_editable_chip_rpc") { diff --git a/examples/common/pigweed/rpc_console/common_setup.cfg b/examples/common/pigweed/rpc_console/common_setup.cfg deleted file mode 100644 index af1401ac1d5658..00000000000000 --- a/examples/common/pigweed/rpc_console/common_setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2022 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -[metadata] -name = chip_rpc -version = 0.0.1 - -[options] -zip_safe = False diff --git a/scripts/BUILD.gn b/scripts/BUILD.gn index e14db546195de5..0ebcab979112ca 100644 --- a/scripts/BUILD.gn +++ b/scripts/BUILD.gn @@ -25,7 +25,8 @@ import("$dir_pw_build/python_dist.gni") pw_python_distribution("matter_yamltests_distribution") { packages = [ "${chip_root}/scripts/py_matter_yamltests:matter_yamltests" ] generate_setup_cfg = { - common_config_file = "common_setup.cfg" + name = "matter_yamltests" + version = "0.0.1" include_default_pyproject_file = true append_date_to_version = true } diff --git a/scripts/build_python.sh b/scripts/build_python.sh index 866244c7e5379b..1af80368cdccf7 100755 --- a/scripts/build_python.sh +++ b/scripts/build_python.sh @@ -63,6 +63,7 @@ Input Options: no: Do not install build-env: install to virtual env for build matter separate: install to another virtual env (out/python_env) + --include_yamltests Whether to install the matter_yamltests wheel. -z --pregen_dir DIRECTORY Directory where generated zap files have been pre-generated. " } @@ -95,6 +96,9 @@ while (($#)); do install_wheel=$2 shift ;; + --include_yamltests) + include_yamltests="yes" + ;; --pregen_dir | -z) pregen_dir=$2 shift @@ -121,20 +125,41 @@ source "$CHIP_ROOT/scripts/activate.sh" gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="chip_detail_logging=$chip_detail_logging enable_pylib=$enable_pybindings enable_rtti=$enable_pybindings chip_project_config_include_dirs=[\"//config/python\"] $chip_mdns_arg $chip_case_retry_arg $pregen_dir_arg" -# Compiles python files -# Check pybindings was requested -if [ "$enable_pybindings" == true ]; then - ninja -C "$OUTPUT_ROOT" pycontroller -else - ninja -C "$OUTPUT_ROOT" chip-repl -fi +function ninja_target() { + # Print the ninja target required to build a gn label. + local GN_LABEL="$1" + local NINJA_TARGET="$(gn ls "$OUTPUT_ROOT" --as=output "$GN_LABEL")" + echo "$NINJA_TARGET" +} +function wheel_output_dir() { + # Print the wheel output directory for a pw_python_package or + # pw_python_distribution. The label must end in "._build_wheel". + local GN_LABEL="$1" + local NINJA_TARGET="$(ninja_target "$GN_LABEL")" + local WHEEL_DIR="$OUTPUT_ROOT"/"$(dirname "$NINJA_TARGET")/$(basename -s .stamp "$NINJA_TARGET")" + echo "$WHEEL_DIR" +} + +# Compile Python wheels +ninja -C "$OUTPUT_ROOT" python_wheels + +# Add wheels from chip_python_wheel_action templates. if [ "$enable_pybindings" == true ]; then WHEEL=("$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl) else WHEEL=("$OUTPUT_ROOT"/controller/python/chip*.whl) fi +if [ -n "$include_yamltests" ]; then + YAMLTESTS_GN_LABEL="//scripts:matter_yamltests_distribution._build_wheel" + + # Add wheels from pw_python_package or pw_python_distribution templates. + WHEEL+=( + "$(ls -tr "$(wheel_output_dir "$YAMLTESTS_GN_LABEL")"/*.whl | head -n 1)" + ) +fi + if [ "$install_wheel" = "no" ]; then exit 0 elif [ "$install_wheel" = "separate" ]; then @@ -143,7 +168,7 @@ elif [ "$install_wheel" = "separate" ]; then source "$ENVIRONMENT_ROOT"/bin/activate "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip - "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}" + "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall "${WHEEL[@]}" echo "" echo_green "Compilation completed and WHL package installed in: " diff --git a/scripts/common_setup.cfg b/scripts/common_setup.cfg deleted file mode 100644 index 76987454de4736..00000000000000 --- a/scripts/common_setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2022 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -[metadata] -name = matter_yamltests -version = 0.0.1 - -[options] -zip_safe = False