Skip to content

Commit

Permalink
python: New //:python_wheels target
Browse files Browse the repository at this point in the history
- //BUILD.gn
  - Cleanup some existing python related groups
  - Add group("python_wheels") which is used by //scripts/build_python.sh
- //scripts/build_python.sh
  - Install matter_yamltests wheel into the venv if --include_yamltests
  • Loading branch information
AnthonyDiGirolamo committed Feb 15, 2023
1 parent e294edb commit 64a901e
Show file tree
Hide file tree
Showing 7 changed files with 58 additions and 100 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ jobs:
timeout-minutes: 45
if: github.event_name == 'pull_request'
run: |
scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env --include-yamltests'
./scripts/run_in_build_env.sh \
"./scripts/tests/run_test_suite.py \
--run-yamltests-with-chip-repl \
Expand All @@ -255,6 +256,7 @@ jobs:
timeout-minutes: 45
if: github.event_name == 'push'
run: |
scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env --include-yamltests'
./scripts/run_in_build_env.sh \
"./scripts/tests/run_test_suite.py \
--run-yamltests-with-chip-repl \
Expand Down
69 changes: 18 additions & 51 deletions BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -58,56 +58,12 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
}
}

# Pigweed Python packages expected to be used in the :matter_build_venv
# target. If all packages are needed this list should match
# _pigweed_python_deps in:
# https://cs.opensource.google/pigweed/pigweed/+/master:pw_env_setup/BUILD.gn?q=_pigweed_python_deps
_pigweed_python_packages = [
"$dir_pw_allocator/py",
"$dir_pw_arduino_build/py",
"$dir_pw_bloat/py",
"$dir_pw_build/py",
"$dir_pw_build_info/py",
"$dir_pw_build_mcuxpresso/py",
"$dir_pw_cli/py",
"$dir_pw_console/py",
"$dir_pw_cpu_exception_cortex_m/py",
"$dir_pw_docgen/py",
"$dir_pw_doctor/py",
"$dir_pw_env_setup/py",
"$dir_pw_hdlc/py",
"$dir_pw_log:protos.python",
"$dir_pw_log_tokenized/py",
"$dir_pw_metric/py",
"$dir_pw_module/py",
"$dir_pw_package/py",
"$dir_pw_presubmit/py",
"$dir_pw_protobuf/py",
"$dir_pw_protobuf_compiler/py",
"$dir_pw_rpc/py",
"$dir_pw_snapshot/py:pw_snapshot",
"$dir_pw_snapshot/py:pw_snapshot_metadata",
"$dir_pw_software_update/py",
"$dir_pw_status/py",
"$dir_pw_stm32cube_build/py",
"$dir_pw_symbolizer/py",
"$dir_pw_system/py",
"$dir_pw_thread/py",
"$dir_pw_tls_client/py",
"$dir_pw_tokenizer/py",
"$dir_pw_toolchain/py",
"$dir_pw_trace/py",
"$dir_pw_trace_tokenized/py",
"$dir_pw_transfer/py",
"$dir_pw_unit_test/py",
"$dir_pw_watch/py",
]

# Matter's in-tree pw_python_package or pw_python_distribution targets.
_matter_python_packages = [
"//integrations/mobly:chip_mobly",
"//examples/chef",
"//examples/common/pigweed/rpc_console/py:chip_rpc",
"//integrations/mobly:chip_mobly",
"//scripts/py_matter_yamltests:matter_yamltests",
]

pw_python_venv("matter_build_venv") {
Expand All @@ -116,7 +72,8 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
requirements = pw_build_PIP_REQUIREMENTS

# Packages available to import within GN's build venv.
source_packages = _matter_python_packages + _pigweed_python_packages
source_packages = _matter_python_packages +
[ "$dir_pw_env_setup:core_pigweed_python_packages" ]
}

pw_python_pip_install("pip_install_matter_packages") {
Expand All @@ -141,6 +98,19 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
editable = true
}

# Python wheel targets, used by //scripts/build_python.sh
group("python_wheels") {
if (chip_enable_python_modules) {
deps = [
"${chip_root}/scripts:matter_yamltests_distribution.wheel",
"${chip_root}/src/controller/python:chip-repl",
]
if (enable_pylib) {
deps += [ "${chip_root}/src/pybindings/pycontroller" ]
}
}
}

# This is a real toolchain. Build CHIP.
group("default") {
deps = [
Expand Down Expand Up @@ -201,10 +171,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
deps += [ "${chip_root}/src/tools/chip-cert" ]
}
if (chip_enable_python_modules) {
if (enable_pylib) {
deps += [ "${chip_root}/src/pybindings/pycontroller" ]
}
deps += [ "${chip_root}/src/controller/python:chip-repl" ]
deps += [ ":python_wheels" ]
}
}

Expand Down
5 changes: 3 additions & 2 deletions examples/common/pigweed/rpc_console/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,11 @@ group("default") {
pw_python_distribution("chip_rpc_distribution") {
packages = [ "py:chip_rpc" ]
generate_setup_cfg = {
common_config_file = "common_setup.cfg"
name = "chip_rpc"
version = "0.0.1"
include_default_pyproject_file = true
append_date_to_version = true
}
extra_files = [ "py/pyproject.toml > pyproject.toml" ]
}

pw_internal_pip_install("pip_install_editable_chip_rpc") {
Expand Down
19 changes: 0 additions & 19 deletions examples/common/pigweed/rpc_console/common_setup.cfg

This file was deleted.

3 changes: 2 additions & 1 deletion scripts/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import("$dir_pw_build/python_dist.gni")
pw_python_distribution("matter_yamltests_distribution") {
packages = [ "${chip_root}/scripts/py_matter_yamltests:matter_yamltests" ]
generate_setup_cfg = {
common_config_file = "common_setup.cfg"
name = "matter_yamltests"
version = "0.0.1"
include_default_pyproject_file = true
append_date_to_version = true
}
Expand Down
41 changes: 33 additions & 8 deletions scripts/build_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ Input Options:
no: Do not install
build-env: install to virtual env for build matter
separate: install to another virtual env (out/python_env)
--include_yamltests Whether to install the matter_yamltests wheel.
-z --pregen_dir DIRECTORY Directory where generated zap files have been pre-generated.
"
}
Expand Down Expand Up @@ -95,6 +96,9 @@ while (($#)); do
install_wheel=$2
shift
;;
--include_yamltests)
include_yamltests="yes"
;;
--pregen_dir | -z)
pregen_dir=$2
shift
Expand All @@ -121,20 +125,41 @@ source "$CHIP_ROOT/scripts/activate.sh"

gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="chip_detail_logging=$chip_detail_logging enable_pylib=$enable_pybindings enable_rtti=$enable_pybindings chip_project_config_include_dirs=[\"//config/python\"] $chip_mdns_arg $chip_case_retry_arg $pregen_dir_arg"

# Compiles python files
# Check pybindings was requested
if [ "$enable_pybindings" == true ]; then
ninja -C "$OUTPUT_ROOT" pycontroller
else
ninja -C "$OUTPUT_ROOT" chip-repl
fi
function ninja_target() {
# Print the ninja target required to build a gn label.
local GN_LABEL="$1"
local NINJA_TARGET="$(gn ls "$OUTPUT_ROOT" --as=output "$GN_LABEL")"
echo "$NINJA_TARGET"
}

function wheel_output_dir() {
# Print the wheel output directory for a pw_python_package or
# pw_python_distribution. The label must end in "._build_wheel".
local GN_LABEL="$1"
local NINJA_TARGET="$(ninja_target "$GN_LABEL")"
local WHEEL_DIR="$OUTPUT_ROOT"/"$(dirname "$NINJA_TARGET")/$(basename -s .stamp "$NINJA_TARGET")"
echo "$WHEEL_DIR"
}

# Compile Python wheels
ninja -C "$OUTPUT_ROOT" python_wheels

# Add wheels from chip_python_wheel_action templates.
if [ "$enable_pybindings" == true ]; then
WHEEL=("$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl)
else
WHEEL=("$OUTPUT_ROOT"/controller/python/chip*.whl)
fi

if [ -n "$include_yamltests" ]; then
YAMLTESTS_GN_LABEL="//scripts:matter_yamltests_distribution._build_wheel"

# Add wheels from pw_python_package or pw_python_distribution templates.
WHEEL+=(
"$(ls -tr "$(wheel_output_dir "$YAMLTESTS_GN_LABEL")"/*.whl | head -n 1)"
)
fi

if [ "$install_wheel" = "no" ]; then
exit 0
elif [ "$install_wheel" = "separate" ]; then
Expand All @@ -143,7 +168,7 @@ elif [ "$install_wheel" = "separate" ]; then

source "$ENVIRONMENT_ROOT"/bin/activate
"$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
"$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}"
"$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall "${WHEEL[@]}"

echo ""
echo_green "Compilation completed and WHL package installed in: "
Expand Down
19 changes: 0 additions & 19 deletions scripts/common_setup.cfg

This file was deleted.

0 comments on commit 64a901e

Please sign in to comment.