From 0c906697af42894e9f3c1f7865c255a2f643565d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 9 Aug 2023 13:09:16 -0400 Subject: [PATCH] Separate out python dependencies for running scripts (#28582) * Separate oput requirements for various python scripts * Update certbins to install requirements for test script running instead of hard-coding dependencies * Update text for help in build_python * Fix the requirements install - need to copy the dependencies locally * Update the sha to something that supports the latest options. This is the TE2 for 1.2 SHA * A few more dependencies for py_matter_yamltests ... dependencies there seem off * Revert submodule updates * Update dependencies for chip repl to allow py_matter_idl * Fix dependencies in yaml: need dependencies at runtime not at build time --------- Co-authored-by: Andrei Litvin --- .github/workflows/tests.yaml | 2 +- .../docker/images/chip-cert-bins/Dockerfile | 10 ++++++++-- scripts/build_python.sh | 6 ++++-- scripts/py_matter_idl/setup.cfg | 4 ++++ scripts/py_matter_yamltests/setup.cfg | 4 +++- scripts/setup/requirements.python_tests.txt | 16 ---------------- scripts/tests/requirements.txt | 5 +++++ src/python_testing/requirements.txt | 3 +++ 8 files changed, 28 insertions(+), 22 deletions(-) delete mode 100644 scripts/setup/requirements.python_tests.txt create mode 100644 scripts/tests/requirements.txt create mode 100644 src/python_testing/requirements.txt diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 92a410b9901f90..d34c03d7db29ad 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -427,7 +427,7 @@ jobs: - name: Build Python REPL and example apps run: | - scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv --extra_packages "mobly"' + scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv' ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \ diff --git a/integrations/docker/images/chip-cert-bins/Dockerfile b/integrations/docker/images/chip-cert-bins/Dockerfile index 47d7cc0b606742..6c3872df43102e 100644 --- a/integrations/docker/images/chip-cert-bins/Dockerfile +++ b/integrations/docker/images/chip-cert-bins/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:22.04 as chip-build-cert LABEL org.opencontainers.image.source https://github.com/project-chip/connectedhomeip ARG TARGETPLATFORM # COMMITHASH defines the target commit to build from. May be passed in using --build-arg. -ARG COMMITHASH=7b99e6399c6069037c613782d78132c69b9dcabb +ARG COMMITHASH=c1ec2d777456924dcaa59b53351b00d73caf378f # Ensure TARGETPLATFORM is set RUN case ${TARGETPLATFORM} in \ @@ -268,5 +268,11 @@ COPY --from=chip-build-cert-bins /root/connectedhomeip/out/chip-app1 chip-app1 # Stage 3.1 Setup the Matter Python environment COPY --from=chip-build-cert-bins /root/connectedhomeip/out/python_lib python_lib COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing python_testing -RUN pip install click websockets lark diskcache + +COPY --from=chip-build-cert-bins /root/connectedhomeip/scripts/tests/requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt + +COPY --from=chip-build-cert-bins /root/connectedhomeip/src/python_testing/requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt + RUN pip install --no-cache-dir python_lib/controller/python/chip*.whl diff --git a/scripts/build_python.sh b/scripts/build_python.sh index 08ad42965d5b99..5a0a12f9df84fe 100755 --- a/scripts/build_python.sh +++ b/scripts/build_python.sh @@ -64,7 +64,8 @@ Input Options: represents where the virtual environment is to be created. -c, --clean_virtual_env When installing a virtual environment, create/clean it first. Defaults to yes. - --include_pytest_deps Install requirements.python_tests.txt. + --include_pytest_deps Install requirements.txt for running scripts/tests and + src/python_testing scripts. Defaults to yes. --extra_packages PACKAGES Install extra Python packages from PyPI --include_yamltests Whether to install the matter_yamltests wheel. @@ -214,7 +215,8 @@ if [ -n "$install_virtual_env" ]; then if [ "$install_pytest_requirements" = "yes" ]; then echo_blue "Installing python test dependencies ..." - "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/scripts/setup/requirements.python_tests.txt" + "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/scripts/tests/requirements.txt" + "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/src/python_testing/requirements.txt" fi echo "" diff --git a/scripts/py_matter_idl/setup.cfg b/scripts/py_matter_idl/setup.cfg index 2a7d386375a2e7..57f93c3dd9aa82 100644 --- a/scripts/py_matter_idl/setup.cfg +++ b/scripts/py_matter_idl/setup.cfg @@ -21,6 +21,10 @@ description = Parse matter idl files [options] packages = find: zip_safe = False +install_requires= + lark + jinja2 + stringcase [options.package_data] matter_idl = diff --git a/scripts/py_matter_yamltests/setup.cfg b/scripts/py_matter_yamltests/setup.cfg index 497022ce1459d9..f115c8e28e17be 100644 --- a/scripts/py_matter_yamltests/setup.cfg +++ b/scripts/py_matter_yamltests/setup.cfg @@ -20,5 +20,7 @@ description = Parse matter yaml tests files [options] packages = find: - zip_safe = False +install_requires= + websockets + lark diff --git a/scripts/setup/requirements.python_tests.txt b/scripts/setup/requirements.python_tests.txt deleted file mode 100644 index 92d6db9016814e..00000000000000 --- a/scripts/setup/requirements.python_tests.txt +++ /dev/null @@ -1,16 +0,0 @@ -# scripts/tests -click -colorama -diskcache -websockets - -# scripts/py_matter_idl -# TODO: these should be wheel dependencies -lark -jinja2 -stringcase - -# src/python_testing -mobly -pyasn1 -pyasn1_modules diff --git a/scripts/tests/requirements.txt b/scripts/tests/requirements.txt new file mode 100644 index 00000000000000..d3c26e9cba23c6 --- /dev/null +++ b/scripts/tests/requirements.txt @@ -0,0 +1,5 @@ +# Python requirements for scripts in this location +click +colorama +diskcache +websockets diff --git a/src/python_testing/requirements.txt b/src/python_testing/requirements.txt new file mode 100644 index 00000000000000..84196f3f9e48f2 --- /dev/null +++ b/src/python_testing/requirements.txt @@ -0,0 +1,3 @@ +mobly +pyasn1 +pyasn1_modules