From 0f6e5c651483f57d7ca144cbb7f0e3dd1b8d5c50 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 27 Jun 2023 16:16:14 -0400 Subject: [PATCH 1/7] Add a python unit test requirement file --- scripts/setup/requirements.python_tests.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 scripts/setup/requirements.python_tests.txt diff --git a/scripts/setup/requirements.python_tests.txt b/scripts/setup/requirements.python_tests.txt new file mode 100644 index 00000000000000..6eeaf4d3ed4bab --- /dev/null +++ b/scripts/setup/requirements.python_tests.txt @@ -0,0 +1,8 @@ +# scripts/tests +click +colorama + +# src/python_testing +mobly +pyasn1 +pyasn1_modules From fbaebf1d0d9e3efea768ebb8e359ff216c18ebdd Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 27 Jun 2023 16:24:40 -0400 Subject: [PATCH 2/7] Update build_python to install test dependencies by default --- .github/workflows/tests.yaml | 6 ------ scripts/build_python.sh | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 27c5ac86a1c1e6..a44b72f81262c8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -183,8 +183,6 @@ jobs: - name: Build Apps run: | scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv --include_yamltests' - scripts/run_in_python_env.sh out/venv 'pip install -r scripts/setup/requirements.build.txt' - scripts/run_in_python_env.sh out/venv 'pip install -r scripts/setup/requirements.yaml_tests.txt' ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ --target linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \ @@ -424,8 +422,6 @@ jobs: - name: Build Python REPL and example apps run: | scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv --extra_packages "mobly"' - scripts/run_in_python_env.sh out/venv 'pip install -r scripts/setup/requirements.build.txt' - scripts/run_in_python_env.sh out/venv 'pip install colorama pyasn1 pyasn1_modules' ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \ @@ -499,8 +495,6 @@ jobs: - name: Build Python REPL and example apps run: | scripts/run_in_build_env.sh './scripts/build_python.sh --install_virtual_env out/venv' - scripts/run_in_python_env.sh out/venv 'pip install -r scripts/setup/requirements.build.txt' - scripts/run_in_python_env.sh out/venv 'pip install colorama' ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ --target darwin-x64-all-clusters-${BUILD_VARIANT}-test \ diff --git a/scripts/build_python.sh b/scripts/build_python.sh index 6cc3d09469bb9c..d9754c2b04cff3 100755 --- a/scripts/build_python.sh +++ b/scripts/build_python.sh @@ -43,6 +43,7 @@ declare chip_mdns declare case_retry_delta declare install_virtual_env declare clean_virtual_env=yes +declare install_pytest_requirements=yes help() { @@ -63,6 +64,8 @@ Input Options: represents where the virtual environment is to be created. -c, --clean_virtual_env When installing a virtual environment, create/clean it first. Defaults to yes. + --include_pytest_deps Install requirements.python_tests.txt. + Defaults to yes. --extra_packages PACKAGES Install extra Python packages from PyPI --include_yamltests Whether to install the matter_yamltests wheel. -z --pregen_dir DIRECTORY Directory where generated zap files have been pre-generated. @@ -101,6 +104,10 @@ while (($#)); do clean_virtual_env=$2 shift ;; + --include_pytest_deps) + install_pytest_requirements=$2 + shift + ;; --extra_packages) extra_packages=$2 shift @@ -178,13 +185,22 @@ if [ -n "$install_virtual_env" ]; then if [ "$clean_virtual_env" = "yes" ]; then # Create a virtual environment that has access to the built python tools + echo_blue "Creating a clear VirtualEnv in '$ENVIRONMENT_ROOT' ..." virtualenv --clear "$ENVIRONMENT_ROOT" + elif [ ! -f "$ENVIRONMENT_ROOT"/bin/activate ]; then + echo_blue "Creating a new VirtualEnv in '$ENVIRONMENT_ROOT' ..." + virtualenv "$ENVIRONMENT_ROOT" fi source "$ENVIRONMENT_ROOT"/bin/activate "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip "$ENVIRONMENT_ROOT"/bin/pip install --upgrade "${WHEEL[@]}" + if [ "$install_pytest_requirements" = "yes" ]; then + echo_blue "Installing python test dependencies ..." + "$ENVIRONMENT_ROOT"/bin/pip install -r "${CHIP_ROOT}/scripts/setup/requirements.python_tests.txt" + fi + echo "" echo_green "Compilation completed and WHL package installed in: " echo_blue " $ENVIRONMENT_ROOT" From e8b05b9142932a91839cb77070fa61863bdb0fd1 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 27 Jun 2023 16:29:06 -0400 Subject: [PATCH 3/7] Restyle --- scripts/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_python.sh b/scripts/build_python.sh index d9754c2b04cff3..0952f3adbf7386 100755 --- a/scripts/build_python.sh +++ b/scripts/build_python.sh @@ -198,7 +198,7 @@ if [ -n "$install_virtual_env" ]; then if [ "$install_pytest_requirements" = "yes" ]; then echo_blue "Installing python test dependencies ..." - "$ENVIRONMENT_ROOT"/bin/pip install -r "${CHIP_ROOT}/scripts/setup/requirements.python_tests.txt" + "$ENVIRONMENT_ROOT"/bin/pip install -r "$CHIP_ROOT/scripts/setup/requirements.python_tests.txt" fi echo "" From 7cfdf7293d727f0e7b7718d19bbdc248914049f8 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 28 Jun 2023 08:43:55 -0400 Subject: [PATCH 4/7] Add diskcache as a requirement for yaml tests --- scripts/setup/requirements.python_tests.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/setup/requirements.python_tests.txt b/scripts/setup/requirements.python_tests.txt index 6eeaf4d3ed4bab..58731869d8069e 100644 --- a/scripts/setup/requirements.python_tests.txt +++ b/scripts/setup/requirements.python_tests.txt @@ -6,3 +6,6 @@ colorama mobly pyasn1 pyasn1_modules + +# scripts/tests/yaml/paths_finder.py +diskcache From 6b4bd4bdf408825c2a34756b3debdcbbfca5eddd Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 28 Jun 2023 08:44:54 -0400 Subject: [PATCH 5/7] Add websockets as well since python yaml tests need it and I expect python tests to want both --- scripts/setup/requirements.python_tests.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/setup/requirements.python_tests.txt b/scripts/setup/requirements.python_tests.txt index 58731869d8069e..5f705fe72da3d9 100644 --- a/scripts/setup/requirements.python_tests.txt +++ b/scripts/setup/requirements.python_tests.txt @@ -1,11 +1,10 @@ # scripts/tests click colorama +diskcache +websockets # src/python_testing mobly pyasn1 pyasn1_modules - -# scripts/tests/yaml/paths_finder.py -diskcache From e633140134513b110d3f34dda1a67871c631be4a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 28 Jun 2023 11:06:33 -0400 Subject: [PATCH 6/7] lark/jinja2 wheel requirements do not seem to take effect. Add them --- scripts/setup/requirements.python_tests.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/setup/requirements.python_tests.txt b/scripts/setup/requirements.python_tests.txt index 5f705fe72da3d9..fbb014cfc3920a 100644 --- a/scripts/setup/requirements.python_tests.txt +++ b/scripts/setup/requirements.python_tests.txt @@ -4,6 +4,11 @@ colorama diskcache websockets +# scripts/py_matter_idl +# TODO: these should be wheel dependencies +lark +jinja2 + # src/python_testing mobly pyasn1 From 93483f84be63759618b82682455cfd29c09f0d3f Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 28 Jun 2023 15:46:36 -0400 Subject: [PATCH 7/7] Add stringcase dependency --- scripts/setup/requirements.python_tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup/requirements.python_tests.txt b/scripts/setup/requirements.python_tests.txt index fbb014cfc3920a..92d6db9016814e 100644 --- a/scripts/setup/requirements.python_tests.txt +++ b/scripts/setup/requirements.python_tests.txt @@ -8,6 +8,7 @@ websockets # TODO: these should be wheel dependencies lark jinja2 +stringcase # src/python_testing mobly