From 496db90f7670c2d05f6faca5d0b336bd6917c4ac Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 16 Apr 2023 18:13:23 +0200 Subject: [PATCH] Parallelize Helm tests with multiple job runners Helm Unit tests are using template rendering and the rendering uses a lot of CPU for `helm template command`. We have a lot of those rendering tests (>800) so even running the tests in parallel on a multi-cpu machine does not lead to a decreased elapsed time to execute the tests. However, each of the tests is run entirely independently and we should be able to achieve much faster elapsed time if we run a subset of tetsts on separate, multi-CPU machine. This will not lower the job build time, however it might speed up elapsed time and thus get a faster feedback. This PR achieves that. --- .github/workflows/ci.yml | 11 ++++++--- Dockerfile.ci | 15 +++++++++--- TESTING.rst | 12 +++++++++- .../commands/testing_commands.py | 17 +++++++++++-- .../commands/testing_commands_config.py | 1 + .../src/airflow_breeze/global_constants.py | 17 +++++++++++++ .../airflow_breeze/utils/selective_checks.py | 5 ++++ docs/exts/docs_build/helm_chart_utils.py | 3 ++- images/breeze/output-commands-hash.txt | 4 ++-- images/breeze/output_testing_helm-tests.svg | 24 +++++++++++-------- scripts/ci/docker-compose/_docker.env | 1 + scripts/ci/docker-compose/base.yml | 1 + scripts/ci/docker-compose/devcontainer.env | 1 + .../ci/pre_commit/pre_commit_chart_schema.py | 2 +- scripts/docker/entrypoint_ci.sh | 15 +++++++++--- tests/charts/airflow_aux/__init__.py | 16 +++++++++++++ .../{ => airflow_aux}/test_airflow_common.py | 0 .../{ => airflow_aux}/test_annotations.py | 0 .../test_basic_helm_chart.py | 0 .../test_celery_kubernetes_executor.py | 0 .../{ => airflow_aux}/test_chart_quality.py | 2 +- .../{ => airflow_aux}/test_cleanup_pods.py | 0 .../{ => airflow_aux}/test_configmap.py | 0 .../{ => airflow_aux}/test_create_user_job.py | 0 .../test_extra_env_env_from.py | 0 .../test_logs_persistent_volume_claim.py | 0 .../test_migrate_database_job.py | 0 .../test_pod_launcher_role.py | 0 .../test_pod_template_file.py | 2 +- tests/charts/airflow_core/__init__.py | 16 +++++++++++++ .../{ => airflow_core}/test_dag_processor.py | 0 .../{ => airflow_core}/test_pdb_scheduler.py | 0 .../{ => airflow_core}/test_scheduler.py | 0 .../{ => airflow_core}/test_triggerer.py | 0 .../charts/{ => airflow_core}/test_worker.py | 0 tests/charts/helm_template_generator.py | 4 ++-- tests/charts/other/__init__.py | 16 +++++++++++++ .../test_dags_persistent_volume_claim.py | 0 tests/charts/{ => other}/test_flower.py | 0 .../{ => other}/test_git_sync_scheduler.py | 0 .../{ => other}/test_git_sync_triggerer.py | 0 .../{ => other}/test_git_sync_webserver.py | 0 .../{ => other}/test_git_sync_worker.py | 0 tests/charts/{ => other}/test_keda.py | 0 tests/charts/{ => other}/test_limit_ranges.py | 0 .../charts/{ => other}/test_pdb_pgbouncer.py | 0 tests/charts/{ => other}/test_pgbouncer.py | 0 tests/charts/{ => other}/test_redis.py | 0 .../charts/{ => other}/test_resource_quota.py | 0 tests/charts/{ => other}/test_statsd.py | 0 tests/charts/security/__init__.py | 16 +++++++++++++ .../test_elasticsearch_secret.py | 0 .../test_extra_configmaps_secrets.py | 0 tests/charts/{ => security}/test_kerberos.py | 0 .../test_metadata_connection_secret.py | 0 tests/charts/{ => security}/test_rbac.py | 0 .../test_rbac_pod_log_reader.py | 0 .../test_result_backend_connection_secret.py | 0 .../{ => security}/test_scc_rolebinding.py | 0 .../{ => security}/test_security_context.py | 0 tests/charts/webserver/__init__.py | 16 +++++++++++++ .../{ => webserver}/test_ingress_flower.py | 0 .../{ => webserver}/test_ingress_web.py | 0 .../{ => webserver}/test_pdb_webserver.py | 0 .../charts/{ => webserver}/test_webserver.py | 0 65 files changed, 187 insertions(+), 30 deletions(-) create mode 100644 tests/charts/airflow_aux/__init__.py rename tests/charts/{ => airflow_aux}/test_airflow_common.py (100%) rename tests/charts/{ => airflow_aux}/test_annotations.py (100%) rename tests/charts/{ => airflow_aux}/test_basic_helm_chart.py (100%) rename tests/charts/{ => airflow_aux}/test_celery_kubernetes_executor.py (100%) rename tests/charts/{ => airflow_aux}/test_chart_quality.py (96%) rename tests/charts/{ => airflow_aux}/test_cleanup_pods.py (100%) rename tests/charts/{ => airflow_aux}/test_configmap.py (100%) rename tests/charts/{ => airflow_aux}/test_create_user_job.py (100%) rename tests/charts/{ => airflow_aux}/test_extra_env_env_from.py (100%) rename tests/charts/{ => airflow_aux}/test_logs_persistent_volume_claim.py (100%) rename tests/charts/{ => airflow_aux}/test_migrate_database_job.py (100%) rename tests/charts/{ => airflow_aux}/test_pod_launcher_role.py (100%) rename tests/charts/{ => airflow_aux}/test_pod_template_file.py (99%) create mode 100644 tests/charts/airflow_core/__init__.py rename tests/charts/{ => airflow_core}/test_dag_processor.py (100%) rename tests/charts/{ => airflow_core}/test_pdb_scheduler.py (100%) rename tests/charts/{ => airflow_core}/test_scheduler.py (100%) rename tests/charts/{ => airflow_core}/test_triggerer.py (100%) rename tests/charts/{ => airflow_core}/test_worker.py (100%) create mode 100644 tests/charts/other/__init__.py rename tests/charts/{ => other}/test_dags_persistent_volume_claim.py (100%) rename tests/charts/{ => other}/test_flower.py (100%) rename tests/charts/{ => other}/test_git_sync_scheduler.py (100%) rename tests/charts/{ => other}/test_git_sync_triggerer.py (100%) rename tests/charts/{ => other}/test_git_sync_webserver.py (100%) rename tests/charts/{ => other}/test_git_sync_worker.py (100%) rename tests/charts/{ => other}/test_keda.py (100%) rename tests/charts/{ => other}/test_limit_ranges.py (100%) rename tests/charts/{ => other}/test_pdb_pgbouncer.py (100%) rename tests/charts/{ => other}/test_pgbouncer.py (100%) rename tests/charts/{ => other}/test_redis.py (100%) rename tests/charts/{ => other}/test_resource_quota.py (100%) rename tests/charts/{ => other}/test_statsd.py (100%) create mode 100644 tests/charts/security/__init__.py rename tests/charts/{ => security}/test_elasticsearch_secret.py (100%) rename tests/charts/{ => security}/test_extra_configmaps_secrets.py (100%) rename tests/charts/{ => security}/test_kerberos.py (100%) rename tests/charts/{ => security}/test_metadata_connection_secret.py (100%) rename tests/charts/{ => security}/test_rbac.py (100%) rename tests/charts/{ => security}/test_rbac_pod_log_reader.py (100%) rename tests/charts/{ => security}/test_result_backend_connection_secret.py (100%) rename tests/charts/{ => security}/test_scc_rolebinding.py (100%) rename tests/charts/{ => security}/test_security_context.py (100%) create mode 100644 tests/charts/webserver/__init__.py rename tests/charts/{ => webserver}/test_ingress_flower.py (100%) rename tests/charts/{ => webserver}/test_ingress_web.py (100%) rename tests/charts/{ => webserver}/test_pdb_webserver.py (100%) rename tests/charts/{ => webserver}/test_webserver.py (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cff0259e32ae..95fdabc14a661 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,7 @@ jobs: default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} docs-filter: ${{ steps.selective-checks.outputs.docs-filter }} skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} suspended-providers-folders: ${{ steps.selective-checks.outputs.suspended-providers-folders }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} @@ -810,9 +811,13 @@ jobs: tests-helm: timeout-minutes: 80 - name: "Python unit tests for Helm chart" + name: "Unit tests Helm: ${{matrix.helm-test-package}}" runs-on: "${{needs.build-info.outputs.runs-on}}" needs: [build-info, wait-for-ci-images] + strategy: + fail-fast: false + matrix: + helm-test-package: ${{fromJson(needs.build-info.outputs.helm-test-packages)}} env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "Helm" @@ -835,8 +840,8 @@ jobs: - name: > Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image - - name: "Helm Unit Tests" - run: breeze testing helm-tests + - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" + run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}" - name: "Post Helm Tests" uses: ./.github/actions/post_tests diff --git a/Dockerfile.ci b/Dockerfile.ci index dfd931cfdc28a..30201db0c54f0 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -868,8 +868,13 @@ if [[ "${RUN_TESTS}" != "true" ]]; then fi set -u -export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" -export WARNINGS_FILE="/files/warnings-${TEST_TYPE/\[*\]/}-${BACKEND}.txt" +if [[ ${HELM_TEST_PACKAGE=} != "" ]]; then + export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE/\[*\]/}-${HELM_TEST_PACKAGE}-${BACKEND}.xml" + export WARNINGS_FILE="/files/warnings-${TEST_TYPE/\[*\]/}-${HELM_TEST_PACKAGE}-${BACKEND}.txt" +else + export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" + export WARNINGS_FILE="/files/warnings-${TEST_TYPE/\[*\]/}-${BACKEND}.txt" +fi EXTRA_PYTEST_ARGS=( "--verbosity=0" @@ -1022,7 +1027,11 @@ else elif [[ ${TEST_TYPE:=""} == "WWW" ]]; then SELECTED_TESTS=("${WWW_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then - SELECTED_TESTS=("${HELM_CHART_TESTS[@]}") + if [[ ${HELM_TEST_PACKAGE=} != "" ]]; then + SELECTED_TESTS=("tests/charts/${HELM_TEST_PACKAGE}") + else + SELECTED_TESTS=("${HELM_CHART_TESTS[@]}") + fi elif [[ ${TEST_TYPE:=""} == "Integration" ]]; then if [[ ${SKIP_PROVIDER_TESTS:=""} == "true" ]]; then SELECTED_TESTS=("${NO_PROVIDERS_INTEGRATION_TESTS[@]}") diff --git a/TESTING.rst b/TESTING.rst index a3d773799e175..a89b2d287c959 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -606,7 +606,7 @@ Example test here: .. code-block:: python - from tests.charts.helm_template_generator import render_chart, render_k8s_object + from tests.charts.common.helm_template_generator import render_chart, render_k8s_object git_sync_basic = """ dags: @@ -634,6 +634,16 @@ following command (but it takes quite a long time even in a multi-processor mach breeze testing helm-tests +You can also execute tests from a selected package only. Tests in ``tests/chart`` are grouped by packages +so rather than running all tests, you can run only tests from a selected package. For example: + +.. code-block:: bash + + breeze testing helm-tests --helm-test-package basic + +Will run all tests from ``tests/charts/basic`` package. + + You can also run Helm tests individually via the usual ``breeze`` command. Just enter breeze and run the tests with pytest as you would do with regular unit tests (you can add ``-n auto`` command to run Helm tests in parallel - unlike most of the regular unit tests of ours that require a database, the Helm tests are diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index b6ebe571f93cf..752764bd9bb41 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -25,7 +25,11 @@ from click import IntRange from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed -from airflow_breeze.global_constants import ALLOWED_TEST_TYPE_CHOICES, all_selective_test_types +from airflow_breeze.global_constants import ( + ALLOWED_HELM_TEST_PACKAGES, + ALLOWED_TEST_TYPE_CHOICES, + all_selective_test_types, +) from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.ci_group import ci_group @@ -51,7 +55,7 @@ option_verbose, ) from airflow_breeze.utils.console import Output, get_console -from airflow_breeze.utils.custom_param_types import NotVerifiedBetterChoice +from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice from airflow_breeze.utils.docker_command_utils import ( DOCKER_COMPOSE_COMMAND, get_env_variables_for_docker_commands, @@ -545,11 +549,18 @@ def integration_tests( @option_github_repository @option_verbose @option_dry_run +@click.option( + "--helm-test-package", + help="Package to tests", + default="all", + type=BetterChoice(ALLOWED_HELM_TEST_PACKAGES), +) @click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED) def helm_tests( extra_pytest_args: tuple, image_tag: str | None, mount_sources: str, + helm_test_package: str, github_repository: str, ): exec_shell_params = ShellParams( @@ -560,6 +571,8 @@ def helm_tests( env_variables = get_env_variables_for_docker_commands(exec_shell_params) env_variables["RUN_TESTS"] = "true" env_variables["TEST_TYPE"] = "Helm" + if helm_test_package != "all": + env_variables["HELM_TEST_PACKAGE"] = helm_test_package perform_environment_checks() cleanup_python_generated_files() cmd = [*DOCKER_COMPOSE_COMMAND, "run", "--service-ports", "--rm", "airflow"] diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index 80b5d60911332..5696510939a0f 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -88,6 +88,7 @@ "options": [ "--image-tag", "--mount-sources", + "--helm-test-package", "--github-repository", ], }, diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 8595f2dba3d95..22088120c843e 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -115,6 +115,23 @@ class SelectiveUnitTestTypes(Enum): "Quarantine", ] + +@lru_cache(maxsize=None) +def all_helm_test_packages() -> list[str]: + return sorted( + [ + candidate.name + for candidate in (AIRFLOW_SOURCES_ROOT / "tests" / "charts").iterdir() + if candidate.is_dir() + ] + ) + + +ALLOWED_HELM_TEST_PACKAGES = [ + "all", + *all_helm_test_packages(), +] + ALLOWED_PACKAGE_FORMATS = ["wheel", "sdist", "both"] ALLOWED_INSTALLATION_PACKAGE_FORMATS = ["wheel", "sdist"] ALLOWED_INSTALLATION_METHODS = [".", "apache-airflow"] diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 6ce85c0f0aac6..36a85d56dc8bb 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -58,6 +58,7 @@ KIND_VERSION, GithubEvents, SelectiveUnitTestTypes, + all_helm_test_packages, all_selective_test_types, ) from airflow_breeze.utils.console import get_console @@ -644,3 +645,7 @@ def debug_resources(self) -> bool: @cached_property def suspended_providers_folders(self) -> str: return " ".join(get_suspended_providers_folders()) + + @cached_property + def helm_test_packages(self) -> str: + return json.dumps(all_helm_test_packages()) diff --git a/docs/exts/docs_build/helm_chart_utils.py b/docs/exts/docs_build/helm_chart_utils.py index e7db194790e9c..40c33cb2b484a 100644 --- a/docs/exts/docs_build/helm_chart_utils.py +++ b/docs/exts/docs_build/helm_chart_utils.py @@ -17,10 +17,11 @@ from __future__ import annotations import os +from pathlib import Path import yaml -CHART_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "chart")) +CHART_DIR = Path(__file__).resolve().parents[2] / "chart" CHART_YAML_PATH = os.path.join(CHART_DIR, "Chart.yaml") diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 56e28009fb275..7a36b71736c6c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -58,7 +58,7 @@ start-airflow:5e8460ac38f8e9ea2a0ac7e248fd7bc9 static-checks:543f0c776d0f198e80a0f75058445bb2 stop:e5aa686b4e53707ced4039d8414d5cd6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c -testing:helm-tests:94a442e7f3f63b34c4831a84d165690a +testing:helm-tests:936cf28fd84ce4ff5113795fdae9624b testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98 testing:tests:86441445a2b521e8d5aee04d74978451 -testing:68efcf0731170e4ba2029121a5209e3a +testing:2d95034763ee699f2e2fc1804f2fd7f0 diff --git a/images/breeze/output_testing_helm-tests.svg b/images/breeze/output_testing_helm-tests.svg index 9983efe5eb0cb..34a6c6de0158b 100644 --- a/images/breeze/output_testing_helm-tests.svg +++ b/images/breeze/output_testing_helm-tests.svg @@ -1,4 +1,4 @@ - +