diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py index da5364d087e18..af37c71dbe939 100644 --- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py @@ -33,7 +33,7 @@ class BuildCiParams: CI build parameters. Those parameters are used to determine command issued to build CI image. """ - upgrade_to_newer_dependencies: str = "false" + upgrade_to_newer_dependencies: bool = False python: str = "3.7" airflow_branch: str = AIRFLOW_BRANCH build_id: int = 0 diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py index 116fae9333c11..91a06e6e3f6ef 100644 --- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py @@ -52,7 +52,7 @@ class BuildProdParams: empty_image: bool = False airflow_is_in_context: bool = False install_packages_from_context: bool = False - upgrade_to_newer_dependencies: str = "false" + upgrade_to_newer_dependencies: bool = False airflow_version: str = get_airflow_version() python: str = "3.7" airflow_branch_for_pypi_preloading: str = AIRFLOW_BRANCH diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py index e169697cd21cf..a2d2e8c0f98a2 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_options.py @@ -197,9 +197,8 @@ option_upgrade_to_newer_dependencies = click.option( "-u", '--upgrade-to-newer-dependencies', - default="false", - show_default=True, - help='When other than "false", upgrade all PIP packages to latest.', + is_flag=True, + help='When set, upgrade all PIP packages to latest.', envvar='UPGRADE_TO_NEWER_DEPENDENCIES', ) option_additional_extras = click.option( diff --git a/dev/breeze/src/airflow_breeze/commands/release_management.py b/dev/breeze/src/airflow_breeze/commands/release_management.py index 72e2756878800..33765f9afd77b 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management.py @@ -320,13 +320,12 @@ def generate_constraints( set_forced_answer(answer) if run_in_parallel: given_answer = user_confirm( - f"Did you build all CI images {python_versions} with --upgrade-to-newer-dependencies " - f"true flag set?", + f"Did you build all CI images {python_versions} with --upgrade-to-newer-dependencies flag set?", timeout=None, ) else: given_answer = user_confirm( - f"Did you build CI image {python} with --upgrade-to-newer-dependencies true flag set?", + f"Did you build CI image {python} with --upgrade-to-newer-dependencies flag set?", timeout=None, ) if given_answer != Answer.YES: @@ -334,7 +333,7 @@ def generate_constraints( get_console().print("\n[info]Use this command to build the images:[/]\n") get_console().print( f" breeze build-image --run-in-parallel --python-versions '{python_versions}' " - f"--upgrade-to-newer-dependencies true\n" + f"--upgrade-to-newer-dependencies\n" ) else: shell_params = ShellParams( @@ -343,7 +342,7 @@ def generate_constraints( get_console().print("\n[info]Use this command to build the image:[/]\n") get_console().print( f" breeze build-image --python'{shell_params.python}' " - f"--upgrade-to-newer-dependencies true\n" + f"--upgrade-to-newer-dependencies\n" ) sys.exit(1) if run_in_parallel: diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py index 51294d55bb9da..c9d93abb3d9a8 100644 --- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py +++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py @@ -120,7 +120,7 @@ def run_shell_with_build_image_checks( build_ci_image_check_cache = Path( BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python}" ) - ci_image_params = BuildCiParams(python=shell_params.python, upgrade_to_newer_dependencies="false") + ci_image_params = BuildCiParams(python=shell_params.python, upgrade_to_newer_dependencies=False) if build_ci_image_check_cache.exists(): get_console().print(f'[info]{shell_params.the_image_type} image already built locally.[/]') else: diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 4598a6ce97d56..813ad67ea2d0d 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -18,6 +18,7 @@ import os import re import subprocess +from random import randint from typing import Dict, List, Tuple, Union from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams @@ -286,7 +287,14 @@ def construct_arguments_for_docker_build_command( image_params: Union[BuildCiParams, BuildProdParams], required_args: List[str], optional_args: List[str] ) -> List[str]: """ - Constructs docker compose command arguments list based on parameters passed + Constructs docker compose command arguments list based on parameters passed. Maps arguments to + argument values. + + It maps: + * all the truthy/falsy values are converted to "true" / "false" respectively + * if upgrade_to_newer_dependencies is set to True, it is replaced by a random string to account + for the need of always triggering upgrade for docker build. + :param image_params: parameters of the image :param required_args: build argument that are required :param optional_args: build arguments that are optional (should not be used if missing or empty) @@ -295,8 +303,10 @@ def construct_arguments_for_docker_build_command( def get_env_variable_value(arg_name: str): value = str(getattr(image_params, arg_name)) - value = "true" if value == "True" else value - value = "false" if value == "False" else value + value = "true" if value.lower() in ["true", "t", "yes", "y"] else value + value = "false" if value.lower() in ["false", "f", "no", "n"] else value + if arg_name == "upgrade_to_newer_dependencies" and value == "true": + value = f"{randint(0, 2**32):x}" return value args_command = [] diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 01b392f174799..f55442611cdd6 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ -
│ --python -p Python major/minor version used in Airflow image for images. │
│ (>3.7< | 3.8 | 3.9 | 3.10) │
│ [default: 3.7] │
-
│ --upgrade-to-newer-dependencies -u When other than "false", upgrade all PIP packages to latest. (TEXT) │
-
│ [default: false] │
+
│ --upgrade-to-newer-dependencies -u When set, upgrade all PIP packages to latest. │
│ --debian-version -d Debian version used for the image. (bullseye | buster) [default: bullseye] │
│ --image-tag -t Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT) │
│ --docker-cache -c Cache option for image used during the build. (pulled | local | disabled) │
diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index def3afbc88821..7d693c4d73654 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ -
│ (>3.7< | 3.8 | 3.9 | 3.10) │
│ [default: 3.7] │
│ --install-airflow-version -V Install version of Airflow from PyPI. (TEXT) │
-
│ --upgrade-to-newer-dependencies -u When other than "false", upgrade all PIP packages to latest. (TEXT) │
-
│ [default: false] │
+
│ --upgrade-to-newer-dependencies -u When set, upgrade all PIP packages to latest. │
│ --debian-version -d Debian version used for the image. (bullseye | buster) [default: bullseye] │
│ --image-tag -t Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT) │
│ --docker-cache -c Cache option for image used during the build. (pulled | local | disabled) │
diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index 85f7a1c07e4a9..c9f215c53c251 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -44,13 +44,8 @@ fi function check_upgrade_to_newer_dependencies_needed() { if [[ ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} == "scheduled" ]]; then - # Trigger upgrading to latest constraints when we are in push or schedule event. We use the - # random string so that it always triggers rebuilding layer in the docker image - # Each build that upgrades to latest constraints will get truly latest constraints, not those - # cached in the image because docker layer will get invalidated. - # This upgrade_to_newer_dependencies variable can later be overridden - # in case we find that any of the setup.* files changed (see below) - upgrade_to_newer_dependencies="${RANDOM}" + # Trigger upgrading to latest constraints when we are in push or schedule event + upgrade_to_newer_dependencies="true" fi } @@ -358,9 +353,8 @@ function check_if_setup_files_changed() { if [[ $(count_changed_files) != "0" ]]; then # In case the setup files changed, we automatically force upgrading to newer dependencies - # no matter what was set before. We set it to random number to make sure that it will be - # always invalidating the layer in Docker that triggers installing the dependencies - upgrade_to_newer_dependencies="${RANDOM}" + # no matter what was set before. + upgrade_to_newer_dependencies="true" fi start_end::group_end }