From 44871b967c4ef16d1c780f1e569c0a43c53032bc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 30 May 2022 15:38:01 +0200 Subject: [PATCH] Only run separate per-platform build when preparing build cache (#24023) Apparently pushing multi-platform images when building cache on CI has some problems recently, connected with ghcr.io being more vulnerable to race condition described in this issue: https://github.com/containerd/containerd/issues/5978 Apparently when two, different platform layers are pushed about the same time to ghcr.io, the error "cannot reuse body, request must be retried" is generated. However we actually do not even need to build the multiplatform latest images because as of recently we have separate cache for each platform, and the ghcr.io/:latest images are not used any more not even for docker builds. We we always build images rather than pull and we use --from-cache for that - specific per platform. The only image pulling we do is when we pull the :COMMIT_HASH images in CI- but those are single-platform images (amd64) and even if we add tests for arm, they will have different tag. Hopefully we can still build release images without causing the race condition too frequently - this is more likely because when we build images for cache we use machines with different performance characteristics and the same layers are pushed at different times from different platforms. (cherry picked from commit 5d05fcd11367753ec859e860f5fdf0cc7e86db1f) --- .github/workflows/ci.yml | 8 +- .../commands/ci_image_commands.py | 89 +++++---- .../commands/production_image_commands.py | 64 +++---- .../airflow_breeze/params/build_ci_params.py | 9 +- .../params/build_prod_params.py | 4 +- ...build_params.py => common_build_params.py} | 9 +- .../airflow_breeze/utils/common_options.py | 3 +- .../utils/docker_command_utils.py | 71 +++++--- dev/breeze/src/airflow_breeze/utils/image.py | 8 +- .../src/airflow_breeze/utils/registry.py | 4 +- .../src/airflow_breeze/utils/run_utils.py | 59 ------ images/breeze/output-build-image.svg | 136 +++++++------- images/breeze/output-build-prod-image.svg | 170 +++++++++--------- images/breeze/output-commands-hash.txt | 2 +- 14 files changed, 296 insertions(+), 340 deletions(-) rename dev/breeze/src/airflow_breeze/params/{_common_build_params.py => common_build_params.py} (96%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8de7a7f9b184..4b82120db2406 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1664,6 +1664,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" fail-fast: false matrix: python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} + platform: ["linux/amd64", "linux/arm64"] env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} @@ -1685,11 +1686,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze free-space - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh + if: matrix.platform == 'linux/arm64' - name: "Build & Push CI image ${{ matrix.python-version }}:latest" run: > breeze build-image --prepare-buildx-cache - --platform linux/amd64,linux/arm64 + --platform ${{ matrix.platform }} env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - name: > @@ -1722,12 +1724,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --install-packages-from-context --prepare-buildx-cache --disable-airflow-repo-cache - --platform linux/amd64,linux/arm64 + --platform ${{ matrix.platform }} env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() + if: always() && matrix.platform == 'linux/arm64' - name: "Fix ownership" run: breeze fix-ownership if: always() diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 020cd30698a6a..c27e5cc25d7ff 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -72,7 +72,7 @@ build_cache, perform_environment_checks, prepare_docker_build_command, - prepare_empty_docker_build_command, + prepare_docker_build_from_input, ) from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed @@ -143,13 +143,13 @@ { "name": "Preparing cache and push (for maintainers and CI)", "options": [ - "--platform", - "--prepare-buildx-cache", - "--push-image", - "--empty-image", "--github-token", "--github-username", + "--platform", "--login-to-github-registry", + "--push-image", + "--empty-image", + "--prepare-buildx-cache", ], }, ], @@ -460,54 +460,45 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) return 0, f"Image build: {ci_image_params.python}" if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) - cmd = prepare_docker_build_command( - image_params=ci_image_params, - verbose=verbose, - ) - if ci_image_params.empty_image: - env = os.environ.copy() - env['DOCKER_BUILDKIT'] = "1" - get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n") - cmd = prepare_empty_docker_build_command(image_params=ci_image_params) - build_command_result = run_command( - cmd, - input="FROM scratch\n", - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - text=True, - env=env, - ) + if ci_image_params.prepare_buildx_cache: + build_command_result = build_cache(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) else: - get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n") - build_command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False - ) - if build_command_result.returncode == 0: - if ci_image_params.prepare_buildx_cache: - build_command_result = build_cache( - image_params=ci_image_params, dry_run=dry_run, verbose=verbose - ) - - if not ci_image_params.prepare_buildx_cache: - if not dry_run: + if ci_image_params.empty_image: + env = os.environ.copy() + env['DOCKER_BUILDKIT'] = "1" + get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n") + build_command_result = run_command( + prepare_docker_build_from_input(image_params=ci_image_params), + input="FROM scratch\n", + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + env=env, + ) + else: + get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n") + build_command_result = run_command( + prepare_docker_build_command( + image_params=ci_image_params, + verbose=verbose, + ), + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + check=False, + ) if build_command_result.returncode == 0: if ci_image_params.tag_as_latest: build_command_result = tag_image_as_latest(ci_image_params, dry_run, verbose) - if ( - ci_image_params.airflow_image_name == ci_image_params.airflow_image_name_with_tag - or ci_image_params.tag_as_latest - and build_command_result.returncode == 0 - ): - mark_image_as_refreshed(ci_image_params) - else: - get_console().print("[error]Error when building image![/]") - return ( - build_command_result.returncode, - f"Image build: {ci_image_params.python}", - ) - else: - get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]") + if ci_image_params.preparing_latest_image(): + if dry_run: + get_console().print( + "[info]Not updating build hash because we are in `dry_run` mode.[/]" + ) + else: + mark_image_as_refreshed(ci_image_params) return build_command_result.returncode, f"Image build: {ci_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 32aed85d9ed14..e10d46cda4081 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -70,7 +70,7 @@ build_cache, perform_environment_checks, prepare_docker_build_command, - prepare_empty_docker_build_command, + prepare_docker_build_from_input, ) from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR @@ -148,11 +148,11 @@ "options": [ "--github-token", "--github-username", + "--platform", "--login-to-github-registry", "--push-image", - "--prepare-buildx-cache", - "--platform", "--empty-image", + "--prepare-buildx-cache", ], }, ], @@ -497,36 +497,36 @@ def build_production_image( if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") - if prod_image_params.empty_image: - env = os.environ.copy() - env['DOCKER_BUILDKIT'] = "1" - get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n") - cmd = prepare_empty_docker_build_command(image_params=prod_image_params) - build_command_result = run_command( - cmd, - input="FROM scratch\n", - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - check=False, - text=True, - env=env, - ) + if prod_image_params.prepare_buildx_cache: + build_command_result = build_cache(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) else: - cmd = prepare_docker_build_command( - image_params=prod_image_params, - verbose=verbose, - ) - build_command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True - ) - if build_command_result.returncode == 0: - if prod_image_params.prepare_buildx_cache: - build_command_result = build_cache( - image_params=prod_image_params, dry_run=dry_run, verbose=verbose - ) - else: + if prod_image_params.empty_image: + env = os.environ.copy() + env['DOCKER_BUILDKIT'] = "1" + get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n") + build_command_result = run_command( + prepare_docker_build_from_input(image_params=prod_image_params), + input="FROM scratch\n", + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + check=False, + text=True, + env=env, + ) + else: + build_command_result = run_command( + prepare_docker_build_command( + image_params=prod_image_params, + verbose=verbose, + ), + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + check=False, + text=True, + ) + if build_command_result.returncode == 0: if prod_image_params.tag_as_latest: build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose) - return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index b012993519a92..aa9681fc955fe 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -20,13 +20,12 @@ from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import get_airflow_version -from airflow_breeze.params._common_build_params import _CommonBuildParams -from airflow_breeze.utils.console import get_console +from airflow_breeze.params.common_build_params import CommonBuildParams from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR @dataclass -class BuildCiParams(_CommonBuildParams): +class BuildCiParams(CommonBuildParams): """ CI build parameters. Those parameters are used to determine command issued to build CI image. """ @@ -96,6 +95,4 @@ def optional_image_args(self) -> List[str]: ] def __post_init__(self): - if self.prepare_buildx_cache: - get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]") - self.push_image = True + pass diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index c480570f52ef4..e81a5d6739ac8 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -30,12 +30,12 @@ get_airflow_extras, get_airflow_version, ) -from airflow_breeze.params._common_build_params import _CommonBuildParams +from airflow_breeze.params.common_build_params import CommonBuildParams from airflow_breeze.utils.console import get_console @dataclass -class BuildProdParams(_CommonBuildParams): +class BuildProdParams(CommonBuildParams): """ PROD build parameters. Those parameters are used to determine command issued to build PROD image. """ diff --git a/dev/breeze/src/airflow_breeze/params/_common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py similarity index 96% rename from dev/breeze/src/airflow_breeze/params/_common_build_params.py rename to dev/breeze/src/airflow_breeze/params/common_build_params.py index 4f2139d20e723..fd8a2b82e8c8c 100644 --- a/dev/breeze/src/airflow_breeze/params/_common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -27,7 +27,7 @@ @dataclass -class _CommonBuildParams: +class CommonBuildParams: """ Common build parameters. Those parameters are common parameters for CI And PROD build. """ @@ -147,6 +147,9 @@ def get_cache(self, single_platform: str) -> str: def is_multi_platform(self) -> bool: return "," in self.platform + def preparing_latest_image(self) -> bool: + return self.tag_as_latest or self.airflow_image_name == self.airflow_image_name_with_tag + @property def platforms(self) -> List[str]: return self.platform.split(",") @@ -160,6 +163,4 @@ def optional_image_args(self) -> List[str]: raise NotImplementedError() def __post_init__(self): - if self.prepare_buildx_cache: - get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]") - self.push_image = True + pass diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 545a6ec149912..deca1b4882a4e 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -284,8 +284,7 @@ ) option_prepare_buildx_cache = click.option( '--prepare-buildx-cache', - help='Prepares build cache additionally to building images (this is done as two separate steps after' - 'the images are build). Implies --push-image flag', + help='Prepares build cache (this is done as separate per-platform steps instead of building the image).', is_flag=True, envvar='PREPARE_BUILDX_CACHE', ) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index fc3f1b17fca73..e838e380f781d 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -24,9 +24,9 @@ from subprocess import DEVNULL, STDOUT, CalledProcessError, CompletedProcess from typing import Dict, List, Union -from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.params.build_ci_params import BuildCiParams from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.params.common_build_params import CommonBuildParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.host_info_utils import get_host_group_id, get_host_os, get_host_user_id from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT @@ -58,9 +58,8 @@ from airflow_breeze.utils.console import get_console from airflow_breeze.utils.run_utils import ( RunCommandResult, + check_if_buildx_plugin_installed, commit_sha, - prepare_base_build_command, - prepare_build_cache_command, run_command, ) @@ -313,7 +312,7 @@ def check_docker_compose_version(verbose: bool): ) -def get_env_variable_value(arg_name: str, params: Union[BuildCiParams, BuildProdParams, ShellParams]): +def get_env_variable_value(arg_name: str, params: Union[CommonBuildParams, ShellParams]): raw_value = getattr(params, arg_name, None) value = str(raw_value) if raw_value is not None else '' value = "true" if raw_value is True else value @@ -323,7 +322,7 @@ def get_env_variable_value(arg_name: str, params: Union[BuildCiParams, BuildProd return value -def prepare_arguments_for_docker_build_command(image_params: _CommonBuildParams) -> List[str]: +def prepare_arguments_for_docker_build_command(image_params: CommonBuildParams) -> List[str]: """ Constructs docker compose command arguments list based on parameters passed. Maps arguments to argument values. @@ -353,9 +352,7 @@ def prepare_arguments_for_docker_build_command(image_params: _CommonBuildParams) def prepare_docker_build_cache_command( - image_params: _CommonBuildParams, - dry_run: bool, - verbose: bool, + image_params: CommonBuildParams, ) -> List[str]: """ Constructs docker build_cache command based on the parameters passed. @@ -365,11 +362,10 @@ def prepare_docker_build_cache_command( :return: Command to run as list of string """ arguments = prepare_arguments_for_docker_build_command(image_params) - build_command = prepare_build_cache_command() build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) - final_command.extend(build_command) + final_command.extend(["buildx", "build", "--builder", "airflow_cache", "--progress=tty"]) final_command.extend(build_flags) final_command.extend(["--pull"]) final_command.extend(arguments) @@ -381,16 +377,42 @@ def prepare_docker_build_cache_command( final_command.extend( [f"--cache-to=type=registry,ref={image_params.get_cache(image_params.platform)},mode=max"] ) - cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] - buildx_command_result = run_command(cmd, verbose=verbose, dry_run=dry_run, text=True) - if buildx_command_result and buildx_command_result.returncode != 0: - next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] - run_command(next_cmd, verbose=verbose, text=True, check=False) return final_command +def prepare_base_build_command(image_params: CommonBuildParams, verbose: bool) -> List[str]: + """ + Prepare build command for docker build. Depending on whether we have buildx plugin installed or not, + and whether we run cache preparation, there might be different results: + + * if buildx plugin is installed - `docker buildx` command is returned - using regular or cache builder + depending on whether we build regular image or cache + * if no buildx plugin is installed, and we do not prepare cache, regular docker `build` command is used. + * if no buildx plugin is installed, and we prepare cache - we fail. Cache can only be done with buildx + :param image_params: parameters of the image + :param verbose: print commands when running + :return: command to use as docker build command + """ + build_command_param = [] + is_buildx_available = check_if_buildx_plugin_installed(verbose=verbose) + if is_buildx_available: + build_command_param.extend( + [ + "buildx", + "build", + "--builder", + "default", + "--progress=tty", + "--push" if image_params.push_image else "--load", + ] + ) + else: + build_command_param.append("build") + return build_command_param + + def prepare_docker_build_command( - image_params: _CommonBuildParams, + image_params: CommonBuildParams, verbose: bool, ) -> List[str]: """ @@ -417,7 +439,7 @@ def prepare_docker_build_command( def construct_docker_push_command( - image_params: _CommonBuildParams, + image_params: CommonBuildParams, ) -> List[str]: """ Constructs docker push command based on the parameters passed. @@ -427,8 +449,8 @@ def construct_docker_push_command( return ["docker", "push", image_params.airflow_image_name_with_tag] -def prepare_empty_docker_build_command( - image_params: _CommonBuildParams, +def prepare_docker_build_from_input( + image_params: CommonBuildParams, ) -> List[str]: """ Constructs docker build empty image command based on the parameters passed. @@ -438,18 +460,21 @@ def prepare_empty_docker_build_command( return ["docker", "build", "-t", image_params.airflow_image_name_with_tag, "-"] -def build_cache(image_params: _CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult: +def build_cache(image_params: CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult: build_command_result: Union[CompletedProcess, CalledProcessError] = CompletedProcess( args=[], returncode=0 ) + cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] + buildx_command_result = run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, check=False) + if buildx_command_result and buildx_command_result.returncode != 0: + next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] + run_command(next_cmd, verbose=verbose, text=True, check=False) for platform in image_params.platforms: platform_image_params = deepcopy(image_params) # override the platform in the copied params to only be single platform per run # as a workaround to https://github.com/docker/buildx/issues/1044 platform_image_params.platform = platform - cmd = prepare_docker_build_cache_command( - image_params=platform_image_params, dry_run=dry_run, verbose=verbose - ) + cmd = prepare_docker_build_cache_command(image_params=platform_image_params) build_command_result = run_command( cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True ) diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py index 1962951f949ea..80da7f75c5273 100644 --- a/dev/breeze/src/airflow_breeze/utils/image.py +++ b/dev/breeze/src/airflow_breeze/utils/image.py @@ -25,9 +25,9 @@ DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL, ) -from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.params.build_ci_params import BuildCiParams from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.params.common_build_params import CommonBuildParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.console import get_console from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed @@ -82,7 +82,7 @@ def run_pull_in_parallel( def run_pull_image( - image_params: _CommonBuildParams, + image_params: CommonBuildParams, dry_run: bool, verbose: bool, wait_for_image: bool, @@ -156,7 +156,7 @@ def run_pull_image( return command_result.returncode, f"Image Python {image_params.python}" -def tag_image_as_latest(image_params: _CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult: +def tag_image_as_latest(image_params: CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult: if image_params.airflow_image_name_with_tag == image_params.airflow_image_name: get_console().print( f"[info]Skip tagging {image_params.airflow_image_name} as latest as it is already 'latest'[/]" @@ -177,7 +177,7 @@ def tag_image_as_latest(image_params: _CommonBuildParams, dry_run: bool, verbose def run_pull_and_verify_image( - image_params: _CommonBuildParams, + image_params: CommonBuildParams, dry_run: bool, verbose: bool, wait_for_image: bool, diff --git a/dev/breeze/src/airflow_breeze/utils/registry.py b/dev/breeze/src/airflow_breeze/utils/registry.py index bf01ec61b3aba..820ae9d2739da 100644 --- a/dev/breeze/src/airflow_breeze/utils/registry.py +++ b/dev/breeze/src/airflow_breeze/utils/registry.py @@ -18,13 +18,13 @@ import os from typing import Tuple -from airflow_breeze.params._common_build_params import _CommonBuildParams +from airflow_breeze.params.common_build_params import CommonBuildParams from airflow_breeze.utils.console import get_console from airflow_breeze.utils.run_utils import run_command def login_to_github_docker_registry( - image_params: _CommonBuildParams, dry_run: bool, verbose: bool + image_params: CommonBuildParams, dry_run: bool, verbose: bool ) -> Tuple[int, str]: """ In case of CI environment, we need to login to GitHub Registry. diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 03f3c0532d205..f228cc657cf60 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -28,7 +28,6 @@ from typing import Dict, Generator, List, Mapping, Optional, Union from airflow_breeze.branch_defaults import AIRFLOW_BRANCH -from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT @@ -302,64 +301,6 @@ def check_if_buildx_plugin_installed(verbose: bool) -> bool: return False -def prepare_base_build_command(image_params: _CommonBuildParams, verbose: bool) -> List[str]: - """ - Prepare build command for docker build. Depending on whether we have buildx plugin installed or not, - and whether we run cache preparation, there might be different results: - - * if buildx plugin is installed - `docker buildx` command is returned - using regular or cache builder - depending on whether we build regular image or cache - * if no buildx plugin is installed, and we do not prepare cache, regular docker `build` command is used. - * if no buildx plugin is installed, and we prepare cache - we fail. Cache can only be done with buildx - :param image_params: parameters of the image - :param verbose: print commands when running - :return: command to use as docker build command - """ - build_command_param = [] - is_buildx_available = check_if_buildx_plugin_installed(verbose=verbose) - if is_buildx_available: - if image_params.prepare_buildx_cache: - build_command_param.extend( - ["buildx", "build", "--builder", "airflow_cache", "--progress=tty", "--push"] - ) - else: - build_command_param.extend( - [ - "buildx", - "build", - "--builder", - "default", - "--progress=tty", - "--push" if image_params.push_image else "--load", - ] - ) - else: - if image_params.prepare_buildx_cache or image_params.push_image: - get_console().print( - '\n[error] Buildx cli plugin is not available and you need it to prepare' - ' buildx cache or push image after build. \n' - ) - get_console().print( - '[error] Please install it following https://docs.docker.com/buildx/working-with-buildx/ \n' - ) - sys.exit(1) - build_command_param.append("build") - return build_command_param - - -def prepare_build_cache_command() -> List[str]: - """ - Prepare build cache command for docker build. We need to have buildx for that command. - This command is needed separately from the build image command because of the bug in multiplatform - support for buildx plugin https://github.com/docker/buildx/issues/1044 where when you run multiple - platform build, cache from one platform overrides cache for the other platform. - - :param verbose: print commands when running - :return: command to use as docker build command - """ - return ["buildx", "build", "--builder", "airflow_cache", "--progress=tty"] - - @lru_cache(maxsize=None) def commit_sha(): """Returns commit SHA of current repo. Cached for various usages.""" diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 1713b02cef206..b779367062c37 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -19,93 +19,93 @@ font-weight: 700; } - .terminal-1074915103-matrix { + .terminal-2884033632-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 26.400000000000002px; font-variant-east-asian: full-width; } - .terminal-1074915103-title { + .terminal-2884033632-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1074915103-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1074915103-r2 { fill: #c5c8c6 } -.terminal-1074915103-r3 { fill: #d0b344;font-weight: bold } -.terminal-1074915103-r4 { fill: #868887 } -.terminal-1074915103-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1074915103-r6 { fill: #98a84b;font-weight: bold } -.terminal-1074915103-r7 { fill: #8d7b39 } + .terminal-2884033632-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2884033632-r2 { fill: #c5c8c6 } +.terminal-2884033632-r3 { fill: #d0b344;font-weight: bold } +.terminal-2884033632-r4 { fill: #868887 } +.terminal-2884033632-r5 { fill: #68a0b3;font-weight: bold } +.terminal-2884033632-r6 { fill: #98a84b;font-weight: bold } +.terminal-2884033632-r7 { fill: #8d7b39 } - Command: build-image + Command: build-image -                                                                                                                          - Usage: breeze build-image [OPTIONS]                                                                                     -                                                                                                                         - Build CI image. Include building multiple images for all python versions (sequentially).                                -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ -│                                       Useful when you build or pull image with --image-tag.                          │ -│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ -│                                       [default: registry]                                                            │ -│  --force-build                        Force image build no matter if it is determined as needed.                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ -│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ -│                                      [default: constraints-source-providers]                                         │ -│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ -│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ -│                            afterthe images are build). Implies --push-image flag                                     │ -│  --push-image              Push image after building it.                                                             │ -│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ -│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ -│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-image [OPTIONS]                                                                                     +                                                                                                                         + Build CI image. Include building multiple images for all python versions (sequentially).                                +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ +│                                       Useful when you build or pull image with --image-tag.                          │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +│  --force-build                        Force image build no matter if it is determined as needed.                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ +│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ +│                                      [default: constraints-source-providers]                                         │ +│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --push-image              Push image after building it.                                                             │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +│  --prepare-buildx-cache    Prepares build cache (this is done as separate per-platform steps instead of building     │ +│                            the image).                                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index d90abce791153..247461d55530c 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -19,110 +19,110 @@ font-weight: 700; } - .terminal-3989203328-matrix { + .terminal-997154497-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 26.400000000000002px; font-variant-east-asian: full-width; } - .terminal-3989203328-title { + .terminal-997154497-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3989203328-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-3989203328-r2 { fill: #c5c8c6 } -.terminal-3989203328-r3 { fill: #d0b344;font-weight: bold } -.terminal-3989203328-r4 { fill: #868887 } -.terminal-3989203328-r5 { fill: #68a0b3;font-weight: bold } -.terminal-3989203328-r6 { fill: #98a84b;font-weight: bold } -.terminal-3989203328-r7 { fill: #8d7b39 } + .terminal-997154497-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-997154497-r2 { fill: #c5c8c6 } +.terminal-997154497-r3 { fill: #d0b344;font-weight: bold } +.terminal-997154497-r4 { fill: #868887 } +.terminal-997154497-r5 { fill: #68a0b3;font-weight: bold } +.terminal-997154497-r6 { fill: #98a84b;font-weight: bold } +.terminal-997154497-r7 { fill: #8d7b39 } - Command: build-prod-image + Command: build-prod-image -                                                                                                                          - Usage: breeze build-prod-image [OPTIONS]                                                                                -                                                                                                                         - Build Production image. Include building multiple images for all or selected Python versions sequentially.              -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --install-airflow-version        -V  Install version of Airflow from PyPI. (TEXT)                                   │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ -│                                       Useful when you build or pull image with --image-tag.                          │ -│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ -│                                       [default: registry]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --airflow-extras                    Extras to install by default.                                                   │ -│                                      (TEXT)                                                                          │ -│                                      [default:                                                                       │ -│                                      amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,goog…  │ -│  --airflow-constraints-mode          Mode of constraints for PROD image building                                     │ -│                                      (constraints | constraints-no-providers | constraints-source-providers)         │ -│                                      [default: constraints]                                                          │ -│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│  --install-packages-from-context           Install wheels from local docker-context-files when building image.       │ -│  --airflow-is-in-context                   If set Airflow is installed from docker-context-files only rather than    │ -│                                            from PyPI or sources.                                                     │ -│  --cleanup-context                         Clean up docker context files before running build (cannot be used        │ -│                                            together with --install-packages-from-context).                           │ -│  --disable-mysql-client-installation       Do not install MySQL client.                                              │ -│  --disable-mssql-client-installation       Do not install MsSQl client.                                              │ -│  --disable-postgres-client-installation    Do not install Postgres client.                                           │ -│  --disable-airflow-repo-cache              Disable cache from Airflow repository during building.                    │ -│  --install-airflow-reference               Install Airflow using GitHub tag or branch. (TEXT)                        │ -│  --installation-method                     Install Airflow from: sources or PyPI. (. | apache-airflow)               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ -│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ -│  --push-image              Push image after building it.                                                             │ -│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ -│                            afterthe images are build). Implies --push-image flag                                     │ -│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ -│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-prod-image [OPTIONS]                                                                                +                                                                                                                         + Build Production image. Include building multiple images for all or selected Python versions sequentially.              +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --install-airflow-version        -V  Install version of Airflow from PyPI. (TEXT)                                   │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ +│                                       Useful when you build or pull image with --image-tag.                          │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-extras                    Extras to install by default.                                                   │ +│                                      (TEXT)                                                                          │ +│                                      [default:                                                                       │ +│                                      amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,goog…  │ +│  --airflow-constraints-mode          Mode of constraints for PROD image building                                     │ +│                                      (constraints | constraints-no-providers | constraints-source-providers)         │ +│                                      [default: constraints]                                                          │ +│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│  --install-packages-from-context           Install wheels from local docker-context-files when building image.       │ +│  --airflow-is-in-context                   If set Airflow is installed from docker-context-files only rather than    │ +│                                            from PyPI or sources.                                                     │ +│  --cleanup-context                         Clean up docker context files before running build (cannot be used        │ +│                                            together with --install-packages-from-context).                           │ +│  --disable-mysql-client-installation       Do not install MySQL client.                                              │ +│  --disable-mssql-client-installation       Do not install MsSQl client.                                              │ +│  --disable-postgres-client-installation    Do not install Postgres client.                                           │ +│  --disable-airflow-repo-cache              Disable cache from Airflow repository during building.                    │ +│  --install-airflow-reference               Install Airflow using GitHub tag or branch. (TEXT)                        │ +│  --installation-method                     Install Airflow from: sources or PyPI. (. | apache-airflow)               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --push-image              Push image after building it.                                                             │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +│  --prepare-buildx-cache    Prepares build cache (this is done as separate per-platform steps instead of building     │ +│                            the image).                                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index fe6d5cdba65af..cf38decd7f769 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -4538f69421a320083444736d334314b7 +ce7896bb79ac7862ff8a4d6c7e326671