From d04402a52540f8035427e59b03a2a4e3d19af650 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 12 Jun 2022 16:27:32 +0200 Subject: [PATCH] Restore capability of reproducing CI failures using new Breeze The old breeze-legacy used to have a possibility of very easy reproduction of CI failures by executing the right breeze command that contained the commit hash of the PR being tested. This has been broken for some time after we migrated to the new breeze, but finally it was the time when it was needed again. This PR brings back the capability by: * addding --image-tag parameters for tests, shell and start-airflow commands * if --image-tag is specified, then rather than building the image, it is pulled using the specified hash * if --image-tag is specified, the local sources are not mounted to breeze when started, but the sources already embedded in the image are used ("skipped" set for --mount-sources). * new "removed" command value is added to --mount-sources, it causes breeze command to remove the sources from the image (it is used when installing airflow during the tests for specified version (it's automatically used when --use-airflow-version is used). --- .github/workflows/ci.yml | 5 + CI.rst | 9 +- IMAGES.rst | 5 +- breeze-complete | 3 - breeze-legacy | 24 -- .../commands/ci_image_commands.py | 30 +- .../commands/developer_commands.py | 16 +- .../commands/release_management_commands.py | 14 +- .../commands/testing_commands.py | 22 +- .../src/airflow_breeze/global_constants.py | 5 +- .../src/airflow_breeze/params/shell_params.py | 27 +- .../airflow_breeze/utils/common_options.py | 13 +- .../utils/docker_command_utils.py | 3 +- dev/breeze/src/airflow_breeze/utils/image.py | 4 +- dev/breeze/tests/test_commands.py | 12 +- images/breeze/output-build-image.svg | 268 ++++++------- images/breeze/output-build-prod-image.svg | 336 ++++++++-------- images/breeze/output-commands-hash.txt | 1 - images/breeze/output-commands.svg | 368 +++++++++--------- images/breeze/output-docker-compose-tests.svg | 104 ++--- images/breeze/output-generate-constraints.svg | 152 ++++---- images/breeze/output-pull-image.svg | 148 +++---- images/breeze/output-pull-prod-image.svg | 148 +++---- images/breeze/output-shell.svg | 224 ++++++----- images/breeze/output-start-airflow.svg | 232 ++++++----- images/breeze/output-tests.svg | 164 ++++---- images/breeze/output-verify-image.svg | 104 ++--- images/breeze/output-verify-prod-image.svg | 104 ++--- .../output-verify-provider-packages.svg | 132 +++---- .../ci_run_single_airflow_test_in_docker.sh | 6 +- 30 files changed, 1423 insertions(+), 1260 deletions(-) delete mode 100644 images/breeze/output-commands-hash.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0996257cfc53f..c112a6da727a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -923,6 +923,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" uses: actions/upload-artifact@v2 if: failure() @@ -996,6 +997,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" uses: actions/upload-artifact@v2 if: failure() @@ -1068,6 +1070,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" uses: actions/upload-artifact@v2 if: failure() @@ -1138,6 +1141,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" uses: actions/upload-artifact@v2 if: failure() @@ -1206,6 +1210,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" uses: actions/upload-artifact@v2 if: failure() diff --git a/CI.rst b/CI.rst index f24639271e977..7798b077acf0c 100644 --- a/CI.rst +++ b/CI.rst @@ -96,7 +96,7 @@ You can read more about Breeze in `BREEZE.rst `_ but in essence it i you to re-create CI environment in your local development instance and interact with it. In its basic form, when you do development you can run all the same tests that will be run in CI - but locally, before you submit them as PR. Another use case where Breeze is useful is when tests fail on CI. You can -take the full ``COMMIT_SHA`` of the failed build pass it as ``--github-image-id`` parameter of Breeze and it will +take the full ``COMMIT_SHA`` of the failed build pass it as ``--image-tag`` parameter of Breeze and it will download the very same version of image that was used in CI and run it locally. This way, you can very easily reproduce any failed test that happens in CI - even if you do not check out the sources connected with the run. @@ -275,7 +275,7 @@ You can use those variables when you try to reproduce the build locally. | | | | | should set it to false, especially | | | | | | in case our local sources are not the | | | | | | ones we intend to use (for example | -| | | | | when ``--github-image-id`` is used | +| | | | | when ``--image-tag`` is used | | | | | | in Breeze. | | | | | | | | | | | | In CI jobs it is set to true | @@ -668,12 +668,11 @@ For example knowing that the CI job was for commit ``cd27124534b46c9688a1d89e75f But you usually need to pass more variables and complex setup if you want to connect to a database or enable some integrations. Therefore it is easiest to use `Breeze `_ for that. For example if -you need to reproduce a MySQL environment with kerberos integration enabled for commit -cd27124534b46c9688a1d89e75fcd137ab5137e3, in python 3.8 environment you can run: +you need to reproduce a MySQL environment in python 3.8 environment you can run: .. code-block:: bash - ./breeze-legacy --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3 --python 3.8 + breeze --image-tag cd27124534b46c9688a1d89e75fcd137ab5137e3 --python 3.8 --backend mysql You will be dropped into a shell with the exact version that was used during the CI run and you will be able to run pytest tests manually, easily reproducing the environment that was used in CI. Note that in diff --git a/IMAGES.rst b/IMAGES.rst index 58ef0cca54852..634c4ac8a0e12 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -281,7 +281,7 @@ to refresh them. Every developer can also pull and run images being result of a specific CI run in GitHub Actions. This is a powerful tool that allows to reproduce CI failures locally, enter the images and fix them much -faster. It is enough to pass ``--github-image-id`` and the registry and Breeze will download and execute +faster. It is enough to pass ``--image-tag`` and the registry and Breeze will download and execute commands using the same image that was used during the CI tests. For example this command will run the same Python 3.8 image as was used in build identified with @@ -289,8 +289,7 @@ For example this command will run the same Python 3.8 image as was used in build .. code-block:: bash - ./breeze-legacy --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e \ - --python 3.8 --integration rabbitmq + breeze --image-tag 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e --python 3.8 --integration rabbitmq You can see more details and examples in `Breeze `_ diff --git a/breeze-complete b/breeze-complete index 389d18f64d439..2fd1a9d1a8cfa 100644 --- a/breeze-complete +++ b/breeze-complete @@ -173,9 +173,6 @@ function breeze_complete::get_known_values_breeze() { -g | --github-repository) _breeze_known_values="${_breeze_default_github_repository}" ;; - -s | --github-image-id) - _breeze_known_values="${_breeze_default_github_image_id}" - ;; kind-cluster) _breeze_known_values="${_breeze_allowed_kind_operations}" ;; diff --git a/breeze-legacy b/breeze-legacy index d18d3c25e132b..0bb257494c915 100755 --- a/breeze-legacy +++ b/breeze-legacy @@ -970,21 +970,6 @@ function breeze::parse_arguments() { export GITHUB_REPOSITORY="${2}" shift 2 ;; - -s | --github-image-id) - echo - echo "GitHub image id: ${2}" - echo - echo "Force pulling the image, using GitHub registry and skip mounting local sources." - echo "This is in order to get the exact same version as used in CI environment for SHA!." - echo "You can specify --skip-mounting-local-sources to not mount local sources to get exact. " - echo "behaviour as in the CI environment." - echo - export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}" - export CHECK_IMAGE_FOR_REBUILD="false" - export SKIP_BUILDING_PROD_IMAGE="true" - export SKIP_CHECK_REMOTE_IMAGE="true" - shift 2 - ;; --init-script) export INIT_SCRIPT_FILE="${2}" echo "The initialization file is in ${INIT_SCRIPT_FILE}" @@ -1370,15 +1355,6 @@ ${CMDNAME} shell [FLAGS] [-- ] '${CMDNAME} shell -- -c \"ls -la\"' '${CMDNAME} -- -c \"ls -la\"' - For GitHub repository, the --github-repository flag can be used to specify the repository - to pull and push images. You can also use --github-image-id in case - you want to pull the image with specific COMMIT_SHA tag. - - '${CMDNAME} shell \\ - --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA - '${CMDNAME} \\ - --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA - " readonly DETAILED_USAGE_SHELL export DETAILED_USAGE_EXEC=" diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 879b45ea815e5..c68b3839df386 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -319,7 +319,6 @@ def pull_image( verbose=verbose, wait_for_image=wait_for_image, tag_as_latest=tag_as_latest, - poll_time=10.0, ) if return_code != 0: get_console().print(f"[error]There was an error when pulling CI image: {info}[/]") @@ -528,26 +527,41 @@ def build_ci_image_in_parallel( pool.close() -def rebuild_ci_image_if_needed( - build_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose: bool +def rebuild_or_pull_ci_image_if_needed( + command_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose: bool ) -> None: """ Rebuilds CI image if needed and user confirms it. - :param build_params: parameters of the shell + :param command_params: parameters of the command to execute :param dry_run: whether it's a dry_run :param verbose: should we print verbose messages """ build_ci_image_check_cache = Path( - BUILD_CACHE_DIR, build_params.airflow_branch, f".built_{build_params.python}" + BUILD_CACHE_DIR, command_params.airflow_branch, f".built_{command_params.python}" ) - ci_image_params = BuildCiParams(python=build_params.python, upgrade_to_newer_dependencies=False) + ci_image_params = BuildCiParams( + python=command_params.python, upgrade_to_newer_dependencies=False, image_tag=command_params.image_tag + ) + if command_params.image_tag is not None: + return_code, message = run_pull_image( + image_params=ci_image_params, + dry_run=dry_run, + verbose=verbose, + parallel=False, + wait_for_image=True, + tag_as_latest=False, + ) + if return_code != 0: + get_console().print(f"[error]Pulling image with {command_params.image_tag} failed! {message}[/]") + sys.exit(return_code) + return if build_ci_image_check_cache.exists(): if verbose: - get_console().print(f'[info]{build_params.image_type} image already built locally.[/]') + get_console().print(f'[info]{command_params.image_type} image already built locally.[/]') else: get_console().print( - f'[warning]{build_params.image_type} image was never built locally or deleted. ' + f'[warning]{command_params.image_type} image was never built locally or deleted. ' 'Forcing build.[/]' ) ci_image_params.force_build = True diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index a8ad686c708c5..bc324bf7e3798 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -21,7 +21,7 @@ import rich_click as click -from airflow_breeze.commands.ci_image_commands import rebuild_ci_image_if_needed +from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ( DEFAULT_PYTHON_MAJOR_MINOR_VERSION, @@ -45,6 +45,7 @@ option_force_build, option_forward_credentials, option_github_repository, + option_image_tag, option_installation_package_format, option_integration, option_load_default_connection, @@ -113,6 +114,7 @@ "--force-build", "--mount-sources", "--debian-version", + "--image-tag", ], }, ], @@ -141,6 +143,7 @@ "--force-build", "--mount-sources", "--debian-version", + "--image-tag", ], }, ], @@ -170,6 +173,7 @@ "--package-format", "--force-build", "--mount-sources", + "--image-tag", ], }, ], @@ -238,6 +242,7 @@ @option_mount_sources @option_integration @option_db_reset +@option_image_tag @option_answer @click.argument('extra-args', nargs=-1, type=click.UNPROCESSED) def shell( @@ -261,6 +266,7 @@ def shell( force_build: bool, db_reset: bool, answer: Optional[str], + image_tag: Optional[str], extra_args: Tuple, ): """Enter breeze.py environment. this is the default command use when no other is selected.""" @@ -289,6 +295,7 @@ def shell( extra_args=extra_args, answer=answer, debian_version=debian_version, + image_tag=image_tag, ) @@ -312,6 +319,7 @@ def shell( @option_installation_package_format @option_mount_sources @option_integration +@option_image_tag @option_db_reset @option_answer @click.argument('extra-args', nargs=-1, type=click.UNPROCESSED) @@ -335,6 +343,7 @@ def start_airflow( use_packages_from_dist: bool, package_format: str, force_build: bool, + image_tag: Optional[str], db_reset: bool, answer: Optional[str], extra_args: Tuple, @@ -362,6 +371,7 @@ def start_airflow( force_build=force_build, db_reset=db_reset, start_airflow=True, + image_tag=image_tag, extra_args=extra_args, answer=answer, ) @@ -398,7 +408,7 @@ def build_docs( """Build documentation in the container.""" perform_environment_checks(verbose=verbose) params = BuildCiParams(github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION) - rebuild_ci_image_if_needed(build_params=params, dry_run=dry_run, verbose=verbose) + rebuild_or_pull_ci_image_if_needed(command_params=params, dry_run=dry_run, verbose=verbose) ci_image_name = params.airflow_image_name doc_builder = DocBuildParams( package_filter=package_filter, @@ -581,7 +591,7 @@ def enter_shell(**kwargs) -> RunCommandResult: get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE) enter_shell_params = ShellParams(**filter_out_none(**kwargs)) enter_shell_params.include_mypy_volume = True - rebuild_ci_image_if_needed(build_params=enter_shell_params, dry_run=dry_run, verbose=verbose) + rebuild_or_pull_ci_image_if_needed(command_params=enter_shell_params, dry_run=dry_run, verbose=verbose) return run_shell(verbose, dry_run, enter_shell_params) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index ce80e6a1932a8..036e52ab0e8f1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -24,7 +24,7 @@ import click -from airflow_breeze.commands.ci_image_commands import rebuild_ci_image_if_needed +from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ( ALLOWED_PLATFORMS, @@ -260,7 +260,7 @@ def prepare_airflow_packages( install_providers_from_sources=False, mount_sources=MOUNT_ALL, ) - rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run, verbose=verbose) + rebuild_or_pull_ci_image_if_needed(command_params=shell_params, dry_run=dry_run, verbose=verbose) result_command = run_with_debug( params=shell_params, command=["/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh"], @@ -299,7 +299,7 @@ def prepare_provider_documentation( answer=answer, skip_environment_initialization=True, ) - rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run, verbose=verbose) + rebuild_or_pull_ci_image_if_needed(command_params=shell_params, dry_run=dry_run, verbose=verbose) cmd_to_run = ["/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh", *packages] result_command = run_with_debug( params=shell_params, @@ -351,7 +351,7 @@ def prepare_provider_packages( skip_environment_initialization=True, version_suffix_for_pypi=version_suffix_for_pypi, ) - rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run, verbose=verbose) + rebuild_or_pull_ci_image_if_needed(command_params=shell_params, dry_run=dry_run, verbose=verbose) cmd_to_run = ["/opt/airflow/scripts/in_container/run_prepare_provider_packages.sh", *packages_list] result_command = run_with_debug( params=shell_params, @@ -540,7 +540,7 @@ def verify_provider_packages( use_packages_from_dist=use_packages_from_dist, package_format=package_format, ) - rebuild_ci_image_if_needed(build_params=shell_params, dry_run=dry_run, verbose=verbose) + rebuild_or_pull_ci_image_if_needed(command_params=shell_params, dry_run=dry_run, verbose=verbose) cmd_to_run = [ "-c", "python /opt/airflow/scripts/in_container/verify_providers.py", @@ -621,8 +621,8 @@ def release_prod_images( dry_run: bool, ): perform_environment_checks(verbose=verbose) - rebuild_ci_image_if_needed( - build_params=ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION), + rebuild_or_pull_ci_image_if_needed( + command_params=ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION), dry_run=dry_run, verbose=verbose, ) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index ebe4701b73ceb..3946acd2ab3a9 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -23,7 +23,7 @@ import tempfile from threading import Event, Thread from time import sleep -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple import click @@ -40,6 +40,7 @@ option_image_name, option_image_tag, option_integration, + option_mount_sources, option_mssql_version, option_mysql_version, option_postgres_version, @@ -77,7 +78,6 @@ "options": [ "--integration", "--test-type", - "--limit-progress-output", "--db-reset", "--backend", "--python", @@ -85,7 +85,15 @@ "--mysql-version", "--mssql-version", ], - } + }, + { + "name": "Advanced flag for tests command", + "options": [ + "--limit-progress-output", + "--image-tag", + "--mount-sources", + ], + }, ], } @@ -235,7 +243,8 @@ def run_with_progress( help="Limit progress to percentage only and just show the summary when tests complete.", is_flag=True, ) -@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) +@option_image_tag +@option_mount_sources @click.option( "--test-type", help="Type of test to run.", @@ -243,6 +252,7 @@ def run_with_progress( type=BetterChoice(ALLOWED_TEST_TYPES), ) @option_db_reset +@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) def tests( dry_run: bool, verbose: bool, @@ -256,6 +266,8 @@ def tests( extra_pytest_args: Tuple, test_type: str, db_reset: bool, + image_tag: Optional[str], + mount_sources: str, ): os.environ["RUN_TESTS"] = "true" if test_type: @@ -274,6 +286,8 @@ def tests( postgres_version=postgres_version, mysql_version=mysql_version, mssql_version=mssql_version, + image_tag=image_tag, + mount_sources=mount_sources, ) env_variables = get_env_variables_for_docker_commands(exec_shell_params) perform_environment_checks(verbose=verbose) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 1c9b69c59ed13..bc73e1e721b78 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -60,9 +60,10 @@ MOUNT_SELECTED = "selected" MOUNT_ALL = "all" -MOUNT_NONE = "none" +MOUNT_SKIP = "skip" +MOUNT_REMOVE = "remove" -ALLOWED_MOUNT_OPTIONS = [MOUNT_SELECTED, MOUNT_ALL, MOUNT_NONE] +ALLOWED_MOUNT_OPTIONS = [MOUNT_SELECTED, MOUNT_ALL, MOUNT_SKIP, MOUNT_REMOVE] ALLOWED_POSTGRES_VERSIONS = ['10', '11', '12', '13', '14'] ALLOWED_MYSQL_VERSIONS = ['5.7', '8'] ALLOWED_MSSQL_VERSIONS = ['2017-latest', '2019-latest'] diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 3ec62e0fafd94..8b908aefa77d8 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -32,8 +32,9 @@ ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, AVAILABLE_INTEGRATIONS, MOUNT_ALL, - MOUNT_NONE, + MOUNT_REMOVE, MOUNT_SELECTED, + MOUNT_SKIP, get_airflow_version, ) from airflow_breeze.utils.console import get_console @@ -64,7 +65,7 @@ class ShellParams: github_actions: str = os.environ.get('GITHUB_ACTIONS', "false") github_repository: str = "apache/airflow" github_token: str = os.environ.get('GITHUB_TOKEN', "") - image_tag: str = "latest" + image_tag: Optional[str] = None include_mypy_volume: bool = False install_airflow_version: str = "" install_providers_from_sources: bool = True @@ -208,11 +209,23 @@ def compose_files(self): ) compose_ci_file.extend([main_ci_docker_compose_file, *backend_files, files_docker_compose_file]) + if self.image_tag is not None and self.image_tag != "latest": + get_console().print( + f"[warning]Running tagged image tag = {self.image_tag}. " + f"Forcing mounted sources to be 'skip'[/]" + ) + self.mount_sources = MOUNT_SKIP + if self.use_airflow_version is not None: + get_console().print( + "[info]Forcing --mount-sources to `remove` since we are not installing airflow " + f"from sources but from {self.use_airflow_version}[/]" + ) + self.mount_sources = MOUNT_REMOVE if self.mount_sources == MOUNT_SELECTED: compose_ci_file.extend([local_docker_compose_file]) elif self.mount_sources == MOUNT_ALL: compose_ci_file.extend([local_all_sources_docker_compose_file]) - else: # none + elif self.mount_sources == MOUNT_REMOVE: compose_ci_file.extend([remove_sources_docker_compose_file]) if self.forward_credentials: compose_ci_file.append(forward_credentials_docker_compose_file) @@ -235,11 +248,3 @@ def command_passed(self): if len(self.extra_args) > 0: cmd = str(self.extra_args[0]) return cmd - - def __post_init__(self): - if self.use_airflow_version is not None: - get_console().print( - "[info]Forcing --mount-sources to `none` since we are not installing airflow " - f"from sources but from {self.use_airflow_version}[/]" - ) - self.mount_sources = MOUNT_NONE diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index deca1b4882a4e..628bb8197972b 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -133,7 +133,7 @@ '--use-airflow-version', help="Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or `sdist`" " if Airflow should be removed, installed from wheel packages or sdist packages available in dist " - "folder respectively. Implies --mount-sources `none`.", + "folder respectively. Implies --mount-sources `remove`.", type=UseAirflowVersionType(ALLOWED_USE_AIRFLOW_VERSIONS), envvar='USE_AIRFLOW_VERSION', ) @@ -149,7 +149,7 @@ type=BetterChoice(ALLOWED_MOUNT_OPTIONS), default=ALLOWED_MOUNT_OPTIONS[0], show_default=True, - help="Choose scope of local sources should be mounted (default = selected).", + help="Choose scope of local sources that should be mounted, skipped, or removed (default = selected).", ) option_force_build = click.option( '--force-build', help="Force image build no matter if it is determined as needed.", is_flag=True @@ -186,16 +186,11 @@ help='The user name used to authenticate to GitHub.', envvar='GITHUB_USERNAME', ) -option_github_image_id = click.option( - '-s', - '--github-image-id', - help='Commit SHA of the image. \ - Breeze can automatically pull the commit SHA id specified Default: latest', -) option_image_tag = click.option( '-t', '--image-tag', - help='Tag added to the default naming conventions of Airflow CI/PROD images.', + help='Tag of the image which is used to pull or run the image (implies --mount-sources=skip' + ' when using to run shell or tests) ', envvar='IMAGE_TAG', ) option_image_name = click.option( diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index d1471aae0e124..45fd17b8580ef 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -43,6 +43,7 @@ MIN_DOCKER_COMPOSE_VERSION, MIN_DOCKER_VERSION, MOUNT_ALL, + MOUNT_REMOVE, MOUNT_SELECTED, MSSQL_HOST_PORT, MYSQL_HOST_PORT, @@ -112,7 +113,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: extra_docker_flags.extend( ['--mount', "type=volume,src=mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] ) - else: # none + elif mount_sources == MOUNT_REMOVE: extra_docker_flags.extend( ["--mount", f"type=bind,src={AIRFLOW_SOURCES_ROOT / 'empty'},dst=/opt/airflow/airflow"] ) diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py index 8cd516bb47cfc..36c40cf05a3e7 100644 --- a/dev/breeze/src/airflow_breeze/utils/image.py +++ b/dev/breeze/src/airflow_breeze/utils/image.py @@ -88,7 +88,7 @@ def run_pull_image( verbose: bool, wait_for_image: bool, tag_as_latest: bool, - poll_time: float, + poll_time: float = 10.0, parallel: bool = False, ) -> Tuple[int, str]: """ @@ -98,7 +98,7 @@ def run_pull_image( :param verbose: whether it's verbose :param wait_for_image: whether we should wait for the image to be available :param tag_as_latest: tag the image as latest - :param poll_time: what's the polling time between checks if images are there + :param poll_time: what's the polling time between checks if images are there (default 10 s) :param parallel: whether the pull is run as part of parallel execution :return: Tuple of return code and description of the image pulled """ diff --git a/dev/breeze/tests/test_commands.py b/dev/breeze/tests/test_commands.py index 8866d75af073a..3d375e8613fae 100644 --- a/dev/breeze/tests/test_commands.py +++ b/dev/breeze/tests/test_commands.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from airflow_breeze.global_constants import MOUNT_ALL, MOUNT_NONE, MOUNT_SELECTED +from airflow_breeze.global_constants import MOUNT_ALL, MOUNT_REMOVE, MOUNT_SELECTED, MOUNT_SKIP from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags from airflow_breeze.utils.visuals import ASCIIART @@ -35,7 +35,13 @@ def test_get_extra_docker_flags_selected(): assert len(flags) > 40 -def test_get_extra_docker_flags_none(): - flags = get_extra_docker_flags(MOUNT_NONE) +def test_get_extra_docker_flags_remove(): + flags = get_extra_docker_flags(MOUNT_REMOVE) assert "empty" in "".join(flags) assert len(flags) < 10 + + +def test_get_extra_docker_flags_skip(): + flags = get_extra_docker_flags(MOUNT_SKIP) + assert "empty" not in "".join(flags) + assert len(flags) < 10 diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 34c28afd8b91b..81703735d0155 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: build-image + Command: build-image - + - - -Usage: breeze build-image [OPTIONS] - -Build CI image. Include building multiple images for all python versions (sequentially). - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building                              │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-image [OPTIONS] + +Build CI image. Include building multiple images for all python versions (sequentially). + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies               │ +│--mount-sources=skip when using to run shell or tests)                         │ +│(TEXT)                                                                         │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-modeMode of constraints for CI image building                              │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index 494e77574c3e0..dcbd32a96662c 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: build-prod-image + Command: build-prod-image - + - - -Usage: breeze build-prod-image [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building                            │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ -│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ -│from PyPI or sources.                                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-prod-image [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies               │ +│--mount-sources=skip when using to run shell or tests)                         │ +│(TEXT)                                                                         │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building                            │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ +│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ +│from PyPI or sources.                                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt deleted file mode 100644 index 50377a3fee1ce..0000000000000 --- a/images/breeze/output-commands-hash.txt +++ /dev/null @@ -1 +0,0 @@ -2942c0bca323521e3e9af5922d527201 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index a592f4642e045..e9dd89b22e4dc 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Breeze commands + Breeze commands - + - - -Usage: breeze [OPTIONS] COMMAND [ARGS]... - -╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│available in dist folder respectively. Implies --mount-sources `none`.                  │ -│(none | wheel | sdist | <airflow_version>)                                              │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ -│entering breeze.                                                                        │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources should be mounted (default = selected).│ -│(selected | all | none)                                              │ -│[default: selected]                                                  │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│shell          Enter breeze.py environment. this is the default command use when no other is selected.            │ -│start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                 │ -│exec           Joins the interactive shell of running airflow container                                           │ -│stop           Stop running breeze environment.                                                                   │ -│build-docs     Build documentation in the container.                                                              │ -│static-checks  Run static checks.                                                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│docker-compose-tests Run docker-compose tests.                                                                    │ -│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ -│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ -│images.                                                                                  │ -│self-upgrade             Self upgrade Breeze.                                                                     │ -│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ -│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ -│resource-check           Check if available docker resources are enough.                                          │ -│free-space               Free space for jobs run in CI.                                                           │ -│fix-ownership            Fix ownership of source files to be same as host user.                                   │ -│regenerate-command-imagesRegenerate breeze command images.                                                        │ -│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ -│images should be regenerated).                                                           │ -│version                  Print information about version of apache-airflow-breeze.                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ -│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ -│verify-image  Verify CI image.                                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ -│sequentially.                                                                                    │ -│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ -│verify-prod-imageVerify Production image.                                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -│find-newer-dependencies          Finds which dependencies are being upgraded.                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze [OPTIONS] COMMAND [ARGS]... + +╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ +│available in dist folder respectively. Implies --mount-sources `remove`.                │ +│(none | wheel | sdist | <airflow_version>)                                              │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ +│entering breeze.                                                                        │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =    │ +│selected).                                                                              │ +│(selected | all | skip | remove)                                                        │ +│[default: selected]                                                                     │ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│shell          Enter breeze.py environment. this is the default command use when no other is selected.            │ +│start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                 │ +│exec           Joins the interactive shell of running airflow container                                           │ +│stop           Stop running breeze environment.                                                                   │ +│build-docs     Build documentation in the container.                                                              │ +│static-checks  Run static checks.                                                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│docker-compose-tests Run docker-compose tests.                                                                    │ +│tests                Run the specified unit test targets. Multiple targets may be specified separated by spaces.  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ +│cleanup                  Cleans the cache of parameters, docker cache and optionally - currently downloaded       │ +│images.                                                                                  │ +│self-upgrade             Self upgrade Breeze.                                                                     │ +│setup-autocomplete       Enables autocompletion of breeze commands.                                               │ +│config                   Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                       │ +│resource-check           Check if available docker resources are enough.                                          │ +│free-space               Free space for jobs run in CI.                                                           │ +│fix-ownership            Fix ownership of source files to be same as host user.                                   │ +│regenerate-command-imagesRegenerate breeze command images.                                                        │ +│command-hash-export      Outputs hash of all click commands to file or stdout if `-` is used (useful to see if    │ +│images should be regenerated).                                                           │ +│version                  Print information about version of apache-airflow-breeze.                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build-image   Build CI image. Include building multiple images for all python versions (sequentially).            │ +│pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                │ +│verify-image  Verify CI image.                                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build-prod-image Build Production image. Include building multiple images for all or selected Python versions     │ +│sequentially.                                                                                    │ +│pull-prod-image  Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ +│verify-prod-imageVerify Production image.                                                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +│find-newer-dependencies          Finds which dependencies are being upgraded.                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-docker-compose-tests.svg b/images/breeze/output-docker-compose-tests.svg index 75f5c1a31b102..c652741d47579 100644 --- a/images/breeze/output-docker-compose-tests.svg +++ b/images/breeze/output-docker-compose-tests.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: docker-compose-tests + Command: docker-compose-tests - + - - -Usage: breeze docker-compose-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run docker-compose tests. - -╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze docker-compose-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run docker-compose tests. + +╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ +│to run shell or tests)                                                                            │ +│(TEXT)                                                                                            │ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-generate-constraints.svg b/images/breeze/output-generate-constraints.svg index ca049566070eb..e0b54f182ebba 100644 --- a/images/breeze/output-generate-constraints.svg +++ b/images/breeze/output-generate-constraints.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: generate-constraints + Command: generate-constraints - + - - -Usage: breeze generate-constraints [OPTIONS] - -Generates pinned constraint files with all extras from setup.py in parallel. - -╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--airflow-constraints-modeMode of constraints for CI image building                              │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze generate-constraints [OPTIONS] + +Generates pinned constraint files with all extras from setup.py in parallel. + +╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to pull or run the image (implies                    │ +│--mount-sources=skip when using to run shell or tests)                              │ +│(TEXT)                                                                              │ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--airflow-constraints-modeMode of constraints for CI image building                              │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-pull-image.svg b/images/breeze/output-pull-image.svg index e1cce7beebc1c..edb77e062ed5a 100644 --- a/images/breeze/output-pull-image.svg +++ b/images/breeze/output-pull-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: pull-image + Command: pull-image - + - - -Usage: breeze pull-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Pull and optionally verify CI images - possibly in parallel for all Python versions. - -╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verify-imageVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ -│or pull image with --image-tag.                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze pull-image [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Pull and optionally verify CI images - possibly in parallel for all Python versions. + +╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when    │ +│using to run shell or tests)                                                                  │ +│(TEXT)                                                                                        │ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verify-imageVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ +│or pull image with --image-tag.                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-pull-prod-image.svg b/images/breeze/output-pull-prod-image.svg index d6ee293d03b8a..e714d08acd989 100644 --- a/images/breeze/output-pull-prod-image.svg +++ b/images/breeze/output-pull-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: pull-prod-image + Command: pull-prod-image - + - - -Usage: breeze pull-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Pull and optionally verify Production images - possibly in parallel for all Python versions. - -╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verify-imageVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ -│or pull image with --image-tag.                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze pull-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Pull and optionally verify Production images - possibly in parallel for all Python versions. + +╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when    │ +│using to run shell or tests)                                                                  │ +│(TEXT)                                                                                        │ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verify-imageVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ +│or pull image with --image-tag.                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-shell.svg b/images/breeze/output-shell.svg index f279eab2d729e..2c29bda22ac20 100644 --- a/images/breeze/output-shell.svg +++ b/images/breeze/output-shell.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + - Command: shell + Command: shell - + - - -Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... - -Enter breeze.py environment. this is the default command use when no other is selected. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│available in dist folder respectively. Implies --mount-sources `none`.                  │ -│(none | wheel | sdist | <airflow_version>)                                              │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ -│entering breeze.                                                                        │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources should be mounted (default = selected).│ -│(selected | all | none)                                              │ -│[default: selected]                                                  │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... + +Enter breeze.py environment. this is the default command use when no other is selected. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ +│available in dist folder respectively. Implies --mount-sources `remove`.              │ +│(none | wheel | sdist | <airflow_version>)                                            │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ +│entering breeze.                                                                      │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ +│selected).                                                                            │ +│(selected | all | skip | remove)                                                      │ +│[default: selected]                                                                   │ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip │ +│when using to run shell or tests)                                                     │ +│(TEXT)                                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-start-airflow.svg b/images/breeze/output-start-airflow.svg index 159eec8cb29a0..3e21e1f5e97fa 100644 --- a/images/breeze/output-start-airflow.svg +++ b/images/breeze/output-start-airflow.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + - Command: start-airflow + Command: start-airflow - + - - -Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... - -Enter breeze.py environment and starts all Airflow components in the tmux session. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ -│| all)                                                                              │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│available in dist folder respectively. Implies --mount-sources `none`.                  │ -│(none | wheel | sdist | <airflow_version>)                                              │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when   │ -│entering breeze.                                                                        │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)[default: wheel]│ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources should be mounted (default = selected).│ -│(selected | all | none)                                              │ -│[default: selected]                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... + +Enter breeze.py environment and starts all Airflow components in the tmux session. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                       │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ +│| all)                                                                              │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ +│available in dist folder respectively. Implies --mount-sources `remove`.              │ +│(none | wheel | sdist | <airflow_version>)                                            │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ +│entering breeze.                                                                      │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ +│selected).                                                                            │ +│(selected | all | skip | remove)                                                      │ +│[default: selected]                                                                   │ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip │ +│when using to run shell or tests)                                                     │ +│(TEXT)                                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-tests.svg b/images/breeze/output-tests.svg index 914f2c4587a0b..1d819816f2596 100644 --- a/images/breeze/output-tests.svg +++ b/images/breeze/output-tests.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - Command: tests + Command: tests - + - - -Usage: breeze tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run the specified unit test targets. Multiple targets may be specified separated by spaces. - -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                          │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | │ -│all)                                                                                   │ -│--test-typeType of test to run.                                                                   │ -│(All | Always | Core | Providers | API | CLI | Integration | Other | WWW | Postgres |  │ -│MySQL | Helm | Quarantined)                                                            │ -│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run the specified unit test targets. Multiple targets may be specified separated by spaces. + +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ +│--test-typeType of test to run.                                                                        │ +│(All | Always | Core | Providers | API | CLI | Integration | Other | WWW | Postgres | MySQL │ +│| Helm | Quarantined)                                                                       │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--limit-progress-outputLimit progress to percentage only and just show the summary when tests complete.│ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip  │ +│when using to run shell or tests)                                                      │ +│(TEXT)                                                                                 │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =   │ +│selected).                                                                             │ +│(selected | all | skip | remove)                                                       │ +│[default: selected]                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-verify-image.svg b/images/breeze/output-verify-image.svg index b79efb93f2602..9c6d1a641fc52 100644 --- a/images/breeze/output-verify-image.svg +++ b/images/breeze/output-verify-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: verify-image + Command: verify-image - + - - -Usage: breeze verify-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify CI image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze verify-image [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify CI image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ +│to run shell or tests)                                                                            │ +│(TEXT)                                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-verify-prod-image.svg b/images/breeze/output-verify-prod-image.svg index 4bc7a36527741..48b61a30826b5 100644 --- a/images/breeze/output-verify-prod-image.svg +++ b/images/breeze/output-verify-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: verify-prod-image + Command: verify-prod-image - + - - -Usage: breeze verify-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify Production image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag added to the default naming conventions of Airflow CI/PROD images.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze verify-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify Production image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ +│to run shell or tests)                                                                            │ +│(TEXT)                                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-verify-provider-packages.svg b/images/breeze/output-verify-provider-packages.svg index cfb88fbcc4755..12853b46a203f 100644 --- a/images/breeze/output-verify-provider-packages.svg +++ b/images/breeze/output-verify-provider-packages.svg @@ -19,149 +19,149 @@ font-weight: 700; } - .terminal-1322667406-matrix { + .terminal-1578520108-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1322667406-title { + .terminal-1578520108-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1322667406-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1322667406-r2 { fill: #c5c8c6 } -.terminal-1322667406-r3 { fill: #d0b344;font-weight: bold } -.terminal-1322667406-r4 { fill: #868887 } -.terminal-1322667406-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1322667406-r6 { fill: #8d7b39 } -.terminal-1322667406-r7 { fill: #98a84b;font-weight: bold } + .terminal-1578520108-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-1578520108-r2 { fill: #c5c8c6 } +.terminal-1578520108-r3 { fill: #d0b344;font-weight: bold } +.terminal-1578520108-r4 { fill: #868887 } +.terminal-1578520108-r5 { fill: #68a0b3;font-weight: bold } +.terminal-1578520108-r6 { fill: #8d7b39 } +.terminal-1578520108-r7 { fill: #98a84b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: verify-provider-packages + Command: verify-provider-packages - + - - -Usage: breeze verify-provider-packages [OPTIONS] - -Verifies if all provider code is following expectations for providers. - -╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,       │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages  │ -│or sdist packages available in dist folder respectively. Implies --mount-sources│ -│`none`.                                                                          │ -│(none | wheel | sdist | <airflow_version>)                                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to      │ -│specify constraints for the installed version and to find newer dependencies     │ -│(TEXT)                                                                           │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder │ -│when entering breeze.                                                            │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze verify-provider-packages [OPTIONS] + +Verifies if all provider code is following expectations for providers. + +╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,       │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages  │ +│or sdist packages available in dist folder respectively. Implies --mount-sources│ +│`remove`.                                                                        │ +│(none | wheel | sdist | <airflow_version>)                                       │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to      │ +│specify constraints for the installed version and to find newer dependencies     │ +│(TEXT)                                                                           │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder │ +│when entering breeze.                                                            │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh index d5db23fa7410a..d120b50ec75c4 100755 --- a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh +++ b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh @@ -133,10 +133,12 @@ function run_airflow_testing_in_docker() { echo "${COLOR_RED}***********************************************************************************************${COLOR_RESET}" echo echo "${COLOR_BLUE}***********************************************************************************************${COLOR_RESET}" - echo "${COLOR_BLUE}Reproduce the failed tests on your local machine (note that you need to use docker-compose v1 rather than v2 to enable Kerberos integration):${COLOR_RESET}" - echo "${COLOR_YELLOW}./breeze --github-image-id ${GITHUB_REGISTRY_PULL_IMAGE_TAG=} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} ${INTEGRATION_BREEZE_FLAGS[*]} shell${COLOR_RESET}" + echo "${COLOR_BLUE}Enter the same environment that was used for the tests:${COLOR_RESET}" + echo "${COLOR_YELLOW}breeze --image-tag ${IMAGE_TAG=} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} ${INTEGRATION_BREEZE_FLAGS[*]} shell${COLOR_RESET}" echo "${COLOR_BLUE}Then you can run failed tests with:${COLOR_RESET}" echo "${COLOR_YELLOW}pytest [TEST_NAME]${COLOR_RESET}" + echo "${COLOR_BLUE}Or you can run the tests: ${COLOR_RESET}" + echo "${COLOR_YELLOW}breeze --image-tag ${IMAGE_TAG=} --backend ${BACKEND} ${EXTRA_ARGS}--python ${PYTHON_MAJOR_MINOR_VERSION} --db-reset --skip-mounting-local-sources --test-type ${TEST_TYPE} ${INTEGRATION_BREEZE_FLAGS[*]} test${COLOR_RESET}" echo "${COLOR_BLUE}***********************************************************************************************${COLOR_RESET}"