From 5a8209e5096528b6f562efebbe71b6b9c378aaed Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 24 Jun 2022 18:12:14 +0800 Subject: [PATCH] Add --platform to breeze shell and start-airflow (#24626) This allows Breeze to run images of another platform, such as AMD64 on an ARM machine. --- .../commands/ci_image_commands.py | 9 +- .../commands/developer_commands.py | 7 + .../commands/production_image_commands.py | 4 +- .../src/airflow_breeze/global_constants.py | 3 +- .../src/airflow_breeze/params/shell_params.py | 2 + .../airflow_breeze/utils/common_options.py | 9 +- images/breeze/output-commands-hash.txt | 2 +- images/breeze/output-shell.svg | 228 ++++++++--------- images/breeze/output-start-airflow.svg | 236 +++++++++--------- 9 files changed, 264 insertions(+), 236 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 859bb2e905b51..15f672c4211a7 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -51,7 +51,7 @@ option_image_tag, option_install_providers_from_sources, option_parallelism, - option_platform, + option_platform_multiple, option_prepare_buildx_cache, option_pull_image, option_push_image, @@ -231,7 +231,7 @@ def run_build_in_parallel( @option_parallelism @option_python_versions @option_upgrade_to_newer_dependencies -@option_platform +@option_platform_multiple @option_debian_version @option_github_token @option_github_username @@ -575,7 +575,10 @@ def rebuild_or_pull_ci_image_if_needed( BUILD_CACHE_DIR, command_params.airflow_branch, f".built_{command_params.python}" ) ci_image_params = BuildCiParams( - python=command_params.python, upgrade_to_newer_dependencies=False, image_tag=command_params.image_tag + python=command_params.python, + upgrade_to_newer_dependencies=False, + image_tag=command_params.image_tag, + platform=command_params.platform, ) if command_params.image_tag is not None: return_code, message = run_pull_image( diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index bc324bf7e3798..4db880ec14bc4 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -53,6 +53,7 @@ option_mount_sources, option_mssql_version, option_mysql_version, + option_platform_single, option_postgres_version, option_python, option_use_airflow_version, @@ -226,6 +227,7 @@ @option_verbose @option_dry_run @option_python +@option_platform_single @option_backend @option_debian_version @option_github_repository @@ -267,6 +269,7 @@ def shell( db_reset: bool, answer: Optional[str], image_tag: Optional[str], + platform: Optional[str], extra_args: Tuple, ): """Enter breeze.py environment. this is the default command use when no other is selected.""" @@ -296,6 +299,7 @@ def shell( answer=answer, debian_version=debian_version, image_tag=image_tag, + platform=platform, ) @@ -303,6 +307,7 @@ def shell( @main.command(name='start-airflow') @option_dry_run @option_python +@option_platform_single @option_github_repository @option_backend @option_postgres_version @@ -346,6 +351,7 @@ def start_airflow( image_tag: Optional[str], db_reset: bool, answer: Optional[str], + platform: Optional[str], extra_args: Tuple, ): """Enter breeze.py environment and starts all Airflow components in the tmux session.""" @@ -372,6 +378,7 @@ def start_airflow( db_reset=db_reset, start_airflow=True, image_tag=image_tag, + platform=platform, extra_args=extra_args, answer=answer, ) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 9ae3791e31264..905fa6cc6b9ea 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -50,7 +50,7 @@ option_image_tag, option_install_providers_from_sources, option_parallelism, - option_platform, + option_platform_multiple, option_prepare_buildx_cache, option_pull_image, option_push_image, @@ -238,7 +238,7 @@ def run_build_in_parallel( @option_parallelism @option_python_versions @option_upgrade_to_newer_dependencies -@option_platform +@option_platform_multiple @option_debian_version @option_github_repository @option_github_token diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index bc73e1e721b78..7eeaf1c8605d6 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -88,7 +88,8 @@ ALLOWED_DEBIAN_VERSIONS = ['bullseye', 'buster'] ALLOWED_BUILD_CACHE = ["registry", "local", "disabled"] MULTI_PLATFORM = "linux/amd64,linux/arm64" -ALLOWED_PLATFORMS = ["linux/amd64", "linux/arm64", MULTI_PLATFORM] +SINGLE_PLATFORMS = ["linux/amd64", "linux/arm64"] +ALLOWED_PLATFORMS = [*SINGLE_PLATFORMS, MULTI_PLATFORM] ALLOWED_USE_AIRFLOW_VERSIONS = ['none', 'wheel', 'sdist'] PARAM_NAME_DESCRIPTION = { diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 1b5d6925dee9c..b4c1e73f5a827 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -31,6 +31,7 @@ ALLOWED_POSTGRES_VERSIONS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, AVAILABLE_INTEGRATIONS, + DOCKER_DEFAULT_PLATFORM, MOUNT_ALL, MOUNT_REMOVE, MOUNT_SELECTED, @@ -78,6 +79,7 @@ class ShellParams: mysql_version: str = ALLOWED_MYSQL_VERSIONS[0] num_runs: str = "" package_format: str = ALLOWED_INSTALLATION_PACKAGE_FORMATS[0] + platform: str = DOCKER_DEFAULT_PLATFORM postgres_version: str = ALLOWED_POSTGRES_VERSIONS[0] python: str = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0] skip_environment_initialization: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 292e74891dbfa..f6fa31136733c 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -37,6 +37,7 @@ ALLOWED_POSTGRES_VERSIONS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, ALLOWED_USE_AIRFLOW_VERSIONS, + SINGLE_PLATFORMS, get_available_packages, ) from airflow_breeze.utils.custom_param_types import ( @@ -196,12 +197,18 @@ option_image_name = click.option( '-n', '--image-name', help='Name of the image to verify (overrides --python and --image-tag).' ) -option_platform = click.option( +option_platform_multiple = click.option( '--platform', help='Platform for Airflow image.', envvar='PLATFORM', type=BetterChoice(ALLOWED_PLATFORMS), ) +option_platform_single = click.option( + '--platform', + help='Platform for Airflow image.', + envvar='PLATFORM', + type=BetterChoice(SINGLE_PLATFORMS), +) option_debian_version = click.option( '--debian-version', help='Debian version used for the image.', diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 9a22f6dd61e92..cadf246543cf4 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,4 +2,4 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -e5f34a68b51a9a83e96830bdf837e33f +d07fe7a09a846de917cda7820a8ede9d diff --git a/images/breeze/output-shell.svg b/images/breeze/output-shell.svg index 2c29bda22ac20..f81354da6aee9 100644 --- a/images/breeze/output-shell.svg +++ b/images/breeze/output-shell.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: shell + Command: shell - + - - -Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... - -Enter breeze.py environment. this is the default command use when no other is selected. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ -│available in dist folder respectively. Implies --mount-sources `remove`.              │ -│(none | wheel | sdist | <airflow_version>)                                            │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ -│entering breeze.                                                                      │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ -│selected).                                                                            │ -│(selected | all | skip | remove)                                                      │ -│[default: selected]                                                                   │ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip │ -│when using to run shell or tests)                                                     │ -│(TEXT)                                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... + +Enter breeze.py environment. this is the default command use when no other is selected. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ +│available in dist folder respectively. Implies --mount-sources `remove`.              │ +│(none | wheel | sdist | <airflow_version>)                                            │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ +│entering breeze.                                                                      │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ +│selected).                                                                            │ +│(selected | all | skip | remove)                                                      │ +│[default: selected]                                                                   │ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip │ +│when using to run shell or tests)                                                     │ +│(TEXT)                                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-start-airflow.svg b/images/breeze/output-start-airflow.svg index 3e21e1f5e97fa..0378b3a55732c 100644 --- a/images/breeze/output-start-airflow.svg +++ b/images/breeze/output-start-airflow.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: start-airflow + Command: start-airflow - + - - -Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... - -Enter breeze.py environment and starts all Airflow components in the tmux session. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ -│| all)                                                                              │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ -│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ -│available in dist folder respectively. Implies --mount-sources `remove`.              │ -│(none | wheel | sdist | <airflow_version>)                                            │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ -│entering breeze.                                                                      │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ -│selected).                                                                            │ -│(selected | all | skip | remove)                                                      │ -│[default: selected]                                                                   │ -│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip │ -│when using to run shell or tests)                                                     │ -│(TEXT)                                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... + +Enter breeze.py environment and starts all Airflow components in the tmux session. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                       │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ +│| all)                                                                              │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or│ +│`sdist` if Airflow should be removed, installed from wheel packages or sdist packages │ +│available in dist folder respectively. Implies --mount-sources `remove`.              │ +│(none | wheel | sdist | <airflow_version>)                                            │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist' folder when │ +│entering breeze.                                                                      │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =  │ +│selected).                                                                            │ +│(selected | all | skip | remove)                                                      │ +│[default: selected]                                                                   │ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip │ +│when using to run shell or tests)                                                     │ +│(TEXT)                                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯