Skip to content

Commit

Permalink
Add --platform to breeze shell and start-airflow (apache#24626)
Browse files Browse the repository at this point in the history
This allows Breeze to run images of another platform, such as AMD64 on
an ARM machine.
  • Loading branch information
uranusjr authored Jun 24, 2022
1 parent ded22eb commit 5a8209e
Show file tree
Hide file tree
Showing 9 changed files with 264 additions and 236 deletions.
9 changes: 6 additions & 3 deletions dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
option_image_tag,
option_install_providers_from_sources,
option_parallelism,
option_platform,
option_platform_multiple,
option_prepare_buildx_cache,
option_pull_image,
option_push_image,
Expand Down Expand Up @@ -231,7 +231,7 @@ def run_build_in_parallel(
@option_parallelism
@option_python_versions
@option_upgrade_to_newer_dependencies
@option_platform
@option_platform_multiple
@option_debian_version
@option_github_token
@option_github_username
Expand Down Expand Up @@ -575,7 +575,10 @@ def rebuild_or_pull_ci_image_if_needed(
BUILD_CACHE_DIR, command_params.airflow_branch, f".built_{command_params.python}"
)
ci_image_params = BuildCiParams(
python=command_params.python, upgrade_to_newer_dependencies=False, image_tag=command_params.image_tag
python=command_params.python,
upgrade_to_newer_dependencies=False,
image_tag=command_params.image_tag,
platform=command_params.platform,
)
if command_params.image_tag is not None:
return_code, message = run_pull_image(
Expand Down
7 changes: 7 additions & 0 deletions dev/breeze/src/airflow_breeze/commands/developer_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
option_mount_sources,
option_mssql_version,
option_mysql_version,
option_platform_single,
option_postgres_version,
option_python,
option_use_airflow_version,
Expand Down Expand Up @@ -226,6 +227,7 @@
@option_verbose
@option_dry_run
@option_python
@option_platform_single
@option_backend
@option_debian_version
@option_github_repository
Expand Down Expand Up @@ -267,6 +269,7 @@ def shell(
db_reset: bool,
answer: Optional[str],
image_tag: Optional[str],
platform: Optional[str],
extra_args: Tuple,
):
"""Enter breeze.py environment. this is the default command use when no other is selected."""
Expand Down Expand Up @@ -296,13 +299,15 @@ def shell(
answer=answer,
debian_version=debian_version,
image_tag=image_tag,
platform=platform,
)


@option_verbose
@main.command(name='start-airflow')
@option_dry_run
@option_python
@option_platform_single
@option_github_repository
@option_backend
@option_postgres_version
Expand Down Expand Up @@ -346,6 +351,7 @@ def start_airflow(
image_tag: Optional[str],
db_reset: bool,
answer: Optional[str],
platform: Optional[str],
extra_args: Tuple,
):
"""Enter breeze.py environment and starts all Airflow components in the tmux session."""
Expand All @@ -372,6 +378,7 @@ def start_airflow(
db_reset=db_reset,
start_airflow=True,
image_tag=image_tag,
platform=platform,
extra_args=extra_args,
answer=answer,
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
option_image_tag,
option_install_providers_from_sources,
option_parallelism,
option_platform,
option_platform_multiple,
option_prepare_buildx_cache,
option_pull_image,
option_push_image,
Expand Down Expand Up @@ -238,7 +238,7 @@ def run_build_in_parallel(
@option_parallelism
@option_python_versions
@option_upgrade_to_newer_dependencies
@option_platform
@option_platform_multiple
@option_debian_version
@option_github_repository
@option_github_token
Expand Down
3 changes: 2 additions & 1 deletion dev/breeze/src/airflow_breeze/global_constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@
ALLOWED_DEBIAN_VERSIONS = ['bullseye', 'buster']
ALLOWED_BUILD_CACHE = ["registry", "local", "disabled"]
MULTI_PLATFORM = "linux/amd64,linux/arm64"
ALLOWED_PLATFORMS = ["linux/amd64", "linux/arm64", MULTI_PLATFORM]
SINGLE_PLATFORMS = ["linux/amd64", "linux/arm64"]
ALLOWED_PLATFORMS = [*SINGLE_PLATFORMS, MULTI_PLATFORM]
ALLOWED_USE_AIRFLOW_VERSIONS = ['none', 'wheel', 'sdist']

PARAM_NAME_DESCRIPTION = {
Expand Down
2 changes: 2 additions & 0 deletions dev/breeze/src/airflow_breeze/params/shell_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
ALLOWED_POSTGRES_VERSIONS,
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
AVAILABLE_INTEGRATIONS,
DOCKER_DEFAULT_PLATFORM,
MOUNT_ALL,
MOUNT_REMOVE,
MOUNT_SELECTED,
Expand Down Expand Up @@ -78,6 +79,7 @@ class ShellParams:
mysql_version: str = ALLOWED_MYSQL_VERSIONS[0]
num_runs: str = ""
package_format: str = ALLOWED_INSTALLATION_PACKAGE_FORMATS[0]
platform: str = DOCKER_DEFAULT_PLATFORM
postgres_version: str = ALLOWED_POSTGRES_VERSIONS[0]
python: str = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0]
skip_environment_initialization: bool = False
Expand Down
9 changes: 8 additions & 1 deletion dev/breeze/src/airflow_breeze/utils/common_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
ALLOWED_POSTGRES_VERSIONS,
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
ALLOWED_USE_AIRFLOW_VERSIONS,
SINGLE_PLATFORMS,
get_available_packages,
)
from airflow_breeze.utils.custom_param_types import (
Expand Down Expand Up @@ -196,12 +197,18 @@
option_image_name = click.option(
'-n', '--image-name', help='Name of the image to verify (overrides --python and --image-tag).'
)
option_platform = click.option(
option_platform_multiple = click.option(
'--platform',
help='Platform for Airflow image.',
envvar='PLATFORM',
type=BetterChoice(ALLOWED_PLATFORMS),
)
option_platform_single = click.option(
'--platform',
help='Platform for Airflow image.',
envvar='PLATFORM',
type=BetterChoice(SINGLE_PLATFORMS),
)
option_debian_version = click.option(
'--debian-version',
help='Debian version used for the image.',
Expand Down
2 changes: 1 addition & 1 deletion images/breeze/output-commands-hash.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
# This file is automatically generated by pre-commit. If you have a conflict with this file
# Please do not solve it but run `breeze regenerate-command-images`.
# This command should fix the conflict and regenerate help images that you have conflict with.
e5f34a68b51a9a83e96830bdf837e33f
d07fe7a09a846de917cda7820a8ede9d
Loading

0 comments on commit 5a8209e

Please sign in to comment.