diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py new file mode 100644 index 00000000000..153981c0d22 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/branch_defaults.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +AIRFLOW_BRANCH = "main" +DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main" diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index c854d323924..01b2517fa0d 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -21,8 +21,8 @@ import click from click import ClickException -from rich.console import Console +from airflow_breeze.console import console from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE NAME = "Breeze2" @@ -32,8 +32,6 @@ __AIRFLOW_CFG_FILE = "setup.cfg" -console = Console(force_terminal=True, color_system="standard", width=180) - def get_airflow_sources_root(): return __AIRFLOW_SOURCES_ROOT @@ -95,13 +93,107 @@ def shell(verbose: bool): @option_verbose -@main.command() -def build_ci_image(verbose: bool): - """Builds breeze.ci image for breeze.py.""" +@main.command(name='build-ci-image') +@click.option( + '--additional-extras', + help='This installs additional extra package while installing airflow in the image.', +) +@click.option('-p', '--python', help='Choose your python version') +@click.option( + '--additional-dev-apt-deps', help='Additional apt dev dependencies to use when building the images.' +) +@click.option( + '--additional-runtime-apt-deps', + help='Additional apt runtime dependencies to use when building the images.', +) +@click.option( + '--additional-python-deps', help='Additional python dependencies to use when building the images.' +) +@click.option( + '--additional_dev_apt_command', help='Additional command executed before dev apt deps are installed.' +) +@click.option( + '--additional_runtime_apt_command', + help='Additional command executed before runtime apt deps are installed.', +) +@click.option( + '--additional_dev_apt_env', help='Additional environment variables set when adding dev dependencies.' +) +@click.option( + '--additional_runtime_apt_env', + help='Additional environment variables set when adding runtime dependencies.', +) +@click.option('--dev-apt-command', help='The basic command executed before dev apt deps are installed.') +@click.option( + '--dev-apt-deps', + help='The basic apt dev dependencies to use when building the images.', +) +@click.option( + '--runtime-apt-command', help='The basic command executed before runtime apt deps are installed.' +) +@click.option( + '--runtime-apt-deps', + help='The basic apt runtime dependencies to use when building the images.', +) +@click.option('--github-repository', help='Choose repository to push/pull image.') +@click.option('--build-cache', help='Cache option') +@click.option('--upgrade-to-newer-dependencies', is_flag=True) +def build_ci_image( + verbose: bool, + additional_extras: Optional[str], + python: Optional[float], + additional_dev_apt_deps: Optional[str], + additional_runtime_apt_deps: Optional[str], + additional_python_deps: Optional[str], + additional_dev_apt_command: Optional[str], + additional_runtime_apt_command: Optional[str], + additional_dev_apt_env: Optional[str], + additional_runtime_apt_env: Optional[str], + dev_apt_command: Optional[str], + dev_apt_deps: Optional[str], + runtime_apt_command: Optional[str], + runtime_apt_deps: Optional[str], + github_repository: Optional[str], + build_cache: Optional[str], + upgrade_to_newer_dependencies: bool, +): + """Builds docker CI image without entering the container.""" + from airflow_breeze.ci.build_image import build_image + if verbose: console.print(f"\n[blue]Building image of airflow from {__AIRFLOW_SOURCES_ROOT}[/]\n") - raise ClickException("\nPlease implement building the CI image\n") + build_image( + verbose, + additional_extras=additional_extras, + python_version=python, + additional_dev_apt_deps=additional_dev_apt_deps, + additional_runtime_apt_deps=additional_runtime_apt_deps, + additional_python_deps=additional_python_deps, + additional_runtime_apt_command=additional_runtime_apt_command, + additional_dev_apt_command=additional_dev_apt_command, + additional_dev_apt_env=additional_dev_apt_env, + additional_runtime_apt_env=additional_runtime_apt_env, + dev_apt_command=dev_apt_command, + dev_apt_deps=dev_apt_deps, + runtime_apt_command=runtime_apt_command, + runtime_apt_deps=runtime_apt_deps, + github_repository=github_repository, + docker_cache=build_cache, + upgrade_to_newer_dependencies=upgrade_to_newer_dependencies, + ) + + +@option_verbose +@main.command(name='build-prod-image') +def build_prod_image(verbose: bool): + """Builds docker Production image without entering the container.""" + if verbose: + console.print("\n[blue]Building image[/]\n") + raise ClickException("\nPlease implement building the Production image\n") if __name__ == '__main__': + from airflow_breeze.cache import BUILD_CACHE_DIR + + BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True) main() diff --git a/dev/breeze/src/airflow_breeze/cache.py b/dev/breeze/src/airflow_breeze/cache.py new file mode 100644 index 00000000000..2930020e99c --- /dev/null +++ b/dev/breeze/src/airflow_breeze/cache.py @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +from pathlib import Path +from typing import Any, List, Optional, Tuple + +from airflow_breeze import global_constants +from airflow_breeze.breeze import get_airflow_sources_root +from airflow_breeze.console import console + +BUILD_CACHE_DIR = Path(get_airflow_sources_root(), '.build') + + +def check_if_cache_exists(param_name: str) -> bool: + return (Path(BUILD_CACHE_DIR) / f".{param_name}").exists() + + +def read_from_cache_file(param_name: str) -> Optional[str]: + cache_exists = check_if_cache_exists(param_name) + if cache_exists: + return (Path(BUILD_CACHE_DIR) / f".{param_name}").read_text().strip() + else: + return None + + +def write_to_cache_file(param_name: str, param_value: str, check_allowed_values: bool = True) -> None: + allowed = False + if check_allowed_values: + allowed, allowed_values = check_if_values_allowed(param_name, param_value) + if allowed or not check_allowed_values: + Path(BUILD_CACHE_DIR, f".{param_name}").write_text(param_value) + else: + console.print(f'[cyan]You have sent the {param_value} for {param_name}') + console.print(f'[cyan]Allowed value for the {param_name} are {allowed_values}') + console.print('[cyan]Provide one of the supported params. Write to cache dir failed') + sys.exit() + + +def check_cache_and_write_if_not_cached( + param_name: str, default_param_value: str +) -> Tuple[bool, Optional[str]]: + is_cached = False + allowed = False + cached_value = read_from_cache_file(param_name) + if cached_value is None: + write_to_cache_file(param_name, default_param_value) + cached_value = default_param_value + else: + allowed, allowed_values = check_if_values_allowed(param_name, cached_value) + if allowed: + is_cached = True + else: + write_to_cache_file(param_name, default_param_value) + cached_value = default_param_value + return is_cached, cached_value + + +def check_if_values_allowed(param_name: str, param_value: str) -> Tuple[bool, List[Any]]: + allowed = False + allowed_values: List[Any] = [] + allowed_values = getattr(global_constants, f'ALLOWED_{param_name.upper()}') + if param_value in allowed_values: + allowed = True + return allowed, allowed_values diff --git a/dev/breeze/src/airflow_breeze/ci/build_image.py b/dev/breeze/src/airflow_breeze/ci/build_image.py new file mode 100644 index 00000000000..ed6e3dd1d12 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/ci/build_image.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# from airflow_breeze.utils import run_command +from pathlib import Path +from typing import List + +from airflow_breeze.breeze import get_airflow_sources_root +from airflow_breeze.cache import check_cache_and_write_if_not_cached +from airflow_breeze.ci.build_params import BuildParams +from airflow_breeze.console import console +from airflow_breeze.utils import filter_out_none, run_command + +PARAMS_CI_IMAGE = [ + "python_base_image", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints_reference", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", + "commit_sha", +] + +PARAMS_TO_VERIFY_CI_IMAGE = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", +] + + +def construct_arguments_docker_command(ci_image: BuildParams) -> List[str]: + args_command = [] + for param in PARAMS_CI_IMAGE: + args_command.append("--build-arg") + args_command.append(param.upper() + "=" + str(getattr(ci_image, param))) + for verify_param in PARAMS_TO_VERIFY_CI_IMAGE: + param_value = str(getattr(ci_image, verify_param)) + if len(param_value) > 0: + args_command.append("--build-arg") + args_command.append(verify_param.upper() + "=" + param_value) + docker_cache = ci_image.docker_cache_ci_directive + if len(docker_cache) > 0: + args_command.extend(ci_image.docker_cache_ci_directive) + return args_command + + +def construct_docker_command(ci_image: BuildParams) -> List[str]: + arguments = construct_arguments_docker_command(ci_image) + final_command = [] + final_command.extend(["docker", "build"]) + final_command.extend(arguments) + final_command.extend(["-t", ci_image.airflow_ci_image_name, "--target", "main", "."]) + final_command.extend(["-f", str(Path(get_airflow_sources_root(), 'Dockerfile.ci').resolve())]) + return final_command + + +def build_image(verbose, **kwargs): + ci_image_params = BuildParams(filter_out_none(**kwargs)) + is_cached, value = check_cache_and_write_if_not_cached( + "PYTHON_MAJOR_MINOR_VERSION", ci_image_params.python_version + ) + if is_cached: + ci_image_params.python_version = value + cmd = construct_docker_command(ci_image_params) + output = run_command(cmd, verbose=verbose, text=True) + console.print(f"[blue]{output}") diff --git a/dev/breeze/src/airflow_breeze/ci/build_params.py b/dev/breeze/src/airflow_breeze/ci/build_params.py new file mode 100644 index 00000000000..7fc00b0c25d --- /dev/null +++ b/dev/breeze/src/airflow_breeze/ci/build_params.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.breeze import get_airflow_sources_root +from airflow_breeze.utils import run_command + + +@dataclass +class BuildParams: + # To construct ci_image_name + python_version: str = "3.7" + airflow_branch: str = AIRFLOW_BRANCH + build_id: int = 0 + # To construct docker cache ci directive + docker_cache: str = "pulled" + airflow_extras: str = "devel_ci" + additional_airflow_extras: str = "" + additional_python_deps: str = "" + # To construct ci_image_name + tag: str = "latest" + # To construct airflow_image_repository + github_repository: str = "apache/airflow" + constraints_github_repository: str = "apache/airflow" + # Not sure if defaultConstraintsBranch and airflow_constraints_reference are different + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH + airflow_constraints: str = "constraints-source-providers" + airflow_constraints_reference: Optional[str] = "constraints-main" + airflow_constraints_location: Optional[str] = "" + airflow_pre_cached_pip_packages: str = "true" + dev_apt_command: str = "" + dev_apt_deps: str = "" + additional_dev_apt_command: str = "" + additional_dev_apt_deps: str = "" + additional_dev_apt_env: str = "" + runtime_apt_command: str = "" + runtime_apt_deps: str = "" + additional_runtime_apt_command: str = "" + additional_runtime_apt_deps: str = "" + additional_runtime_apt_env: str = "" + upgrade_to_newer_dependencies: str = "true" + + @property + def airflow_image_name(self): + image = f'ghcr.io/{self.github_repository.lower()}' + return image + + @property + def airflow_ci_image_name(self): + """Construct CI image link""" + image = f'{self.airflow_image_name}/{self.airflow_branch}/ci/python{self.python_version}' + return image if not self.tag else image + f":{self.tag}" + + @property + def airflow_image_repository(self): + return f'https://github.com/{self.github_repository}' + + @property + def python_base_image(self): + """Construct Python Base Image""" + # ghcr.io/apache/airflow/main/python:3.8-slim-buster + return f'{self.airflow_image_name}/{self.airflow_branch}/python:{self.python_version}-slim-buster' + + @property + def airflow_ci_local_manifest_image(self): + """Construct CI Local Manifest Image""" + return f'local-airflow-ci-manifest/{self.airflow_branch}/python{self.python_version}' + + @property + def airflow_ci_remote_manifest_image(self): + """Construct CI Remote Manifest Image""" + return f'{self.airflow_ci_image_name}/{self.airflow_branch}/ci-manifest//python:{self.python_version}' + + @property + def airflow_image_date_created(self): + # 2021-12-18T15:19:25Z '%Y-%m-%dT%H:%M:%SZ' + # Set date in above format and return + now = datetime.now() + return now.strftime("%Y-%m-%dT%H:%M:%SZ") + + @property + def commit_sha(self): + output = run_command(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True) + return output.stdout.strip() + + @property + def docker_cache_ci_directive(self) -> List: + docker_cache_ci_directive = [] + if self.docker_cache == "pulled": + docker_cache_ci_directive.append("--cache-from") + docker_cache_ci_directive.append(self.airflow_ci_image_name) + elif self.docker_cache == "disabled": + docker_cache_ci_directive.append("--no-cache") + else: + pass + return docker_cache_ci_directive + + @property + def airflow_version(self): + airflow_setup_file = Path(get_airflow_sources_root()) / 'setup.py' + with open(airflow_setup_file) as setup_file: + for line in setup_file.readlines(): + if "version =" in line: + return line.split()[2][1:-1] diff --git a/dev/breeze/src/airflow_breeze/console.py b/dev/breeze/src/airflow_breeze/console.py new file mode 100644 index 00000000000..175fb7014ea --- /dev/null +++ b/dev/breeze/src/airflow_breeze/console.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from rich.console import Console +from rich.theme import Theme + +custom_theme = Theme({"info": "blue", "warning": "magenta", "error": "red"}) +console = Console(force_terminal=True, color_system="standard", width=180, theme=custom_theme) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py new file mode 100644 index 00000000000..77df24137e5 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +AIRFLOW_SOURCES = "" +DEFAULT_PYTHON_MAJOR_MINOR_VERSION = 3.7 +BUILD_CACHE_DIR = "${AIRFLOW_SOURCES}/.build" + +FORCE_PULL_IMAGES = False +CHECK_IF_BASE_PYTHON_IMAGE_UPDATED = False +FORCE_BUILD_IMAGES = False +LAST_FORCE_ANSWER_FILE = f"{BUILD_CACHE_DIR}/last_force_answer.sh" +FORCE_ANSWER_TO_QUESTION = "" +SKIP_CHECK_REMOTE_IMAGE = False +PUSH_PYTHON_BASE_IMAGE = False + +ALLOWED_PYTHON_MAJOR_MINOR_VERSION = ['3.6', '3.7', '3.8', '3.9'] +ALLOWED_BACKENDS = ['sqlite', 'mysql', 'postgres', 'mssql'] +ALLOWED_STATIC_CHECKS = [ + "all", + "airflow-config-yaml", + "airflow-providers-available", + "airflow-provider-yaml-files-ok", + "base-operator", + "bats-tests", + "bats-in-container-tests", + "black", + "blacken-docs", + "boring-cyborg", + "build", + "build-providers-dependencies", + "chart-schema-lint", + "capitalized-breeze", + "changelog-duplicates", + "check-apache-license", + "check-builtin-literals", + "check-executables-have-shebangs", + "check-extras-order", + "check-hooks-apply", + "check-integrations", + "check-merge-conflict", + "check-xml", + "daysago-import-check", + "debug-statements", + "detect-private-key", + "doctoc", + "dont-use-safe-filter", + "end-of-file-fixer", + "fix-encoding-pragma", + "flake8", + "flynt", + "codespell", + "forbid-tabs", + "helm-lint", + "identity", + "incorrect-use-of-LoggingMixin", + "insert-license", + "isort", + "json-schema", + "language-matters", + "lint-dockerfile", + "lint-openapi", + "markdownlint", + "mermaid", + "mixed-line-ending", + "mypy", + "mypy-helm", + "no-providers-in-core-examples", + "no-relative-imports", + "pre-commit-descriptions", + "pre-commit-hook-names", + "pretty-format-json", + "provide-create-sessions", + "providers-changelogs", + "providers-init-file", + "providers-subpackages-init-file", + "provider-yamls", + "pydevd", + "pydocstyle", + "python-no-log-warn", + "pyupgrade", + "restrict-start_date", + "rst-backticks", + "setup-order", + "setup-extra-packages", + "shellcheck", + "sort-in-the-wild", + "sort-spelling-wordlist", + "stylelint", + "trailing-whitespace", + "ui-lint", + "update-breeze-file", + "update-extras", + "update-local-yml-file", + "update-setup-cfg-file", + "update-versions", + "verify-db-migrations-documented", + "version-sync", + "www-lint", + "yamllint", + "yesqa", +] +ALLOWED_INTEGRATIONS = [ + 'cassandra', + 'kerberos', + 'mongo', + 'openldap', + 'pinot', + 'rabbitmq', + 'redis', + 'statsd', + 'trino', + 'all', +] +ALLOWED_KUBERNETES_MODES = ['image'] +ALLOWED_KUBERNETES_VERSIONS = ['v1.21.1', 'v1.20.2'] +ALLOWED_KIND_VERSIONS = ['v0.11.1'] +ALLOWED_HELM_VERSIONS = ['v3.6.3'] +ALLOWED_EXECUTORS = ['KubernetesExecutor', 'CeleryExecutor', 'LocalExecutor', 'CeleryKubernetesExecutor'] +ALLOWED_KIND_OPERATIONS = ['start', 'stop', 'restart', 'status', 'deploy', 'test', 'shell', 'k9s'] +ALLOWED_INSTALL_AIRFLOW_VERSIONS = ['2.0.2', '2.0.1', '2.0.0', 'wheel', 'sdist'] +ALLOWED_GENERATE_CONSTRAINTS_MODES = ['source-providers', 'pypi-providers', 'no-providers'] +ALLOWED_POSTGRES_VERSIONS = ['10', '11', '12', '13'] +ALLOWED_MYSQL_VERSIONS = ['5.7', '8'] +ALLOWED_MSSQL_VERSIONS = ['2017-latest', '2019-latest'] +ALLOWED_TEST_TYPES = [ + 'All', + 'Always', + 'Core', + 'Providers', + 'API', + 'CLI', + 'Integration', + 'Other', + 'WWW', + 'Postgres', + 'MySQL', + 'Helm', + 'Quarantined', +] +ALLOWED_PACKAGE_FORMATS = ['both', 'sdist', 'wheel'] +ALLOWED_USE_AIRFLOW_VERSION = ['.', 'apache-airflow'] + +PARAM_NAME_DESCRIPTION = { + "BACKEND": "backend", + "MYSQL_VERSION": "Mysql version", + "KUBERNETES_MODE": "Kubernetes mode", + "KUBERNETES_VERSION": "Kubernetes version", + "KIND_VERSION": "KinD version", + "HELM_VERSION": "Helm version", + "EXECUTOR": "Executors", + "POSTGRES_VERSION": "Postgres version", + "MSSQL_VERSION": "MSSql version", +} + +PARAM_NAME_FLAG = { + "BACKEND": "--backend", + "MYSQL_VERSION": "--mysql-version", + "KUBERNETES_MODE": "--kubernetes-mode", + "KUBERNETES_VERSION": "--kubernetes-version", + "KIND_VERSION": "--kind-version", + "HELM_VERSION": "--helm-version", + "EXECUTOR": "--executor", + "POSTGRES_VERSION": "--postgres-version", + "MSSQL_VERSION": "--mssql-version", +} diff --git a/dev/breeze/src/airflow_breeze/utils.py b/dev/breeze/src/airflow_breeze/utils.py new file mode 100644 index 00000000000..5ccf01ac343 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import hashlib +import shlex +import subprocess +from typing import Dict, List + +from airflow_breeze.console import console + + +def run_command(cmd: List[str], *, check: bool = True, verbose: bool = False, **kwargs): + if verbose: + console.print(f"[blue]$ {' '.join(shlex.quote(c) for c in cmd)}") + try: + return subprocess.run(cmd, check=check, **kwargs) + except subprocess.CalledProcessError as ex: + print("========================= OUTPUT start ============================") + print(ex.stderr) + print(ex.stdout) + print("========================= OUTPUT end ============================") + raise + + +def generate_md5(filename, file_size: int = 65536): + hash_md5 = hashlib.md5() + with open(filename, "rb") as f: + for file_chunk in iter(lambda: f.read(file_size), b""): + hash_md5.update(file_chunk) + return hash_md5.hexdigest() + + +def filter_out_none(**kwargs) -> Dict: + for key in list(kwargs): + if kwargs[key] is None: + kwargs.pop(key) + return kwargs