From 18ea01cef2b92fe820ceaa33be7b44f9f576aad4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 5 Nov 2024 12:01:30 +0100 Subject: [PATCH] Fix reproducibility of prepared provider packages (fix flit frontend) (#43683) After some checks it turned out that reproducibility of produced packages depends not only on the build backend configured for the project but also on the build front-end used - because frontend is the one to modify meta-data in prepared packages - including the build tool used, it's version and metadata version supported by the front-end. That's why in order to maintain reproducibility for anyone who builds the packages, we have to pin not only the build backend in pyproject.toml (flit-core) but also build fronted used (flit). Since package preparation is done with breeze, we can do it by pinning flit (and just in case also flit-core) so that anyone who builds specific version of the package will use exactly the same flit as the person who built the original packages. This way we will avoid reproducibility problems experienced with 1.5.0 release of FAB. --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 1 - dev/breeze/README.md | 2 +- dev/breeze/pyproject.toml | 15 ++++++++++++++- .../commands/release_candidate_command.py | 3 --- .../commands/release_management_commands.py | 8 +------- .../templates/pyproject_TEMPLATE.toml.jinja2 | 3 +-- .../src/airflow_breeze/utils/python_versions.py | 14 -------------- .../src/airflow_breeze/utils/reproducible.py | 3 --- dev/breeze/uv.lock | 4 +++- 9 files changed, 20 insertions(+), 33 deletions(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index a96d07d9441b7..49631dd858764 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -349,7 +349,6 @@ export AIRFLOW_REPO_ROOT=$(pwd -P) rm -rf ${AIRFLOW_REPO_ROOT}/dist/* ``` - * Release candidate packages: ```shell script diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 9a3f06066eb22..70ad4076750ad 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -136,6 +136,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: f13c42703e0a262d9f3c1bee608ff32c368be4c6a11f150a2f95809938641f5ec07904d5cc2e3944dfe4d206dc52846f8b81193fc279a333ff898dd033e07be4 +Package config hash: 5d32e2c150de1cc22d522f94d86500c739f434439ae064e35791ac795932a1f21509c3d0fcf9f2351e7901a32601190bd4cb640799620a44d0ff6d9244aef57c --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index a52b2f9775853..f5b4b9b98c4e9 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -47,7 +47,20 @@ dependencies = [ "black>=23.11.0", "click>=8.1.7", "filelock>=3.13.0", - "flit>=3.5.0", + # + # We pin flit in order to make sure reproducibility of provider packages is maintained + # It turns out that when packages are prepared metadata version in the produced packages + # is taken from the front-end not from the backend, so in order to make sure that the + # packages are reproducible, we should pin both backend in "build-system" and frontend in + # "dependencies" of the environment that is used to build the packages. + # + # TODO(potiuk): automate bumping the version of flit in breeze and sync it with + # the version in the template for provider packages with pre-commit also add instructions in + # the source packages explaining that reproducibility can only be achieved by using the same + # version of flit front-end to build the package + # + "flit==3.10.1", + "flit-core==3.10.1", "google-api-python-client>=2.142.0", "google-auth-httplib2>=0.2.0", "google-auth-oauthlib>=1.2.0", diff --git a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py index 7fcba61f10662..d5e2bd6f9d3b1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py @@ -27,7 +27,6 @@ from airflow_breeze.utils.confirm import confirm_action from airflow_breeze.utils.console import console_print from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DIST_DIR, OUT_DIR -from airflow_breeze.utils.python_versions import check_python_version from airflow_breeze.utils.reproducible import get_source_date_epoch, repack_deterministically from airflow_breeze.utils.run_utils import run_command @@ -341,7 +340,6 @@ def remove_old_releases(version, repo_root): "--version", required=True, help="The release candidate version e.g. 2.4.3rc1", envvar="VERSION" ) def prepare_airflow_tarball(version: str): - check_python_version() from packaging.version import Version airflow_version = Version(version) @@ -367,7 +365,6 @@ def prepare_airflow_tarball(version: str): ) @option_answer def publish_release_candidate(version, previous_version, github_token): - check_python_version() from packaging.version import Version airflow_version = Version(version) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 61a26e9993f88..6955d38e0eb78 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -151,7 +151,7 @@ generate_providers_metadata_for_package, get_related_providers, ) -from airflow_breeze.utils.python_versions import check_python_version, get_python_version_list +from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.reproducible import get_source_date_epoch, repack_deterministically from airflow_breeze.utils.run_utils import ( run_command, @@ -530,7 +530,6 @@ def prepare_airflow_packages( version_suffix_for_pypi: str, use_local_hatch: bool, ): - check_python_version() perform_environment_checks() fix_ownership_using_docker() cleanup_python_generated_files() @@ -576,7 +575,6 @@ def prepare_airflow_task_sdk_packages( package_format: str, use_local_hatch: bool, ): - check_python_version() perform_environment_checks() fix_ownership_using_docker() cleanup_python_generated_files() @@ -907,7 +905,6 @@ def prepare_provider_packages( skip_tag_check: bool, version_suffix_for_pypi: str, ): - check_python_version(release_provider_packages=True) perform_environment_checks() fix_ownership_using_docker() cleanup_python_generated_files() @@ -3182,7 +3179,6 @@ def prepare_helm_chart_tarball( ) -> None: import yaml - check_python_version() chart_yaml_file_content = CHART_YAML_FILE.read_text() chart_yaml_dict = yaml.safe_load(chart_yaml_file_content) version_in_chart = chart_yaml_dict["version"] @@ -3324,8 +3320,6 @@ def prepare_helm_chart_tarball( @option_dry_run @option_verbose def prepare_helm_chart_package(sign_email: str): - check_python_version() - import yaml from airflow_breeze.utils.kubernetes_utils import ( diff --git a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 index 363b02e6136c7..9849bfe8bc527 100644 --- a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 +++ b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 @@ -39,9 +39,8 @@ # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY -# [build-system] -requires = ["flit_core==3.10.0"] +requires = ["flit_core==3.10.1"] build-backend = "flit_core.buildapi" [project] diff --git a/dev/breeze/src/airflow_breeze/utils/python_versions.py b/dev/breeze/src/airflow_breeze/utils/python_versions.py index 4f5a7a00bb588..3ac3f8be30ff3 100644 --- a/dev/breeze/src/airflow_breeze/utils/python_versions.py +++ b/dev/breeze/src/airflow_breeze/utils/python_versions.py @@ -43,17 +43,3 @@ def get_python_version_list(python_versions: str) -> list[str]: ) sys.exit(1) return python_version_list - - -def check_python_version(release_provider_packages: bool = False): - if not sys.version_info < (3, 12) and release_provider_packages: - get_console().print("[error]Python 3.12 is not supported.\n") - get_console().print( - "[warning]Please reinstall Breeze using Python 3.9 - 3.11 environment because not all " - "provider packages support Python 3.12 yet.[/]\n\n" - "If you are using uv:\n\n" - " uv tool install --force --reinstall --python 3.9 -e ./dev/breeze\n\n" - "If you are using pipx:\n\n" - " pipx install --python $(which python3.9) --force -e ./dev/breeze\n" - ) - sys.exit(1) diff --git a/dev/breeze/src/airflow_breeze/utils/reproducible.py b/dev/breeze/src/airflow_breeze/utils/reproducible.py index 1429333d64152..cf4005d9ddd10 100644 --- a/dev/breeze/src/airflow_breeze/utils/reproducible.py +++ b/dev/breeze/src/airflow_breeze/utils/reproducible.py @@ -43,7 +43,6 @@ from subprocess import CalledProcessError, CompletedProcess from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, OUT_DIR, REPRODUCIBLE_DIR -from airflow_breeze.utils.python_versions import check_python_version from airflow_breeze.utils.run_utils import run_command @@ -91,7 +90,6 @@ def reset(tarinfo): tarinfo.mtime = timestamp return tarinfo - check_python_version() OUT_DIR.mkdir(exist_ok=True) shutil.rmtree(REPRODUCIBLE_DIR, ignore_errors=True) REPRODUCIBLE_DIR.mkdir(exist_ok=True) @@ -149,7 +147,6 @@ def reset(tarinfo): def main(): - check_python_version() parser = ArgumentParser() parser.add_argument("-a", "--archive", help="archive to repack") parser.add_argument("-o", "--out", help="archive destination") diff --git a/dev/breeze/uv.lock b/dev/breeze/uv.lock index bcfc8c4ce785f..0aed5ae9cd12e 100644 --- a/dev/breeze/uv.lock +++ b/dev/breeze/uv.lock @@ -29,6 +29,7 @@ dependencies = [ { name = "click" }, { name = "filelock" }, { name = "flit" }, + { name = "flit-core" }, { name = "gitpython" }, { name = "google-api-python-client" }, { name = "google-auth-httplib2" }, @@ -60,7 +61,8 @@ requires-dist = [ { name = "black", specifier = ">=23.11.0" }, { name = "click", specifier = ">=8.1.7" }, { name = "filelock", specifier = ">=3.13.0" }, - { name = "flit", specifier = ">=3.5.0" }, + { name = "flit", specifier = "==3.10.1" }, + { name = "flit-core", specifier = "==3.10.1" }, { name = "gitpython", specifier = ">=3.1.40" }, { name = "google-api-python-client", specifier = ">=2.142.0" }, { name = "google-auth-httplib2", specifier = ">=0.2.0" },