From 5005ff2735b0bb80dc913f2070d3a507156a9394 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 1 Jun 2022 01:55:31 +0200 Subject: [PATCH] Re-add --force-build flag (#24061) After #24052 we also need to add --force-build flag as for Python 3.7 rebuilding CI cache would have been silently ignored as no image building would be needed --- .github/workflows/ci.yml | 1 + .../commands/ci_image_commands.py | 2 + dev/refresh_images.sh | 1 + images/breeze/output-build-image.svg | 137 +++++++++--------- images/breeze/output-commands-hash.txt | 2 +- 5 files changed, 74 insertions(+), 69 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 781cdbb08edd4..05781d33d7cc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1712,6 +1712,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: > breeze build-image --prepare-buildx-cache + --force-build --platform ${{ matrix.platform }} env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 325808b7047b5..879b45ea815e5 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -44,6 +44,7 @@ option_docker_cache, option_dry_run, option_empty_image, + option_force_build, option_github_repository, option_github_token, option_github_username, @@ -216,6 +217,7 @@ @option_additional_runtime_apt_command @option_dev_apt_command @option_dev_apt_deps +@option_force_build @option_runtime_apt_command @option_runtime_apt_deps @option_airflow_constraints_mode_ci diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 333b3b87ebdac..ed5591585aeb0 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -28,6 +28,7 @@ breeze self-upgrade --force breeze build-image \ --build-multiple-images \ --prepare-buildx-cache \ + --force-build \ --platform linux/amd64,linux/arm64 \ --verbose diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 6b555c27f3428..b779367062c37 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ - + - Command: build-image + Command: build-image -                                                                                                                          - Usage: breeze build-image [OPTIONS]                                                                                     -                                                                                                                         - Build CI image. Include building multiple images for all python versions (sequentially).                                -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ -│                                       Useful when you build or pull image with --image-tag.                          │ -│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ -│                                       [default: registry]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ -│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ -│                                      [default: constraints-source-providers]                                         │ -│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ -│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ -│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ -│  --push-image              Push image after building it.                                                             │ -│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ -│  --prepare-buildx-cache    Prepares build cache (this is done as separate per-platform steps instead of building     │ -│                            the image).                                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-image [OPTIONS]                                                                                     +                                                                                                                         + Build CI image. Include building multiple images for all python versions (sequentially).                                +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --tag-as-latest                      Tags the image as latest and update checksum of all files after pulling.       │ +│                                       Useful when you build or pull image with --image-tag.                          │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +│  --force-build                        Force image build no matter if it is determined as needed.                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ +│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ +│                                      [default: constraints-source-providers]                                         │ +│  --airflow-constraints-reference     Constraint reference to use when building the image. (TEXT)                     │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --push-image              Push image after building it.                                                             │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +│  --prepare-buildx-cache    Prepares build cache (this is done as separate per-platform steps instead of building     │ +│                            the image).                                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 60451263b17ab..0a73056042294 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -4b9bda7e1b32565b73e604c969dbdc86 +dfa197bfca0465db8174b262c4b9850b