From fcdd872e8a81553d0956e64348cad4e5d8aa3b31 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 17 Jun 2022 12:57:10 +0200 Subject: [PATCH] Add verification steps when releasing the images. After the images are pushed in CI we are running the verification of the AMD image now. This cannot be really done during building and pushing the image, because we are using multi-platform images using remote builders so the image is not even available locally, so we need to actually pull the images after they are built in order to verify them. This PR adds those features: * ability to pull images for verification with --pull-flag * ability to verify slim images (regular tests are skipped and we only expect the preinstalled providers to be available * the steps to verify the images (both regular and slim) are added to the workflow --- .github/workflows/release_dockerhub_image.yml | 21 +++- .../commands/ci_image_commands.py | 12 +- .../commands/production_image_commands.py | 14 +++ .../airflow_breeze/utils/common_options.py | 6 + dev/breeze/src/airflow_breeze/utils/image.py | 1 + .../src/airflow_breeze/utils/run_tests.py | 4 +- docker_tests/test_prod_image.py | 10 +- images/breeze/output-commands-hash.txt | 1 + images/breeze/output-verify-image.svg | 108 +++++++++-------- images/breeze/output-verify-prod-image.svg | 112 ++++++++++-------- 10 files changed, 177 insertions(+), 112 deletions(-) create mode 100644 images/breeze/output-commands-hash.txt diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index a6c3e86ddd814..cd5fe6e9ce08d 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -119,12 +119,27 @@ jobs: ${{ needs.build-info.outputs.skipLatest }} ${{ needs.build-info.outputs.limitPlatform }} --limit-python ${{ matrix.python-version }} --slim-images - - name: "Docker logout" - run: docker logout - if: always() - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() && github.repository == 'apache/airflow' + - name: > + Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} + run: > + breeze verify-prod-image + --pull-image + --image-name + ${{github.repository}}:${{github.event.inputs.airflowVersion}}-python${{matrix.python-version}} + - name: > + Verify slim AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} + run: > + breeze verify-prod-image + --pull-image + --slim-image + --image-name + ${{github.repository}}:slim-${{github.event.inputs.airflowVersion}}-python${{matrix.python-version}} + - name: "Docker logout" + run: docker logout + if: always() - name: "Fix ownership" run: breeze fix-ownership if: always() diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index c68b3839df386..9deac3a276817 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -54,6 +54,7 @@ option_parallelism, option_platform, option_prepare_buildx_cache, + option_pull_image, option_push_image, option_python, option_python_versions, @@ -181,6 +182,7 @@ "--image-name", "--python", "--image-tag", + "--pull-image", ], } ], @@ -270,7 +272,7 @@ def run_build(ci_image_params: BuildCiParams) -> None: @option_image_tag @option_tag_as_latest @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) -def pull_image( +def pull_ci_image( verbose: bool, dry_run: bool, python: str, @@ -338,14 +340,16 @@ def pull_image( @option_github_repository @option_image_tag @option_image_name +@option_pull_image @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) -def verify_image( +def verify_ci_image( verbose: bool, dry_run: bool, python: str, github_repository: str, image_name: str, image_tag: str, + pull_image: bool, extra_pytest_args: Tuple, ): """Verify CI image.""" @@ -353,12 +357,16 @@ def verify_image( if image_name is None: build_params = BuildCiParams(python=python, image_tag=image_tag, github_repository=github_repository) image_name = build_params.airflow_image_name_with_tag + if pull_image: + command_to_run = ["docker", "pull", image_name] + run_command(command_to_run, verbose=verbose, dry_run=dry_run, check=True) get_console().print(f"[info]Verifying CI image: {image_name}[/]") return_code, info = verify_an_image( image_name=image_name, verbose=verbose, dry_run=dry_run, image_type='CI', + slim_image=False, extra_pytest_args=extra_pytest_args, ) sys.exit(return_code) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 8ac7aacab0f2f..0b5ed2e68b06d 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -52,6 +52,7 @@ option_parallelism, option_platform, option_prepare_buildx_cache, + option_pull_image, option_push_image, option_python, option_python_versions, @@ -184,6 +185,7 @@ "--image-name", "--python", "--image-tag", + "--pull-image", ], } ], @@ -382,6 +384,12 @@ def pull_prod_image( @option_github_repository @option_image_tag @option_image_name +@option_pull_image +@click.option( + '--slim-image', + help='The image to verify is slim and non-slim tests should be skipped.', + is_flag=True, +) @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED) def verify_prod_image( verbose: bool, @@ -390,6 +398,8 @@ def verify_prod_image( github_repository: str, image_name: str, image_tag: str, + pull_image: bool, + slim_image: bool, extra_pytest_args: Tuple, ): """Verify Production image.""" @@ -399,6 +409,9 @@ def verify_prod_image( python=python, image_tag=image_tag, github_repository=github_repository ) image_name = build_params.airflow_image_name_with_tag + if pull_image: + command_to_run = ["docker", "pull", image_name] + run_command(command_to_run, verbose=verbose, dry_run=dry_run, check=True) get_console().print(f"[info]Verifying PROD image: {image_name}[/]") return_code, info = verify_an_image( image_name=image_name, @@ -406,6 +419,7 @@ def verify_prod_image( dry_run=dry_run, image_type='PROD', extra_pytest_args=extra_pytest_args, + slim_image=slim_image, ) sys.exit(return_code) diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 628bb8197972b..0bc4d5af9fc86 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -434,3 +434,9 @@ show_default=True, help='Mode of constraints for PROD image building', ) +option_pull_image = click.option( + '--pull-image', + help="Pull image is missing before attempting to verify it.", + is_flag=True, + envvar='PULL_IMAGE', +) diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py index 36c40cf05a3e7..57ccc2e3a129b 100644 --- a/dev/breeze/src/airflow_breeze/utils/image.py +++ b/dev/breeze/src/airflow_breeze/utils/image.py @@ -201,6 +201,7 @@ def run_pull_and_verify_image( image_type=image_params.image_type, dry_run=dry_run, verbose=verbose, + slim_image=False, extra_pytest_args=extra_pytest_args, ) diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 63c507afe4c77..6cda9efa1e798 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -26,7 +26,7 @@ def verify_an_image( - image_name: str, image_type: str, dry_run: bool, verbose: bool, extra_pytest_args: Tuple + image_name: str, image_type: str, dry_run: bool, verbose: bool, slim_image: bool, extra_pytest_args: Tuple ) -> Tuple[int, str]: command_result = run_command( ["docker", "inspect", image_name], dry_run=dry_run, verbose=verbose, check=False, stdout=DEVNULL @@ -43,6 +43,8 @@ def verify_an_image( test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" / "test_ci_image.py" env = os.environ.copy() env['DOCKER_IMAGE'] = image_name + if slim_image: + env['TEST_SLIM_IMAGE'] = 'true' command_result = run_command( [sys.executable, "-m", "pytest", str(test_path), *pytest_args, *extra_pytest_args], dry_run=dry_run, diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py index 6e47cb0a9dbcc..aae374c4d02da 100644 --- a/docker_tests/test_prod_image.py +++ b/docker_tests/test_prod_image.py @@ -16,6 +16,7 @@ # under the License. import json +import os import subprocess import tempfile from pathlib import Path @@ -30,6 +31,7 @@ run_bash_in_docker, run_python_in_docker, ) +from setup import PREINSTALLED_PROVIDERS INSTALLED_PROVIDER_PATH = SOURCE_ROOT / "scripts" / "ci" / "installed_providers.txt" @@ -74,8 +76,11 @@ def test_bash_version(self): class TestPythonPackages: def test_required_providers_are_installed(self): - lines = (d.strip() for d in INSTALLED_PROVIDER_PATH.read_text().splitlines()) - lines = (d for d in lines) + if os.environ.get("TEST_SLIM_IMAGE"): + lines = PREINSTALLED_PROVIDERS + else: + lines = (d.strip() for d in INSTALLED_PROVIDER_PATH.read_text().splitlines()) + lines = (d for d in lines) packages_to_install = {f"apache-airflow-providers-{d.replace('.', '-')}" for d in lines} assert len(packages_to_install) != 0 @@ -163,6 +168,7 @@ def test_pip_dependencies_conflict(self): "virtualenv": ["virtualenv"], } + @pytest.mark.skipif(os.environ.get("TEST_SLIM_IMAGE") == "true", reason="Skipped with slim image") @pytest.mark.parametrize("package_name,import_names", PACKAGE_IMPORTS.items()) def test_check_dependencies_imports(self, package_name, import_names): run_python_in_docker(f"import {','.join(import_names)}") diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt new file mode 100644 index 0000000000000..70ac9b305de2f --- /dev/null +++ b/images/breeze/output-commands-hash.txt @@ -0,0 +1 @@ +8b4116c1808c84d491961283a4ddbec2 diff --git a/images/breeze/output-verify-image.svg b/images/breeze/output-verify-image.svg index 9c6d1a641fc52..e2dfddcbb07c9 100644 --- a/images/breeze/output-verify-image.svg +++ b/images/breeze/output-verify-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: verify-image + Command: verify-image - + - - -Usage: breeze verify-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify CI image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ -│to run shell or tests)                                                                            │ -│(TEXT)                                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze verify-image [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify CI image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ +│to run shell or tests)                                                                            │ +│(TEXT)                                                                                            │ +│--pull-imagePull image is missing before attempting to verify it.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-verify-prod-image.svg b/images/breeze/output-verify-prod-image.svg index 48b61a30826b5..31c7b66b725dc 100644 --- a/images/breeze/output-verify-prod-image.svg +++ b/images/breeze/output-verify-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: verify-prod-image + Command: verify-prod-image - + - - -Usage: breeze verify-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify Production image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ -│to run shell or tests)                                                                            │ -│(TEXT)                                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze verify-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify Production image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--image-tag-tTag of the image which is used to pull or run the image (implies --mount-sources=skip when using  │ +│to run shell or tests)                                                                            │ +│(TEXT)                                                                                            │ +│--pull-imagePull image is missing before attempting to verify it.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--slim-imageThe image to verify is slim and non-slim tests should be skipped.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯