From b74b58b1089a149d212da097cec63706b2240581 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 29 May 2022 10:33:40 +0200 Subject: [PATCH] Add better diagnostics capabilities for pre-commits run via CI image (#23980) The pre-commits that require CI image run docker command under the hood that is highly optimized for performance (only mounts files that are necessary to be mounted) - in order to improve performance on Mac OS and make sure that artifacts are not left in the source code of Airflow. However that makes the command slightly more difficult to debug because they generate dynamically the docker command used, including the volumens that should be mounted when the docker command is run. This PR adds better diagnostics to the pre-commit scripts allowing VERBOSE="true" and DRY_RUN="true" variables that can help with diagnosing problems such as running the scripts on WSL2. It also fixes a few documentation bugs that have been missed after changing names of the image-related static checks and thanks to separating the common code to utility function it allows to set SKIP_IMAGE_PRE_COMMITS variable to true which will skip running all pre-commit checks that require breeze image to be available locally. (cherry picked from commit 5af83ce736b92f651f75d5d00c2043be59ab91f5) --- .github/workflows/ci.yml | 11 +- STATIC_CODE_CHECKS.rst | 52 +++++--- .../commands/developer_commands.py | 2 +- .../src/airflow_breeze/utils/path_utils.py | 8 +- .../src/airflow_breeze/utils/run_utils.py | 30 +++++ images/breeze/output-commands-hash.txt | 2 +- images/breeze/output-static-checks.svg | 120 +++++++++--------- .../pre_commit_check_pre_commit_hooks.py | 2 +- scripts/ci/pre_commit/pre_commit_flake8.py | 28 ++-- .../pre_commit_migration_reference.py | 34 ++--- scripts/ci/pre_commit/pre_commit_mypy.py | 28 ++-- scripts/ci/pre_commit/pre_commit_ui_lint.py | 36 +++--- scripts/ci/pre_commit/pre_commit_www_lint.py | 35 ++--- 13 files changed, 215 insertions(+), 173 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5488b765be2c3..e8de7a7f9b184 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -637,9 +637,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Static checks" run: breeze static-checks --all-files --show-diff-on-failure --color always env: - VERBOSE: false + VERBOSE: "false" SKIP: "identity" - COLUMNS: 250 + COLUMNS: "250" - name: "Fix ownership" run: breeze fix-ownership if: always() @@ -654,8 +654,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" needs: [build-info] env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - SKIP: "build,run-mypy,run-flake8,lint-javascript,update-migration-references,identity" - MOUNT_SELECTED_LOCAL_SOURCES: "true" if: needs.build-info.outputs.basic-checks-only == 'true' steps: - name: Cleanup repo @@ -695,7 +693,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze static-checks --all-files --show-diff-on-failure --color always --commit-ref "${{ github.sha }}" env: - VERBOSE: false + VERBOSE: "false" + SKIP_IMAGE_PRE_COMMITS: "true" + SKIP: "identity" + COLUMNS: "250" - name: "Fix ownership" run: breeze fix-ownership if: always() diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 71f132e7d5d10..e2b57eae6c69b 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -111,8 +111,8 @@ For details on advanced usage of the install method, use: Available pre-commit checks ........................... -This table lists pre-commit hooks used by Airflow. The ``Breeze`` column indicates which hooks -require Breeze Docker images to be installed locally. +This table lists pre-commit hooks used by Airflow. The ``Image`` column indicates which hooks +require Breeze Docker image to be build locally. .. note:: Disabling particular checks @@ -123,6 +123,10 @@ require Breeze Docker images to be installed locally. ``export SKIP=run-flake8,run-mypy``. You can also add this to your ``.bashrc`` or ``.zshrc`` if you do not want to set it manually every time you enter the terminal. + In case you do not have breeze image configured locally, you can also disable all checks that require + the image by setting ``SKIP_IMAGE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically + when run locally (note that those checks will anyway run in CI). + .. BEGIN AUTO-GENERATED STATIC CHECK LIST +--------------------------------------------------------+------------------------------------------------------------------+---------+ @@ -242,7 +246,7 @@ require Breeze Docker images to be installed locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | lint-helm-chart | Lint Helm Chart | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| lint-javascript | * ESLint against airflow/ui | | +| lint-javascript | * ESLint against airflow/ui | * | | | * ESLint against current UI JavaScript files | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ | lint-json-schema | * Lint JSON Schema files with JSON Schema | | @@ -269,9 +273,9 @@ require Breeze Docker images to be installed locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | rst-backticks | Check if RST files use double backticks for code | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| run-flake8 | Run flake8 | | +| run-flake8 | Run flake8 | * | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| run-mypy | * Run mypy for dev | | +| run-mypy | * Run mypy for dev | * | | | * Run mypy for core | | | | * Run mypy for providers | | | | * Run mypy for /docs/ folder | | @@ -294,7 +298,7 @@ require Breeze Docker images to be installed locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | update-local-yml-file | Update mounts in the local yml file | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| update-migration-references | Update migration ref doc | | +| update-migration-references | Update migration ref doc | * | +--------------------------------------------------------+------------------------------------------------------------------+---------+ | update-providers-dependencies | Update cross-dependencies for providers packages | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ @@ -387,49 +391,63 @@ Run the ``mypy`` check for the currently staged changes: .. code-block:: bash - breeze static-check --type run-mypy + breeze static-checks --type run-mypy Run the ``mypy`` check for all files: .. code-block:: bash - breeze static-check --type run-mypy --all-files + breeze static-checks --type run-mypy --all-files Run the ``flake8`` check for the ``tests.core.py`` file with verbose output: .. code-block:: bash - breeze static-check --type run-flake8 --file tests/core.py --verbose + breeze static-checks --type run-flake8 --file tests/core.py --verbose Run the ``flake8`` check for the ``tests.core`` package with verbose output: .. code-block:: bash - breeze static-check --type run-flake8 --file tests/core/* --verbose + breeze static-checks --type run-flake8 --file tests/core/* --verbose Run all tests for the currently staged files: .. code-block:: bash - breeze static-check --type all + breeze static-checks --type all Run all tests for all files: .. code-block:: bash - breeze static-check --type all --all-files + breeze static-checks --type all --all-files Run all tests for last commit : .. code-block:: bash - breeze static-check --type all --last-commit + breeze static-checks -type all --last-commit + +Debugging pre-commit check scripts requiring image +-------------------------------------------------- + +Those commits that use Breeze docker image might sometimes fail, depending on your operating system and +docker setup, so sometimes it might be required to run debugging with the commands. This is done via +two environment variables ``VERBOSE`` and ``DRY_RUN``. Setting them to "true" will respectively show the +commands to run before running them or skip running the commands. + +Note that you need to run pre-commit with --verbose command to get the output regardless of the status +of the static check (normally it will only show output on failure). + +Printing the commands while executing: + +.. code-block:: bash + VERBOSE="true" pre-commit run --verbose run-flake8 -The ``license`` check is run via a separate script and a separate Docker image containing the -Apache RAT verification tool that checks for Apache-compatibility of licenses within the codebase. -It does not take pre-commit parameters as extra arguments. +Just performing dry run: .. code-block:: bash - breeze static-check licenses + DRY_RUN="true" pre-commit run --verbose run-flake8 diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index bb1d974e7ce9e..1719bab5fc7a8 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -438,7 +438,7 @@ def build_docs( type=BetterChoice(PRE_COMMIT_LIST), multiple=True, ) -@click.option('-a', '--all-files', help="Run checks on all files.") +@click.option('-a', '--all-files', help="Run checks on all files.", is_flag=True) @click.option('-f', '--file', help="List of files to run the checks on.", type=click.Path(), multiple=True) @click.option( '-s', '--show-diff-on-failure', help="Show diff for files modified by the checks.", is_flag=True diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 49d9d3bf9ceef..759c3e09a7d6a 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -62,13 +62,7 @@ def in_help() -> bool: def skip_upgrade_check(): - return ( - in_self_upgrade() - or in_autocomplete() - or in_help() - or hasattr(sys, '_called_from_test') - or os.environ.get('SKIP_BREEZE_UPGRADE_CHECK') - ) + return in_self_upgrade() or in_autocomplete() or in_help() or hasattr(sys, '_called_from_test') def get_package_setup_metadata_hash() -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index bbfdfcb4326c4..86b84be4c076f 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -27,6 +27,7 @@ from re import match from typing import Dict, List, Mapping, Optional, Union +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import get_console @@ -375,3 +376,32 @@ def filter_out_none(**kwargs) -> dict: if kwargs[key] is None: kwargs.pop(key) return kwargs + + +def fail_if_image_missing(image: str, verbose: bool, dry_run: bool, instruction: str) -> None: + skip_image_pre_commits = os.environ.get('SKIP_IMAGE_PRE_COMMITS', "false") + if skip_image_pre_commits[0].lower() == "t": + get_console().print( + f"[info]Skipping image check as SKIP_IMAGE_PRE_COMMITS is set to {skip_image_pre_commits}[/]" + ) + sys.exit(0) + cmd_result = run_command( + ["docker", "inspect", image], stdout=subprocess.DEVNULL, check=False, verbose=verbose, dry_run=dry_run + ) + if cmd_result.returncode != 0: + print(f'[red]The image {image} is not available.[/]\n') + print(f"\n[yellow]Please run at the earliest convenience:[/]\n\n{instruction}\n\n") + sys.exit(1) + + +def get_runnable_ci_image(verbose: bool, dry_run: bool) -> str: + github_repository = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") + python_version = "3.7" + airflow_image = f"ghcr.io/{github_repository}/{AIRFLOW_BRANCH}/ci/python{python_version}" + fail_if_image_missing( + image=airflow_image, + verbose=verbose, + dry_run=dry_run, + instruction=f"breeze build-image --python {python_version}", + ) + return airflow_image diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 8a3bf0f004ed8..fe6d5cdba65af 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -e275b745e83c5e5f4fc2a54677ab7e0d +4538f69421a320083444736d334314b7 diff --git a/images/breeze/output-static-checks.svg b/images/breeze/output-static-checks.svg index d9188e5fe8863..e76f1a3f27ea3 100644 --- a/images/breeze/output-static-checks.svg +++ b/images/breeze/output-static-checks.svg @@ -19,85 +19,85 @@ font-weight: 700; } - .terminal-2026671122-matrix { + .terminal-158108476-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 26.400000000000002px; font-variant-east-asian: full-width; } - .terminal-2026671122-title { + .terminal-158108476-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2026671122-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-2026671122-r2 { fill: #c5c8c6 } -.terminal-2026671122-r3 { fill: #d0b344;font-weight: bold } -.terminal-2026671122-r4 { fill: #868887 } -.terminal-2026671122-r5 { fill: #68a0b3;font-weight: bold } -.terminal-2026671122-r6 { fill: #98a84b;font-weight: bold } -.terminal-2026671122-r7 { fill: #8d7b39 } + .terminal-158108476-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-158108476-r2 { fill: #c5c8c6 } +.terminal-158108476-r3 { fill: #d0b344;font-weight: bold } +.terminal-158108476-r4 { fill: #868887 } +.terminal-158108476-r5 { fill: #68a0b3;font-weight: bold } +.terminal-158108476-r6 { fill: #98a84b;font-weight: bold } +.terminal-158108476-r7 { fill: #8d7b39 } - Command: static-checks + Command: static-checks -                                                                                                                          - Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]...                                                               -                                                                                                                         - Run static checks.                                                                                                      -                                                                                                                         -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --type                  -t  Type(s) of the static checks to run (multiple can be added).                            │ -│                              (all | black | blacken-docs | check-airflow-2-1-compatibility |                         │ -│                              check-airflow-config-yaml-consistent | check-airflow-providers-have-extras |            │ -│                              check-apache-license-rat | check-base-operator-usage |                                  │ -│                              check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ -│                              check-builtin-literals | check-changelog-has-no-duplicates |                            │ -│                              check-daysago-import-from-utils | check-docstring-param-types |                         │ -│                              check-executables-have-shebangs | check-extra-packages-references | check-extras-order  │ -│                              | check-for-inclusive-language | check-hooks-apply |                                    │ -│                              check-incorrect-use-of-LoggingMixin | check-integrations-are-consistent |               │ -│                              check-merge-conflict | check-newsfragments-are-valid |                                  │ -│                              check-no-providers-in-core-examples | check-no-relative-imports |                       │ -│                              check-persist-credentials-disabled-in-github-workflows |                                │ -│                              check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ -│                              check-provider-yaml-valid | check-providers-init-file-missing |                         │ -│                              check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ -│                              check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ -│                              check-start-date-not-used-in-defaults | check-system-tests-present | check-xml |        │ -│                              codespell | debug-statements | detect-private-key | doctoc | end-of-file-fixer |        │ -│                              fix-encoding-pragma | flynt | forbid-tabs | identity | insert-license | isort |         │ -│                              lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart | lint-javascript |    │ -│                              lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending |                   │ -│                              pretty-format-json | pydocstyle | python-no-log-warn | pyupgrade | rst-backticks |      │ -│                              run-flake8 | run-mypy | run-shellcheck | static-check-autoflake | trailing-whitespace   │ -│                              | update-breeze-file | update-breeze-readme-config-hash | update-extras |               │ -│                              update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ -│                              update-local-yml-file | update-migration-references | update-providers-dependencies |   │ -│                              update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ -│                              update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ -│                              yamllint | yesqa)                                                                       │ -│  --file                  -f  List of files to run the checks on. (PATH)                                              │ -│  --all-files             -a  Run checks on all files. (TEXT)                                                         │ -│  --show-diff-on-failure  -s  Show diff for files modified by the checks.                                             │ -│  --last-commit           -c  Run checks for all files in last commit. Mutually exclusive with --commit-ref.          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --commit-ref         -r  Run checks for this commit reference only (can be any git commit-ish reference). Mutually  │ -│                           exclusive with --last-commit.                                                              │ -│                           (TEXT)                                                                                     │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]...                                                               +                                                                                                                         + Run static checks.                                                                                                      +                                                                                                                         +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --type                  -t  Type(s) of the static checks to run (multiple can be added).                            │ +│                              (all | black | blacken-docs | check-airflow-2-1-compatibility |                         │ +│                              check-airflow-config-yaml-consistent | check-airflow-providers-have-extras |            │ +│                              check-apache-license-rat | check-base-operator-usage |                                  │ +│                              check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ +│                              check-builtin-literals | check-changelog-has-no-duplicates |                            │ +│                              check-daysago-import-from-utils | check-docstring-param-types |                         │ +│                              check-executables-have-shebangs | check-extra-packages-references | check-extras-order  │ +│                              | check-for-inclusive-language | check-hooks-apply |                                    │ +│                              check-incorrect-use-of-LoggingMixin | check-integrations-are-consistent |               │ +│                              check-merge-conflict | check-newsfragments-are-valid |                                  │ +│                              check-no-providers-in-core-examples | check-no-relative-imports |                       │ +│                              check-persist-credentials-disabled-in-github-workflows |                                │ +│                              check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ +│                              check-provider-yaml-valid | check-providers-init-file-missing |                         │ +│                              check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ +│                              check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ +│                              check-start-date-not-used-in-defaults | check-system-tests-present | check-xml |        │ +│                              codespell | debug-statements | detect-private-key | doctoc | end-of-file-fixer |        │ +│                              fix-encoding-pragma | flynt | forbid-tabs | identity | insert-license | isort |         │ +│                              lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart | lint-javascript |    │ +│                              lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending |                   │ +│                              pretty-format-json | pydocstyle | python-no-log-warn | pyupgrade | rst-backticks |      │ +│                              run-flake8 | run-mypy | run-shellcheck | static-check-autoflake | trailing-whitespace   │ +│                              | update-breeze-file | update-breeze-readme-config-hash | update-extras |               │ +│                              update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ +│                              update-local-yml-file | update-migration-references | update-providers-dependencies |   │ +│                              update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ +│                              update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ +│                              yamllint | yesqa)                                                                       │ +│  --file                  -f  List of files to run the checks on. (PATH)                                              │ +│  --all-files             -a  Run checks on all files.                                                                │ +│  --show-diff-on-failure  -s  Show diff for files modified by the checks.                                             │ +│  --last-commit           -c  Run checks for all files in last commit. Mutually exclusive with --commit-ref.          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --commit-ref         -r  Run checks for this commit reference only (can be any git commit-ish reference). Mutually  │ +│                           exclusive with --last-commit.                                                              │ +│                           (TEXT)                                                                                     │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py index 9855091ba8085..2a5a25639b6c5 100755 --- a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py +++ b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py @@ -56,7 +56,7 @@ def get_errors_and_hooks(content: Any, max_length: int) -> Tuple[List[str], Dict else: errors.append(f"The id is missing in {hook}") continue - if hook_id == 'mypy': + if hook_id == 'run-mypy': needs_image = True if 'name' not in hook: errors.append( diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index 7adba2ffc656b..59bd356d24df0 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -30,22 +27,20 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) create_static_check_volumes() - return_code = subprocess.call( - args=[ + cmd_result = run_command( + [ "docker", "run", "-t", @@ -58,9 +53,12 @@ "BACKEND=sqlite", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "/opt/airflow/scripts/in_container/run_flake8.sh", *sys.argv[1:], ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py index e72a7a33265fc..154a795f3e84a 100755 --- a/scripts/ci/pre_commit/pre_commit_migration_reference.py +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -30,31 +27,36 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) + from airflow_breeze.global_constants import MOUNT_SELECTED + from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) - return_code = subprocess.call( - args=[ + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) + create_static_check_volumes() + cmd_result = run_command( + [ "docker", "run", - "-v", - f"{AIRFLOW_SOURCES}:/opt/airflow/", + "-t", + *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "-c", "python3 /opt/airflow/scripts/in_container/run_migration_reference.py", ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 9d92b99e80edf..74d511a65b7ed 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -31,22 +28,20 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) create_static_check_volumes() - return_code = subprocess.call( - args=[ + cmd_result = run_command( + [ "docker", "run", "-t", @@ -59,9 +54,12 @@ "BACKEND=sqlite", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "/opt/airflow/scripts/in_container/run_mypy.sh", *sys.argv[1:], ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py index 93a5c3bf72ff0..7755e3d204bd8 100755 --- a/scripts/ci/pre_commit/pre_commit_ui_lint.py +++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -29,33 +26,36 @@ ) AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() -GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") -AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7" +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) + from airflow_breeze.global_constants import MOUNT_SELECTED + from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) - return_code = subprocess.call( - args=[ + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) + create_static_check_volumes() + cmd_result = run_command( + [ "docker", "run", - "-v", - f"{AIRFLOW_SOURCES}:/opt/airflow/", + "-t", + *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "-c", 'cd airflow/ui && yarn --frozen-lockfile --non-interactive && yarn run lint', ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py index c2807d4082c3f..65553c3d81345 100755 --- a/scripts/ci/pre_commit/pre_commit_www_lint.py +++ b/scripts/ci/pre_commit/pre_commit_www_lint.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -30,32 +27,36 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") -AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7" +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) + from airflow_breeze.global_constants import MOUNT_SELECTED + from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) - return_code = subprocess.call( - args=[ + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) + create_static_check_volumes() + cmd_result = run_command( + [ "docker", "run", - "-v", - f"{AIRFLOW_SOURCES}:/opt/airflow/", + "-t", + *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "-c", 'cd airflow/www && yarn --frozen-lockfile --non-interactive && yarn run lint', ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode)