From 1daff1c4f40b900b0298153d458551e0b86fc900 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 30 Jun 2022 18:10:05 +0200 Subject: [PATCH] Better diagnostics in case someone modified the dependencies file The "provider_dependencies.json" file gets automatically generated by pre-commit and in case it cannot be json-parsed, it will break building the CI image. This is a bit chicken-egg because on CI the image is needed to run pre-commits that could warn the user this is the case (and the image fails build with a bit cryptic message). This PR improves the diagnostics: * it runs the pre-commit check before image building which will fix the generated file (and will let the build run - only to fail at the latest static-checks step * it prints warning to the user seeing pre-commit error to not modify the file manually but let pre-commit do the job. --- .github/workflows/build-images.yml | 22 +++++++++++++++++ .github/workflows/ci.yml | 24 +++++++++++++++++++ ...pre_commit_build_providers_dependencies.py | 4 ++++ 3 files changed, 50 insertions(+) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 8903d87da3717..32fcf1521fc1c 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -224,6 +224,17 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true - name: > Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.allPythonVersionsListAsString }} @@ -316,6 +327,17 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 768e3ad15d831..ea570157d57d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,6 +332,18 @@ jobs: - name: "Free space" run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true + if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: > Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} ${{ needs.build-info.outputs.allPythonVersionsListAsString }} @@ -402,6 +414,18 @@ jobs: - name: "Free space" run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true + if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py index af2471ad62d4c..8ee81908f46c9 100755 --- a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py @@ -195,6 +195,10 @@ def check_if_different_provider_used(file_path: Path) -> None: console.print() sys.exit(1) DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n") + console.print( + f"[yellow]If you see changes to the {DEPENDENCIES_JSON_FILE_PATH} file - " + f"do not modify the file manually. Let pre-commit do the job!" + ) console.print() console.print("[green]Verification complete! Success!\n") console.print(f"Written {DEPENDENCIES_JSON_FILE_PATH}")