Better diagnostics in case someone modified the dependencies file (#2…

…4763) The "provider_dependencies.json" file gets automatically generated by pre-commit and in case it cannot be json-parsed, it will break building the CI image. This is a bit chicken-egg because on CI the image is needed to run pre-commits that could warn the user this is the case (and the image fails build with a bit cryptic message). This PR improves the diagnostics: * it runs the pre-commit check before image building which will fix the generated file (and will let the build run - only to fail at the latest static-checks step * it prints warning to the user seeing pre-commit error to not modify the file manually but let pre-commit do the job.
apache · Jul 1, 2022 · 1b3905e · 1b3905e
1 parent a1679be
commit 1b3905e
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 0 deletions.
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -224,6 +224,17 @@ jobs:
       - run: ./scripts/ci/install_breeze.sh
       - name: "Free space"
         run: breeze free-space
+      - name: Cache pre-commit envs
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+          restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+      - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+        run: >
+          breeze static-checks --type update-providers-dependencies --all-files
+          --show-diff-on-failure --color always || true
       - name: >
           Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
@@ -316,6 +327,17 @@ jobs:
       - run: ./scripts/ci/install_breeze.sh
       - name: "Free space"
         run: breeze free-space
+      - name: Cache pre-commit envs
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+          restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+      - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+        run: >
+          breeze static-checks --type update-providers-dependencies --all-files
+          --show-diff-on-failure --color always || true
       - name: >
           Pull CI image for PROD build:
           ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -337,6 +337,18 @@ jobs:
       - name: "Free space"
         run: breeze free-space
         if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: Cache pre-commit envs
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+          restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+      - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+        run: >
+          breeze static-checks --type update-providers-dependencies --all-files
+          --show-diff-on-failure --color always || true
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
       - name: >
           Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
           ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
@@ -393,6 +405,18 @@ jobs:
       - name: "Free space"
         run: breeze free-space
         if: needs.build-info.outputs.inWorkflowBuild == 'true'
+      - name: Cache pre-commit envs
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
+${{ hashFiles('.pre-commit-config.yaml') }}"
+          restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+      - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+        run: >
+          breeze static-checks --type update-providers-dependencies --all-files
+          --show-diff-on-failure --color always || true
+        if: needs.build-info.outputs.inWorkflowBuild == 'true'
       - name: >
           Pull CI image for PROD build:
           ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"

diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py
@@ -195,6 +195,10 @@ def check_if_different_provider_used(file_path: Path) -> None:
         console.print()
         sys.exit(1)
     DEPENDENCIES_JSON_FILE_PATH.write_text(json.dumps(unique_sorted_dependencies, indent=2) + "\n")
+    console.print(
+        f"[yellow]If you see changes to the {DEPENDENCIES_JSON_FILE_PATH} file - "
+        f"do not modify the file manually. Let pre-commit do the job!"
+    )
     console.print()
     console.print("[green]Verification complete! Success!\n")
     console.print(f"Written {DEPENDENCIES_JSON_FILE_PATH}")