Make Breeze installation and reinstallation support both uv and pipx (#…

…43607) (#43626) So far `breeze` fully supported only `pipx` installation. For example it would reinstall itself automatically with pipx if you attempted to run it from another workspace/checked out repository of Airflow, and it only provided instructions for pipx. With this PR: * the `uv tool` is preferred way to install breeze * the `pipx` is PSF-governance managed alternative * breeze will reinstall itself using uv if it has been installed with uv before when it is run from a different workspace or different airflow repo checked out in another folder Also documentation is updated to make `uv` the recommended tool and describing how to install it - with `pipx` provided as an alternative. Warning is printed in case pre-commit-uv is not installed with the pre-commit (pre-commit-uv significantly speeds up installation of the venvs by pre-commit). This warning also provides instructions how to install it. (cherry picked from commit ddc5670)
apache · Nov 4, 2024 · 07a080e · 07a080e
1 parent c3d246b
commit 07a080e
Show file tree

Hide file tree

Showing 28 changed files with 2,043 additions and 155 deletions.
diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Install pre-commit'
+description: 'Installs pre-commit and related packages'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    default: 3.9
+  uv-version:
+    description: 'uv version to use'
+    default: 0.4.29
+  pre-commit-version:
+    description: 'pre-commit version to use'
+    default: 4.0.1
+  pre-commit-uv-version:
+    description: 'pre-commit-uv version to use'
+    default: 4.1.4
+runs:
+  using: "composite"
+  steps:
+    - name: Install pre-commit, uv, and pre-commit-uv
+      shell: bash
+      run: >
+        pip install
+        pre-commit==${{inputs.pre-commit-version}}
+        uv==${{inputs.uv-version}}
+        pre-commit-uv==${{inputs.pre-commit-uv-version}}
+    - name: Cache pre-commit envs
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+        restore-keys: |
+          pre-commit-${{inputs.python-version}}-
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
@@ -80,6 +80,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources (true/false)"
         required: true
         type: string
+      use-uv:
+        description: "Whether to use uv to build the image (true/false)"
+        required: true
+        type: string
 jobs:
   # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
   # tests to complete - it is run very early in the build process for "main" merges in order to refresh
@@ -109,7 +113,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ inputs.use-uv}}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.branch == 'main'
@@ -165,6 +169,6 @@ jobs:
 #      platform: "linux/arm64"
 #      branch: ${{ inputs.branch }}
 #      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: "true"
+#      use-uv: ${{ inputs.use-uv}}
 #      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
 #      docker-cache: ${{ inputs.docker-cache }}
diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -232,16 +232,11 @@ jobs:
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         id: breeze
-      - name: Cache pre-commit envs
-        uses: actions/cache@v4
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
         with:
-          path: ~/.cache/pre-commit
-          # yamllint disable-line rule:line-length
-          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-          restore-keys: "\
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-\
-            ${{ hashFiles('.pre-commit-config.yaml') }}\n
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-"
+          python-version: ${{steps.breeze.outputs.host-python-version}}
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -16,7 +16,7 @@
 # under the License.
 #
 ---
-name: "Build Images"
+name: Build Images
 run-name: >
   Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
 on:  # yamllint disable-line rule:truthy
@@ -54,7 +54,7 @@ concurrency:
 jobs:
   build-info:
     timeout-minutes: 10
-    name: "Build Info"
+    name: Build Info
     # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners
     runs-on: ["ubuntu-22.04"]
     env:
@@ -71,6 +71,7 @@ jobs:
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
       default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
       runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
       runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
@@ -89,7 +90,7 @@ jobs:
         }}"
     if: github.repository == 'apache/airflow'
     steps:
-      - name: "Cleanup repo"
+      - name: Cleanup repo
         shell: bash
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - name: Discover PR merge commit
@@ -154,13 +155,13 @@ jobs:
       #  COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT
       #  CODE WILL BE RUN FROM THE PR.
       ####################################################################################################
-      - name: "Cleanup docker"
+      - name: Cleanup docker
         run: ./scripts/ci/cleanup_docker.sh
-      - name: "Setup python"
+      - name: Setup python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
-      - name: "Install Breeze"
+          python-version: "3.9"
+      - name: Install Breeze
         uses: ./.github/actions/breeze
       ####################################################################################################
       #  WE RUN SELECTIVE CHECKS HERE USING THE TARGET COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM
@@ -202,7 +203,7 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -212,7 +213,7 @@ jobs:
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
 
   generate-constraints:
-    name: "Generate constraints"
+    name: Generate constraints
     needs: [build-info, build-ci-images]
     uses: ./.github/workflows/generate-constraints.yml
     with:
@@ -245,9 +246,9 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      platform: "linux/amd64"
+      platform: linux/amd64
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}

diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml
@@ -28,6 +28,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
+      canary-run:
+        description: "Whether this is a canary run"
+        required: true
+        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -209,6 +213,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       VERBOSE: "true"
+      CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
     if: inputs.skip-provider-tests != 'true'
     steps:
       - name: "Cleanup repo"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -75,6 +75,7 @@ jobs:
       default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
       default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
       default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
+      force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
       parallel-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
@@ -95,7 +96,7 @@ jobs:
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docs-build: ${{ steps.selective-checks.outputs.docs-build }}
-      mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
+      mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
       needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
       needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
       needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
@@ -199,7 +200,7 @@ jobs:
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       branch: ${{ needs.build-info.outputs.default-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}
@@ -263,6 +264,7 @@ jobs:
       latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
 
 
   generate-constraints:
@@ -290,7 +292,7 @@ jobs:
       runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
-      mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
+      mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -304,6 +306,7 @@ jobs:
       ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+      docs-build: ${{ needs.build-info.outputs.docs-build }}
 
   providers:
     name: "Provider checks"
@@ -319,6 +322,7 @@ jobs:
     with:
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
+      canary-run: ${{ needs.build-info.outputs.canary-run }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }}
@@ -541,7 +545,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}

diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
@@ -145,7 +145,7 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
+      use-uv: ${{ needs.build-info.outputs.force-pip && 'false' || 'true' }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.canary-run == 'true'

diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml
@@ -95,7 +95,7 @@ jobs:
         timeout-minutes: 25
         run: >
           breeze release-management generate-constraints --run-in-parallel
-          --airflow-constraints-mode constraints-no-providers --answer yes
+          --airflow-constraints-mode constraints-no-providers --answer yes --parallelism 3
         # The no providers constraints are only needed when we want to update constraints (in canary builds)
         # They slow down the start of PROD image builds so we want to only run them when needed.
         if: inputs.generate-no-providers-constraints == 'true'
@@ -115,7 +115,7 @@ jobs:
         run: >
           breeze release-management generate-constraints --run-in-parallel
           --airflow-constraints-mode constraints --answer yes
-          --chicken-egg-providers "${{ inputs.chicken-egg-providers }}"
+          --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" --parallelism 3
       - name: "Dependency upgrade summary"
         shell: bash
         run: |

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -59,6 +59,7 @@ on:  # yamllint disable-line rule:truthy
 jobs:
   tests-integration:
     timeout-minutes: 130
+    if: inputs.testable-integrations != '[]'
     name: "Integration Tests: ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:

diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -101,7 +101,7 @@ jobs:
             k8s-env-${{ steps.breeze.outputs.host-python-version }}-\
             ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','hatch_build.py') }}"
       - name: "Switch breeze to use uv"
-        run: breeze setup-config --use-uv
+        run: breeze setup config --use-uv
         if: inputs.use-uv == 'true'
       - name: Run complete K8S tests ${{ inputs.kubernetes-combos-list-as-string }}
         run: breeze k8s run-complete-tests --run-in-parallel --upgrade --no-copy-local-sources