mAKe Breeze installation and reinstallation support both uv and pipx (a…

…pache#43607)

So far `breeze` fully supported only `pipx` installation. For
example it would reinstall itself automatically with pipx if you
attempted to run it from another workspace/checked out repository
of Airflow, and it only provided instructions for pipx.

With this PR:

* the `uv tool` is preferred way to install breeze
* the `pipx` is PSF-governance managed alternative
* breeze will reinstall itself using uv if it has been installed with uv
  before when it is run from a different workspace or different
  airflow repo checked out in another folder

Also documentation is updated to make `uv` the recommended tool and
describing how to install it - with `pipx` provided as an alternative.

Warning is printed in case pre-commit-uv is not installed with the
pre-commit (pre-commit-uv significantly speeds up installation of
the venvs by pre-commit). This warning also provides instructions
how to install it.

(cherry picked from commit ddc5670)

.github/actions/install-pre-commit/action.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Install pre-commit'
+description: 'Installs pre-commit and related packages'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    default: 3.9
+  uv-version:
+    description: 'uv version to use'
+    default: 0.4.29
+  pre-commit-version:
+    description: 'pre-commit version to use'
+    default: 4.0.1
+  pre-commit-uv-version:
+    description: 'pre-commit-uv version to use'
+    default: 4.1.4
+runs:
+  using: "composite"
+  steps:
+    - name: Install pre-commit, uv, and pre-commit-uv
+      shell: bash
+      run: >
+        pip install
+        pre-commit==${{inputs.pre-commit-version}}
+        uv==${{inputs.uv-version}}
+        pre-commit-uv==${{inputs.pre-commit-uv-version}}
+    - name: Cache pre-commit envs
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+        restore-keys: |
+          pre-commit-${{inputs.python-version}}-

.github/workflows/basic-tests.yml

@@ -232,16 +232,11 @@ jobs:
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
         id: breeze
-      - name: Cache pre-commit envs
-        uses: actions/cache@v4
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
         with:
-          path: ~/.cache/pre-commit
-          # yamllint disable-line rule:line-length
-          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-          restore-keys: "\
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-\
-            ${{ hashFiles('.pre-commit-config.yaml') }}\n
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-"
+          python-version: ${{steps.breeze.outputs.host-python-version}}
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:

.github/workflows/static-checks-mypy-docs.yml

@@ -36,7 +36,7 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to run mypy checks (true/false)"
         required: true
         type: string
-      mypy-folders:
+      mypy-checks:
         description: "List of folders to run mypy checks on"
         required: false
         type: string
@@ -92,6 +92,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to debug resources (true/false)"
         required: true
         type: string
+      docs-build:
+        description: "Whether to build docs (true/false)"
+        required: true
+        type: string
 jobs:
   static-checks:
     timeout-minutes: 45
@@ -122,14 +126,11 @@ jobs:
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version}}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
         id: breeze
-      - name: Cache pre-commit envs
-        uses: actions/cache@v4
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
         with:
-          path: ~/.cache/pre-commit
-          # yamllint disable-line rule:line-length
-          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
-          restore-keys: |
-            pre-commit-${{steps.breeze.outputs.host-python-version}}-
+          python-version: ${{steps.breeze.outputs.host-python-version}}
       - name: "Static checks"
         run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment
         env:
@@ -148,7 +149,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mypy-folder: ${{ fromJSON(inputs.mypy-folders) }}
+        mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
       IMAGE_TAG: "${{ inputs.image-tag }}"
@@ -166,10 +167,13 @@ jobs:
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
         id: breeze
-      - name: "MyPy checks for ${{ matrix.mypy-folder }}"
-        run: |
-          pip install pre-commit
-          pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files
+      - name: "Install pre-commit"
+        uses: ./.github/actions/install-pre-commit
+        id: pre-commit
+        with:
+          python-version: ${{steps.breeze.outputs.host-python-version}}
+      - name: "MyPy checks for ${{ matrix.mypy-check }}"
+        run: pre-commit run --color always --verbose --hook-stage manual ${{matrix.mypy-check}} --all-files
         env:
           VERBOSE: "false"
           COLUMNS: "250"
@@ -182,6 +186,7 @@ jobs:
     timeout-minutes: 150
     name: "Build documentation"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+    if: inputs.docs-build == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -231,8 +236,6 @@ jobs:
     timeout-minutes: 150
     name: "Publish documentation"
     needs: build-docs
-    # For canary runs we need to push documentation to AWS S3 and preparing it takes a lot of space
-    # So we should use self-hosted ASF runners for this
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-docs-build) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
@@ -259,16 +262,22 @@ jobs:
         with:
           name: airflow-docs
           path: './docs/_build'
+      - name: Check disk space available
+        run: df -h
+      - name: Create /mnt/airflow-site directory
+        run: sudo mkdir -p /mnt/airflow-site && sudo chown -R "${USER}" /mnt/airflow-site
       - name: "Clone airflow-site"
         run: >
-          git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
-          echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
+          git clone https://github.com/apache/airflow-site.git /mnt/airflow-site/airflow-site &&
+          echo "AIRFLOW_SITE_DIRECTORY=/mnt/airflow-site/airflow-site" >> "$GITHUB_ENV"
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
       - name: "Publish docs"
         run: >
           breeze release-management publish-docs --override-versioned --run-in-parallel
           ${{ inputs.docs-list-as-string }}
+      - name: Check disk space available
+        run: df -h
       - name: "Generate back references for providers"
         run: breeze release-management add-back-references all-providers
       - name: "Generate back references for apache-airflow"

contributing-docs/03_contributors_quick_start.rst

@@ -451,6 +451,39 @@ tests are applied when you commit your code.
 
 To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be run locally before committing changes.
 
+.. note::
+
+    We have recently started to recommend ``uv`` for our local development. Currently (October 2024) ``uv``
+    speeds up installation more than 10x comparing to ``pip``. While we still describe ``pip`` and ``pipx``
+    below, we also show the ``uv`` alternatives.
+
+.. note::
+
+    Remember to have global python set to Python >= 3.9 - Python 3.8 is end-of-life already and we've
+    started to use Python 3.9+ features in Airflow and accompanying scripts.
+
+
+Installing pre-commit is best done with ``pipx``:
+
+.. code-block:: bash
+
+    pipx install pre-commit
+
+You can still add uv support for pre-commit if you use pipx using the commands:
+
+.. code-block:: bash
+
+    pipx install pre-commit
+    pipx inject
+    pipx inject pre-commit pre-commit-uv
+
+Also, if you already use ``uvx`` instead of ``pipx``, use this command:
+
+.. code-block:: bash
+
+    uv tool install pre-commit --with pre-commit-uv --force-reinstall
+
+
 1.  Installing required packages
 
 on Debian / Ubuntu, install via

dev/breeze/doc/01_installation.rst

@@ -151,13 +151,28 @@ Docker in WSL 2
     If VS Code is installed on the Windows host system then in the WSL Linux Distro
     you can run ``code .`` in the root directory of you Airflow repo to launch VS Code.
 
-The pipx tool
---------------
+The uv tool
+-----------
+
+We are recommending to use the ``uv`` tool to manage your virtual environments and generally as a swiss-knife
+of your Python environment (it supports installing various versions of Python, creating virtual environments,
+installing packages, managing workspaces and running development tools.).
+
+Installing ``uv`` is described in the `uv documentation <https://docs.astral.sh/uv/getting-started/installation/>`_.
+We highly recommend using ``uv`` to manage your Python environments, as it is very comprehensive,
+easy to use, it is faster than any of the other tools availables (way faster!) and has a lot of features
+that make it easier to work with Python.
+
+Alternative: pipx tool
+----------------------
 
-We are using ``pipx`` tool to install and manage Breeze. The ``pipx`` tool is created by the creators
+However, we do not want to be entirely dependent on ``uv`` as it is a software governed by a VC-backed vendor,
+so we always want to provide open-source governed alternatives for our tools. If you can't or do not want to
+use ``uv``, we got you covered. Another too you can use to manage development tools (and ``breeze`` development
+environment is Python-Software-Foundation managed ``pipx``. The ``pipx`` tool is created by the creators
 of ``pip`` from `Python Packaging Authority <https://www.pypa.io/en/latest/>`_
 
-Note that ``pipx`` >= 1.4.1 is used.
+Note that ``pipx`` >= 1.4.1 should be used.
 
 Install pipx
 
@@ -172,7 +187,7 @@ environments. This can be done automatically by the following command (follow in
 
     pipx ensurepath
 
-In Mac
+In case ``pipx`` is not in your PATH, you can run it with Python module:
 
 .. code-block:: bash
 

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -836,7 +836,7 @@ def static_checks(
         for attempt in range(1, 1 + max_initialization_attempts):
             get_console().print(f"[info]Attempt number {attempt} to install pre-commit environments")
             initialization_result = run_command(
-                [sys.executable, "-m", "pre_commit", "install", "--install-hooks"],
+                ["pre-commit", "install", "--install-hooks"],
                 check=False,
                 no_output_dump_on_exception=True,
                 text=True,
@@ -849,7 +849,7 @@ def static_checks(
             get_console().print("[error]Could not install pre-commit environments[/]")
             sys.exit(return_code)
 
-    command_to_execute = [sys.executable, "-m", "pre_commit", "run"]
+    command_to_execute = ["pre-commit", "run"]
     if not one_or_none_set([last_commit, commit_ref, only_my_changes, all_files]):
         get_console().print(
             "\n[error]You can only specify "

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -229,13 +229,14 @@ class VersionedFile(NamedTuple):
 AIRFLOW_PIP_VERSION = "24.3.1"
 AIRFLOW_UV_VERSION = "0.4.29"
 AIRFLOW_USE_UV = False
-WHEEL_VERSION = "0.36.2"
-GITPYTHON_VERSION = "3.1.40"
-RICH_VERSION = "13.7.0"
-NODE_VERSION = "21.2.0"
-PRE_COMMIT_VERSION = "3.5.0"
-HATCH_VERSION = "1.9.1"
-PYYAML_VERSION = "6.0.1"
+# TODO: automate thsese as well
+WHEEL_VERSION = "0.44.0"
+GITPYTHON_VERSION = "3.1.43"
+RICH_VERSION = "13.9.4"
+NODE_VERSION = "22.2.0"
+PRE_COMMIT_VERSION = "4.0.1"
+HATCH_VERSION = "1.13.0"
+PYYAML_VERSION = "6.0.2"
 
 AIRFLOW_BUILD_DOCKERFILE = f"""
 FROM python:{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}-slim-{ALLOWED_DEBIAN_VERSIONS[0]}

dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py

@@ -391,10 +391,7 @@ def create_virtualenv(force_venv_setup: bool) -> RunCommandResult:
             "[info]You can uninstall breeze and install it again with earlier Python "
             "version. For example:[/]\n"
         )
-        get_console().print("pipx reinstall --python PYTHON_PATH apache-airflow-breeze\n")
-        get_console().print(
-            f"[info]PYTHON_PATH - path to your Python binary(< {higher_python_version_tuple})[/]\n"
-        )
+
         get_console().print("[info]Then recreate your k8s virtualenv with:[/]\n")
         get_console().print("breeze k8s setup-env --force-venv-setup\n")
         sys.exit(1)

dev/breeze/src/airflow_breeze/utils/path_utils.py

@@ -167,8 +167,9 @@ def reinstall_if_setup_changed() -> bool:
             return False
         if "apache-airflow-breeze" in e.msg:
             print(
-                """Missing Package `apache-airflow-breeze`.
-                   Use `pipx install -e ./dev/breeze` to install the package."""
+                """Missing Package `apache-airflow-breeze`. Please install it.\n
+                   Use `uv tool install -e ./dev/breeze or `pipx install -e ./dev/breeze`
+                   to install the package."""
             )
             return False
     sources_hash = get_installation_sources_config_metadata_hash()
@@ -224,10 +225,10 @@ def get_used_airflow_sources() -> Path:
 @lru_cache(maxsize=None)
 def find_airflow_sources_root_to_operate_on() -> Path:
     """
-    Find the root of airflow sources we operate on. Handle the case when Breeze is installed via `pipx` from
-    a different source tree, so it searches upwards of the current directory to find the right root of
-    airflow directory we are actually in. This **might** be different than the sources of Airflow Breeze
-    was installed from.
+    Find the root of airflow sources we operate on. Handle the case when Breeze is installed via
+    `pipx` or `uv tool` from a different source tree, so it searches upwards of the current directory
+    to find the right root of airflow directory we are actually in. This **might** be different
+    than the sources of Airflow Breeze was installed from.
 
     If not found, we operate on Airflow sources that we were installed it. This handles the case when
     we run Breeze from a "random" directory.

dev/breeze/src/airflow_breeze/utils/python_versions.py

@@ -56,8 +56,9 @@ def check_python_version():
     if error:
         get_console().print(
             "[warning]Please reinstall Breeze using Python 3.9 - 3.11 environment.[/]\n\n"
-            "For example:\n\n"
-            "pipx uninstall apache-airflow-breeze\n"
-            "pipx install --python $(which python3.9) -e ./dev/breeze --force\n"
+            "If you are using uv:\n\n"
+            "   uv tool install --force --reinstall --python 3.9 -e ./dev/breeze\n\n"
+            "If you are using pipx:\n\n"
+            "   pipx install --python $(which python3.9) --force -e ./dev/breeze\n"
         )
         sys.exit(1)

dev/breeze/src/airflow_breeze/utils/reinstall.py

@@ -27,15 +27,24 @@
 
 def reinstall_breeze(breeze_sources: Path, re_run: bool = True):
     """
-    Reinstalls Breeze from specified sources.
+    Re-installs Breeze from specified sources.
     :param breeze_sources: Sources where to install Breeze from.
     :param re_run: whether to re-run the original command that breeze was run with.
     """
+    # First check if `breeze` is installed with uv and if it is, reinstall it using uv
+    # If not - we assume pipx is used and we reinstall it using pipx
     # Note that we cannot use `pipx upgrade` here because we sometimes install
     # Breeze from different sources than originally installed (i.e. when we reinstall airflow
     # From the current directory.
     get_console().print(f"\n[info]Reinstalling Breeze from {breeze_sources}\n")
-    subprocess.check_call(["pipx", "install", "-e", str(breeze_sources), "--force"])
+    result = subprocess.run(["uv", "tool", "list"], text=True, capture_output=True, check=False)
+    if result.returncode == 0:
+        if "apache-airflow-breeze" in result.stdout:
+            subprocess.check_call(
+                ["uv", "tool", "install", "--force", "--reinstall", "-e", breeze_sources.as_posix()]
+            )
+    else:
+        subprocess.check_call(["pipx", "install", "-e", breeze_sources.as_posix(), "--force"])
     if re_run:
         # Make sure we don't loop forever if the metadata hash hasn't been updated yet (else it is tricky to
         # run pre-commit checks via breeze!)