Replace Poetry with UV (#1199)

* replace poetry with uv * use uv for tests * install wheel for python 3.12 installations * resolved several versioning issues for unit tests * bump vcrpy to version 6.x * CI * CI * CI * formatting * CI * update CI nightly tests * CI * CI * CI * convert cluster_tools to UV * update cluster tools docker file with uv * uv in "docs" sub project * update changelog * remove poetry lock files * cleanup * re-add typing-extensions to cluster_tools * CI * updated slurm docker image tag * CI * CI * CI * CI * CI * CI * Ci * Apply PR feedback
scalableminds · Oct 14, 2024 · 846d939 · 846d939
1 parent 3b07223
commit 846d939
Show file tree

Hide file tree

Showing 41 changed files with 6,271 additions and 7,571 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1,3 @@
 webknossos/testdata
 webknossos/testoutput
+.venv
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -44,11 +44,13 @@ jobs:
         working-directory: cluster_tools
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: ${{ matrix.python-version }}
-          architecture: "x64"
+          version: "0.4.19"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
       - name: Build/pull dockered-slurm image
         if: ${{ matrix.executors == 'slurm' }}
         run: |
@@ -70,9 +72,9 @@ jobs:
           docker logs c2
 
           # Run setup.py on all three nodes
-          docker exec -w /cluster_tools slurmctld bash -c "poetry install" &
-          docker exec -w /cluster_tools c1 bash -c "poetry install" &
-          docker exec -w /cluster_tools c2 bash -c "poetry install" &
+          docker exec -w /cluster_tools slurmctld bash -c "uv sync --frozen" &
+          docker exec -w /cluster_tools c1 bash -c "uv sync --frozen" &
+          docker exec -w /cluster_tools c2 bash -c "uv sync --frozen" &
           wait
 
       - name: Setup Kubernetes-in-Docker
@@ -84,21 +86,18 @@ jobs:
           ./kind create cluster --config=tests/cluster-config.yaml
           ./kind export kubeconfig
 
-          cp ../requirements.txt .
           docker build -f tests/Dockerfile -t scalableminds/cluster-tools:latest .
           ./kind load docker-image scalableminds/cluster-tools:latest
 
       - name: Install dependencies (without docker)
         if: ${{ matrix.executors == 'multiprocessing' }}
         run: |
-          pip install -r ../requirements.txt
-          poetry install
+          uv sync --frozen
 
       - name: Install dependencies (without docker)
         if: ${{ matrix.executors == 'kubernetes' || matrix.executors == 'dask' }}
         run: |
-          pip install -r ../requirements.txt
-          poetry install --all-extras
+          uv sync --all-extras --frozen
 
       - name: Check typing
         if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }}
@@ -117,7 +116,7 @@ jobs:
         run: |
           cd tests
           PYTEST_EXECUTORS=multiprocessing,sequential,test_pickling,debug_sequential \
-            poetry run python -m pytest -sv test_all.py test_multiprocessing.py
+            uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py
 
       - name: Run slurm tests
         if: ${{ matrix.executors == 'slurm' }}
@@ -126,22 +125,22 @@ jobs:
           docker exec \
             -w /cluster_tools/tests \
             -e PYTEST_EXECUTORS=slurm \
-            slurmctld bash -c "poetry run python -m pytest -sv test_all.py test_slurm.py"
+            slurmctld bash -c "uv run --frozen python -m pytest -sv test_all.py test_slurm.py"
           docker exec \
             -w /cluster_tools/tests \
-            slurmctld bash -c "poetry run python test_deref_main.py"
+            slurmctld bash -c "uv run --frozen python test_deref_main.py"
 
       - name: Run kubernetes tests
         if: ${{ matrix.executors == 'kubernetes' }}
         run: |
           cd tests
-          PYTEST_EXECUTORS=kubernetes poetry run python -m pytest -sv test_all.py test_kubernetes.py
+          PYTEST_EXECUTORS=kubernetes uv run --frozen python -m pytest -sv test_all.py test_kubernetes.py
 
       - name: Run dask tests
         if: ${{ matrix.executors == 'dask' }}
         run: |
           cd tests
-          PYTEST_EXECUTORS=dask poetry run python -m pytest -sv test_all.py test_dask.py
+          PYTEST_EXECUTORS=dask uv run --frozen python -m pytest -sv test_all.py test_dask.py
 
   webknossos_linux:
     needs: changes
@@ -161,17 +160,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: "x64"
-
-      - name: Install dependencies
-        run: |
-          pip install -r ../requirements.txt
-          poetry install --extras all --with examples --with dev
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          # Install a specific version of uv.
+          version: "0.4.19"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
       - name: Check formatting
         if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }}
         run: ./format.sh check
@@ -192,7 +190,7 @@ jobs:
 
       - name: Check if git is dirty
         run: |
-          git diff --no-ext-diff --quiet --exit-code
+          git diff --no-ext-diff --exit-code
           [[ -z $(git status -s) ]]
 
   webknossos_cli_docker:
@@ -205,21 +203,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: "3.10"
-
+          # Install a specific version of uv.
+          version: "0.4.19"
       - name: Write version file
         run: |
-          pip install -r requirements.txt
           pushd webknossos
 
-          PKG_VERSION="$(dunamai from git)"
+          PKG_VERSION="$(uvx dunamai from git)"
           echo "__version__ = '$PKG_VERSION'" > ./webknossos/version.py
-          poetry version "$PKG_VERSION"
+          sed -i 's/version = "0.0.0"/version = "'"${PKG_VERSION}"'"/g' pyproject.toml
 
-            popd
+          popd
 
       - name: Build docker image
         run: docker build -t scalableminds/webknossos-cli:$GITHUB_SHA -f webknossos/Dockerfile .
@@ -283,16 +280,11 @@ jobs:
         with:
           repository: scalableminds/webknossos
           path: docs/wk-repo
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: 3.8
-          architecture: "x64"
-
-      - name: Install dependencies
-        run: |
-          pip3 install -r requirements.txt
-
+          # Install a specific version of uv.
+          version: "0.4.19"
       - name: Build Docs
         run: |
           cd docs
@@ -325,7 +317,7 @@ jobs:
           SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }}
         run: |
           cd docs
-          poetry run linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \
+          uv run --frozen linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \
             curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \
               "$SLACK_HOOK"
 
@@ -340,20 +332,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: 3.8
-          architecture: "x64"
-      - name: Install dependencies
-        run: pip3 install -r requirements.txt
+          # Install a specific version of uv.
+          version: "0.4.19"
       - name: Publish python packages
         env:
-          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PASSWORD }}
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }}
         run: _tooling/publish.sh
       - name: Prepare github release
         run: |
-          VERSION="$(dunamai from git)"
+          VERSION="$(uvx dunamai from git)"
           _tooling/changelog_for_version.sh $VERSION > Changelog.md
       - name: Publish github release
         id: create_release

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -20,16 +20,14 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: ${{ matrix.python-version }}
-        architecture: 'x64'
-
-    - name: Install dependencies
-      run: |
-        pip install poetry
-        poetry install --extras all --with examples --with dev
+        version: "0.4.19"
+
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
 
     - name: Check if git is dirty
       run: |

diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml
@@ -15,15 +15,11 @@ jobs:
       with:
         repository: scalableminds/webknossos
         path: docs/wk-repo
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v1
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
       with:
-        python-version: "3.10"
-        architecture: 'x64'
-
-    - name: Install dependencies
-      run: |
-        pip install -r requirements.txt
+        # Install a specific version of uv.
+        version: "0.4.19"
 
     - name: Build Docs
       run: |
@@ -45,6 +41,6 @@ jobs:
         SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }}
       run: |
         cd docs
-        poetry run linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \
+        uv run --frozen linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \
           curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \
             "$SLACK_HOOK"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -102,15 +102,11 @@ The [WEBKNOSSOS-libs repository](https://github.com/scalableminds/webknossos-lib
 
 See below for specifics of the different packages. Let's have a look at the common tooling first:
 
-* [**poetry**](https://python-poetry.org) is used for dependency management and publishing.
-  Use `poetry install --all-extras` in each package folder to install all dependencies for development.
+* [**uv**](https://docs.astral.sh/uv) is used for dependency management and publishing.
+  Use `uv sync --all-extras` in each package folder to install all dependencies for development.
   By default, this creates a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for each package.
-  To run commands inside this package, prefix them with `poetry run`, e.g. `poetry run python myscript.py`,
-  or enter the virtual environment with `poetry shell`.
-  The creation of a separate environment can be disabled (e.g. if you want to manage this manually),
-  [see here for details](https://python-poetry.org/docs/configuration/#virtualenvscreate).
-  To install the preferred version for this repository, run
-  [`pip install -f requirements.txt`](https://github.com/scalableminds/webknossos-libs/blob/master/requirements.txt)
+  To run commands inside this package, prefix them with `uv run`, e.g. `uv run python myscript.py`,
+  or enter the virtual environment with `source .venv/bin/activate`.
 
   To install the dependencies for all sub-projects, run `make install`.
 

diff --git a/Makefile b/Makefile
@@ -15,14 +15,8 @@ endef
 list_packages_by_priority:
 	@echo $(packages_by_priority)
 
-update:
-	$(call in_each_pkg_by_dependency, poetry update --without dev)
-
-update-internal:
-	$(call in_each_pkg_by_dependency, poetry update $(packages_by_dependency))
-
 install:
-	$(call in_each_pkg_by_dependency, poetry install --all-extras)
+	$(call in_each_pkg_by_dependency, uv sync --all-extras)
 
 format:
 	$(call in_each_code_pkg, ./format.sh)

diff --git a/README.md b/README.md
@@ -3,6 +3,7 @@
 [![Supported Python Versions](https://img.shields.io/pypi/pyversions/webknossos.svg)](https://pypi.python.org/pypi/webknossos)
 [![Build Status](https://img.shields.io/github/actions/workflow/status/scalableminds/webknossos-libs/.github/workflows/ci.yml?branch=master)](https://github.com/scalableminds/webknossos-libs/actions?query=workflow%3A%22CI%22)
 [![Documentation](https://img.shields.io/badge/docs-passing-brightgreen.svg)](https://docs.webknossos.org/webknossos-py/index.html)
+[![Package Manager](https://img.shields.io/pypi/pyversions/uv.svg)](https://pypi.python.org/pypi/uv)
 [![Code Style](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://docs.astral.sh/ruff/)
 
 <img align="right" src="https://static.webknossos.org/logos/webknossos-icon-only.svg" alt="WEBKNOSSOS Logo" width="100" height="100"/>

diff --git a/_tooling/publish.sh b/_tooling/publish.sh
@@ -9,14 +9,18 @@ for PKG in {cluster_tools,webknossos}/pyproject.toml; do
     pushd "$PKG" > /dev/null
 
     cp pyproject.toml pyproject.toml.bak
-    PKG_VERSION="$(dunamai from git)"
+    PKG_VERSION="$(uvx dunamai from git)"
 
     echo "__version__ = '$PKG_VERSION'" > ./"$PKG"/version.py
 
-    poetry version "$PKG_VERSION"
-    # replace all relative path dependencies with the current version:
-    sed -i 's/\(.*\) = .* path \= \"\.\..*/\1 = "'"$PKG_VERSION"'"/g' pyproject.toml
-    poetry publish --build
+    # Update version number in pyproject.toml
+    sed -i 's/version = "0.0.0"/version = "'"${PKG_VERSION}"'"/g' pyproject.toml    
+
+    # replace relative path dependencies (i.e. cluster-tools) with the current version:
+    sed -i 's/"cluster-tools"/"cluster-tools=='"${PKG_VERSION}"'"/g' pyproject.toml    
+
+    uv build
+    uv publish
 
     # Restore files
     mv pyproject.toml.bak pyproject.toml

diff --git a/cluster_tools/Changelog.md b/cluster_tools/Changelog.md
@@ -14,6 +14,7 @@ For upgrade instructions, please check the respective *Breaking Changes* section
 ### Added
 
 ### Changed
+- Replaced the Python package manager `poetry` with `uv`. [#1199](https://github.com/scalableminds/webknossos-libs/pull/1199)
 
 ### Fixed
 

diff --git a/cluster_tools/README.md b/cluster_tools/README.md
@@ -81,9 +81,9 @@ docker exec -it slurmctld bash
 docker exec -it c1 bash
 ```
 
-Make sure to install all extra dependencies, such as Kubernetes, with `poetry install --all-extras`.
+Make sure to install all extra dependencies, such as Kubernetes, with `uv sync --all-extras`.
 
-Tests can be executed with `cd tests && poetry run pytest -s tests.py` after entering the container.
+Tests can be executed with `cd tests && uv run pytest -s tests.py` after entering the container.
 Linting can be run with `./lint.sh`.
 Code formatting (black) can be run with `./format.sh`.
 

diff --git a/cluster_tools/cluster_tools/executors/dask.py b/cluster_tools/cluster_tools/executors/dask.py
@@ -118,7 +118,7 @@ def __init__(
             import distributed  # noqa: F401 unused import
         except ModuleNotFoundError:
             logger.error(
-                'The distributed Python package for Dask is not installed. cluster_tools does not install this dependency be default. Run `pip install cluster_tools[dask]` or `poetry install --extras "dask"` to install Dask support.'
+                'The distributed Python package for Dask is not installed. cluster_tools does not install this dependency be default. Run `pip install cluster_tools[dask]` or `uv sync --extra "dask"` to install Dask support.'
             )
             exit()
 

diff --git a/cluster_tools/cluster_tools/schedulers/kube.py b/cluster_tools/cluster_tools/schedulers/kube.py
@@ -54,7 +54,7 @@ def __init__(
             import kubernetes  # noqa: F401 unused import
         except ModuleNotFoundError:
             logger.error(
-                'The Kubernetes Python package is not installed. cluster_tools does not install this dependency be default. Run `pip install cluster_tools[kubernetes]` or `poetry install --extras "kubernetes"` to install Kubernetes support.'
+                'The Kubernetes Python package is not installed. cluster_tools does not install this dependency be default. Run `pip install cluster_tools[kubernetes]` or `uv sync --extra "kubernetes"` to install Kubernetes support.'
             )
             exit()