⚡ ♻️ switch to uv (#9)

* ♻️ switch to uv * use hatch for versioning * update README.md
danielgafni · Oct 7, 2024 · 4a7473b · 4a7473b
1 parent 9ab8117
commit 4a7473b
Show file tree

Hide file tree

Showing 10 changed files with 3,733 additions and 142 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -3,7 +3,6 @@
 !dagster_ray
 !tests
 !pyproject.toml
-!poetry.lock
-!poetry.toml
+!uv.lock
 !README.md
 !.pre-commit-config.yaml
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -14,37 +14,34 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  UV_SYSTEM_PYTHON: "1"
+
 jobs:
   test:
-    name: Test Python ${{ matrix.py }} - Kubernetes ${{ matrix.kubernetes }}
+    name: test ${{ matrix.py }} - ${{ matrix.os }}
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
         os:
           - Ubuntu
         py:
-          #- "3.12"
+          # - "3.12"
           - "3.11"
           - "3.10"
           - "3.9"
-        kubernetes:
-          - "1.31.0"
-          - "1.30.0"
-          - "1.29.0"
     steps:
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.py }}
       - uses: actions/checkout@v4
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          version: 1.8.2
-          virtualenvs-create: false
-          virtualenvs-in-project: false
-          installer-parallel: true
+          version: 0.4.18
+          enable-cache: true
+      - name: Set up Python ${{ matrix.py }}
+        run: uv python install ${{ matrix.py }}
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
       - uses: azure/[email protected]
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -53,13 +50,11 @@ jobs:
         with:
           start: false
           driver: docker
-      - name: Install dependencies
-        run: poetry install --all-extras --sync
+          #- uses: mxschmitt/action-tmate@v3
       - name: Run tests
         env:
-          PYTEST_KUBERAY_VERSIONS: "1.2.0"  # will run tests for all these KubeRay versions
-          PYTEST_KUBERNETES_VERSION: ${{ matrix.kubernetes }}
-        run: pytest -v .
+          PYTEST_KUBERAY_VERSIONS: "1.1.0,1.2.2"  # will run tests for all these KubeRay versions
+        run: uv run pytest -v .
 
   lint:
     name: lint ${{ matrix.py }} - ${{ matrix.os }}
@@ -70,27 +65,22 @@ jobs:
         os:
           - Ubuntu
         py:
-          #- "3.12"
           - "3.11"
           - "3.10"
           - "3.9"
     steps:
-      - name: Setup python for test ${{ matrix.py }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.py }}
       - uses: actions/checkout@v2
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          version: 1.8.2
-          virtualenvs-create: false
-          virtualenvs-in-project: false
-          installer-parallel: true
+          version: 0.4.18
+          enable-cache: true
+      - name: Set up Python ${{ matrix.py }}
+        run: uv python install ${{ matrix.py }}
       - name: Install dependencies
-        run: poetry install --all-extras --sync
+        run: uv sync --all-extras --dev
       - name: Run pre-commit hooks
-        run: pre-commit run --all-files
+        run: uv run pre-commit run --all-files
 
   qa_success:
     name: QA Success
@@ -105,24 +95,21 @@ jobs:
     if: contains(github.ref, 'refs/tags/')
     name: Publish to PyPI
     runs-on: ubuntu-latest
+    environment:
+      name: release
     steps:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
-      - uses: actions/checkout@v2
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
         with:
-          version: 1.8.2
-          virtualenvs-create: false
-          virtualenvs-in-project: false
-          installer-parallel: true
-      - name: Install poetry-dynamic-versioning
-        run: poetry self add [email protected]
+          version: 0.3.4
+          enable-cache: true
+      - name: Set up Python ${{ matrix.py }}
+        run: uv python install
+      - name: Generate Version
+        run: uv run hatch version $(dunamai from any --style pep440)
       - name: Publish to PyPI
-        env:
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        run: |
-          poetry config pypi-token.pypi $PYPI_TOKEN
-          poetry publish --build
+        run: uv publish
diff --git a/.python-version b/.python-version
diff --git a/Dockerfile b/Dockerfile
@@ -12,45 +12,22 @@ RUN --mount=type=cache,target=/var/cache/apt \
 
 COPY --from=bitnami/kubectl:1.30.3 /opt/bitnami/kubectl/bin/kubectl /usr/local/bin/
 
-# install poetry
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=on \
-    PIP_DEFAULT_TIMEOUT=100 \
-    POETRY_HOME="/opt/poetry" \
-    POETRY_NO_INTERACTION=1 \
-    PYSETUP_PATH="/opt/pysetup" \
-    VENV_PATH="/opt/pysetup/.venv"
-
-ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" \
-    POETRY_VERSION=1.8.2 \
-    POETRY_DYNAMIC_VERSIONING_PLUGIN_VERSION=1.2.0 \
-    POETRY_DYNAMIC_VERSIONING_COMMANDS=version,build,publish
-
-RUN mkdir "$HOME/opt/" \
-    && curl -sSL https://install.python-poetry.org > /tmp/get-poetry.py \
-    && python3 /tmp/get-poetry.py \
-    && poetry config virtualenvs.create false \
-    && mkdir -p /cache/poetry \
-    && poetry config cache-dir /cache/poetry \
-    && python -m pip install --upgrade pip wheel setuptools \
-    && poetry self add "poetry-dynamic-versioning[plugin]=$POETRY_DYNAMIC_VERSIONING_PLUGIN_VERSION"
-
-ARG INSTALLER_PARALLEL=true
-RUN poetry config installer.parallel $INSTALLER_PARALLEL
+# install uv (https://github.com/astral-sh/uv)
+# docs for using uv with Docker: https://docs.astral.sh/uv/guides/integration/docker/
+COPY --from=ghcr.io/astral-sh/uv:0.4.18 /uv /bin/uv
 
+ENV UV_PROJECT_ENVIRONMENT=/usr/local/
 ENV DAGSTER_HOME=/opt/dagster/dagster_home
 RUN mkdir -p $DAGSTER_HOME
 
 FROM base AS base-prod
 
 WORKDIR /src
 
-COPY pyproject.toml poetry.lock  ./
+COPY pyproject.toml uv.lock  ./
 
-RUN --mount=type=cache,target=/cache/poetry \
-    poetry install --no-root --only main --all-extras
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --all-extras --no-dev --no-install-project
 
 FROM base-prod AS base-dev
 
@@ -64,8 +41,8 @@ RUN --mount=type=cache,target=/cache/downloads \
     curl https://nodejs.org/dist/v$NODE_VERSION/$NODE_PACKAGE.tar.gz -o /cache/downloads/$NODE_PACKAGE.tar.gz \
     && tar -xzC /opt/ -f /cache/downloads/$NODE_PACKAGE.tar.gz
 
-RUN --mount=type=cache,target=/cache/poetry \
-    poetry install --no-root --only dev --all-extras
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --all-extras --no-install-project
 
 # -------------------------------------------------------------
 FROM base-${BUILD_DEPENDENCIES} AS final
@@ -74,10 +51,4 @@ FROM base-${BUILD_DEPENDENCIES} AS final
 COPY . .
 
 # finally install all our code
-RUN poetry install --all-extras
-
-ARG POETRY_DYNAMIC_VERSIONING_BYPASS=unset
-ENV POETRY_DYNAMIC_VERSIONING_BYPASS_TMP=$POETRY_DYNAMIC_VERSIONING_BYPASS
-RUN if [ $POETRY_DYNAMIC_VERSIONING_BYPASS_TMP != unset ]; \
-    then export POETRY_DYNAMIC_VERSIONING_BYPASS=$POETRY_DYNAMIC_VERSIONING_BYPASS_TMP && poetry dynamic-versioning; \
-    fi
+RUN uv sync --frozen --all-extras
diff --git a/README.md b/README.md
@@ -294,13 +294,16 @@ WIP
 # Development
 
 ```shell
-poetry install --all-extras
-poetry shell
-pre-commit install
+uv sync --all-extras
+uv run pre-commit install
 ```
 
 ## Testing
 
+```shell
+uv run pytest
+```
+
 ### KubeRay
 
 Required tools:

diff --git a/dagster_ray/_version.py b/dagster_ray/_version.py
@@ -1 +1 @@
-__version__ = "0.0.0"  # this field is managed by poetry-dynamic-versioning
+__version__ = "0.0.0"  # this field is managed by hatch
diff --git a/dagster_ray/kuberay/client/raycluster/client.py b/dagster_ray/kuberay/client/raycluster/client.py
@@ -141,6 +141,7 @@ def wait_until_ready(
                 and item["metadata"].get("name") == name  # type: ignore
                 and status.get("state") == "ready"
                 and status.get("head")
+                and status.get("endpoints", {}).get("dashboard")
             ):
                 if image is not None:
                     if (

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,13 +1,14 @@
-[tool.poetry]
+
+[project]
 name = "dagster-ray"
-version = "0.0.0"
+urls = { repository = "https://github.com/danielgafni/dagster-ray" }
 description = "Dagster integration library for Ray"
 authors = [
-    "Daniel Gafni <[email protected]>"
+    {name = "Daniel Gafni", email = "[email protected]"},
 ]
+license = {text = "Apache-2.0"}
+requires-python = ">=3.8.1"
 readme = "README.md"
-packages = [{include = "dagster_ray"}]
-repository = "https://github.com/danielgafni/dagster-ray"
 keywords = [
     "dagster",
     "ray",
@@ -23,58 +24,50 @@ classifiers = [
     "Operating System :: OS Independent",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
-license = "Apache-2.0"
 
-[tool.poetry.dependencies]
-python = ">=3.8.1,<3.13"
-dagster = ">=1.8.0"
-tenacity = ">=8.0.0"
-grpcio = [
-    {version = "*", python = "<3.12"},
-    {version = ">=1.60.0", python = ">=3.12"}
-]
-greenlet = [
-    {version = "*", python = "<3.12"},
-    {version = ">=3.0", python = ">=3.12"}
+dependencies = [
+    "pyyaml>=4.0.0",
+    "tenacity>=8.0.0",
+    "dagster>=1.8.0",
+    "tenacity>=8.0.0"
 ]
 
-kubernetes = { version = ">=20.0.0", optional = true  }
-dagster-k8s = { version = ">=0.19.0", optional = true  }
+dynamic = ["version"]
 
-[tool.poetry.extras]
+[project.optional-dependencies]
+executor = [
+    "ray[client]"
+]
 kuberay = [
-    "kubernetes",
-    "dagster-k8s",
+    "pyyaml",
+    "kubernetes>=20.0.0",
+    "dagster-k8s>=0.19.0",
 ]
 
-[tool.poetry.group.dev.dependencies]
-hypothesis = ">=6.77.0"
-pytest = ">=7.3.1"
-ruff = ">=0.3.0"
-pyright = "^1.1.313"
-pre-commit = "^3.3.2"
-dagster-webserver = ">=1.3.9"
-pytest-cases = ">=3.6.14"
-pytest-kubernetes = ">=0.3.1"
-blacken-docs = ">=1.16.0"
-ray = { version = ">=2.7.0", extras = ["all"] }
+[tool.uv]
+dev-dependencies = [
+    "hypothesis<7.0.0,>=6.77.0",
+    "pytest<8.0.0,>=7.3.1",
+    "ruff<1.0.0,>=0.3.0",
+    "pyright<2.0.0,>=1.1.313",
+    "pre-commit>=3.3.2,<4.0.0",
+    "dagit<2.0.0,>=1.3.9",
+    "pytest-cases<4.0.0,>=3.6.14",
+    "pytest-kubernetes<1.0.0,>=0.3.1",
+    "blacken-docs<2.0.0,>=1.16.0",
+    "ray[default,data]>=2.7.0",
+    "hatch>=1.12.0",
+    "dunamai>=1.22.0",
+    "pip>=24.2",
+]
 
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
 
-[tool.poetry-dynamic-versioning]
-enable = true
-strict = false
-vcs = "git"
-style = "pep440"
-dirty = true
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
-[tool.poetry-dynamic-versioning.substitution]
-files = [
-    "pyproject.toml",
-    "dagster_ray/_version.py"
-]
+[tool.hatch.version]
+path = "dagster_ray/_version.py"
 
 [tool.pytest.ini_options]
 addopts = "-vvv --capture=no --log-disable=faker"

diff --git a/tests/kuberay/conftest.py b/tests/kuberay/conftest.py
@@ -70,7 +70,7 @@ def dagster_ray_image():
 # probably should only do it in CI
 KUBERNETES_VERSION = os.getenv("PYTEST_KUBERNETES_VERSION", "1.31.0")
 
-KUBERAY_VERSIONS = os.getenv("PYTEST_KUBERAY_VERSIONS", "1.2.0").split(",")
+KUBERAY_VERSIONS = os.getenv("PYTEST_KUBERAY_VERSIONS", "1.2.2").split(",")
 
 NAMESPACE = "ray"
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		__version__ = "0.0.0" # this field is managed by poetry-dynamic-versioning
		__version__ = "0.0.0" # this field is managed by hatch