From b184f20316845165e1dc79d4fd3db731ad7fa0e8 Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 28 Nov 2024 19:47:49 -0500 Subject: [PATCH 1/8] Use uv to install project, dev and test dependencies --- .devcontainer/Dockerfile | 38 ++++++++++++++++++++++++++++++++- .devcontainer/devcontainer.json | 6 ++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d6df0a9a0eb7..f690489676e8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,40 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:3.11 +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +ARG USERNAME=vscode RUN apt-get update && \ - apt-get install -y --no-install-recommends libgdal-dev + apt-get install -y --no-install-recommends libgdal-dev && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Enable bytecode compilation +ENV UV_COMPILE_BYTECODE=1 + +# Copy from the cache instead of linking since it's a mounted volume +ENV UV_LINK_MODE=copy + +COPY . /app + +# Install the project's dependencies using the lockfile and settings +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + uv sync --group dev --group tests \ + --extra duckdb --extra clickhouse --extra examples --extra geospatial + +ENV VENV_DIR=.venv +RUN chown -R $USERNAME $VENV_DIR && chmod -R 755 $VENV_DIR + +ENV HYPOTHESIS_EXAMPLE_DB_DIR=.hypothesis +RUN chown -R $USERNAME $HYPOTHESIS_EXAMPLE_DB_DIR && chmod -R 755 $HYPOTHESIS_EXAMPLE_DB_DIR + +ENV PYTEST_CACHE=.pytest_cache +RUN chown -R $USERNAME $PYTEST_CACHE && chmod -R 755 $PYTEST_CACHE + +# Place executables in the environment at the front of the path +ENV PATH="/app/.venv/bin:$PATH" + +ENTRYPOINT [] + +USER $USERNAME \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 12fd9b34bb2e..ca893ab847ff 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,8 @@ { - "build": { "dockerfile": "Dockerfile" }, - "postCreateCommand": ".devcontainer/postCreate.sh", + "build": { "dockerfile": "Dockerfile", + "context": ".." }, + "containerUser": "vscode", + "remoteUser": "vscode", "customizations": { "codespaces": { "openFiles": ["docs/tutorials/getting_started.qmd"] From a1f9449276dc1eb2b6b91cd2ea48a450ff2bae4d Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 28 Nov 2024 20:16:14 -0500 Subject: [PATCH 2/8] Update Dockerfile EOF and devcontainer.json formatting --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f690489676e8..dd8258845756 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -37,4 +37,4 @@ ENV PATH="/app/.venv/bin:$PATH" ENTRYPOINT [] -USER $USERNAME \ No newline at end of file +USER $USERNAME diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ca893ab847ff..71ee6ecd31c4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,5 @@ { - "build": { "dockerfile": "Dockerfile", - "context": ".." }, + "build": { "dockerfile": "Dockerfile", "context": ".." }, "containerUser": "vscode", "remoteUser": "vscode", "customizations": { From cbdf64ddbe0351dd88e68ad81c0cba8ac69cb63a Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 12 Dec 2024 03:06:02 +0000 Subject: [PATCH 3/8] Enable editing of project and use of git in container --- .devcontainer/Dockerfile | 17 ++++++++++++----- .devcontainer/devcontainer.json | 2 ++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index dd8258845756..bb1635090bd9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,11 +1,15 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:3.11 -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/uv:0.5.6 /uv /uvx /bin/ ARG USERNAME=vscode RUN apt-get update && \ apt-get install -y --no-install-recommends libgdal-dev && \ rm -rf /var/lib/apt/lists/* +RUN python3 -m pip install pipx --no-cache-dir +RUN python3 -m pipx ensurepath +RUN pipx install rust-just + WORKDIR /app # Enable bytecode compilation @@ -26,11 +30,14 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENV VENV_DIR=.venv RUN chown -R $USERNAME $VENV_DIR && chmod -R 755 $VENV_DIR -ENV HYPOTHESIS_EXAMPLE_DB_DIR=.hypothesis -RUN chown -R $USERNAME $HYPOTHESIS_EXAMPLE_DB_DIR && chmod -R 755 $HYPOTHESIS_EXAMPLE_DB_DIR +ENV IBIS_PROJECT=. +RUN chown -R $USERNAME $IBIS_PROJECT + +# ENV HYPOTHESIS_EXAMPLE_DB_DIR=.hypothesis +# RUN chmod -R 755 $HYPOTHESIS_EXAMPLE_DB_DIR -ENV PYTEST_CACHE=.pytest_cache -RUN chown -R $USERNAME $PYTEST_CACHE && chmod -R 755 $PYTEST_CACHE +# ENV PYTEST_CACHE=.pytest_cache +# RUN chmod -R 755 $PYTEST_CACHE # Place executables in the environment at the front of the path ENV PATH="/app/.venv/bin:$PATH" diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 71ee6ecd31c4..bb2c07c9d749 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,6 +2,8 @@ "build": { "dockerfile": "Dockerfile", "context": ".." }, "containerUser": "vscode", "remoteUser": "vscode", + "postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}", + "workspaceFolder": "/app", "customizations": { "codespaces": { "openFiles": ["docs/tutorials/getting_started.qmd"] From 5be130697ce6763ae78a35621d1224064e45d2e8 Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 12 Dec 2024 04:07:20 +0000 Subject: [PATCH 4/8] Update docs --- docs/contribute/01_environment.qmd | 77 ++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/docs/contribute/01_environment.qmd b/docs/contribute/01_environment.qmd index dd163156f210..d0bf2264903a 100644 --- a/docs/contribute/01_environment.qmd +++ b/docs/contribute/01_environment.qmd @@ -22,9 +22,9 @@ architectures. ### Support matrix [^conda-callout] -| Python Version {{< fa arrow-right >}} | Python 3.9 | Python 3.10 | Python 3.11 | +| | Python 3.9 | Python 3.10 | Python 3.11 | | -----------------------------------------: | :--------------------------: | :--------------: | :--------------: | -| **Operating System** {{< fa arrow-down >}} | | | | +| | | | | | **Linux** | {{< fa check >}}[^supported] | {{< fa check >}} | {{< fa check >}} | | **macOS (x86_64)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | | **macOS (aarch64)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | @@ -124,9 +124,9 @@ for manager, params in managers.items(): ### Support matrix -| Python Version {{< fa arrow-right >}} | Python 3.9 | Python 3.10 | Python 3.11 | +| | Python 3.9 | Python 3.10 | Python 3.11 | | -----------------------------------------: | :--------------------------: | :-----------------------: | :-----------------------: | -| **Operating System** {{< fa arrow-down >}} | | | | +| | | | | | **Linux** | {{< fa check >}}[^supported] | {{< fa check >}} | {{< fa check >}} | | **macOS (x86_64)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | | **macOS (arm64/M1/M2)** | {{< fa ban >}}[^m1] | {{< fa ban >}}[^m1] | {{< fa ban >}}[^m1] | @@ -181,6 +181,75 @@ for manager, params in managers.items(): This will launch a `bash` shell with all of the required dependencies installed. This may take a while due to artifact download from the cache. +## Container (uv) + +### Support matrix + +| | Python 3.9 | Python 3.10 | Python 3.11 | +| -----------------------------------------: | :--------------------------: | :-----------------------: | :-----------------------: | +| | | | | +| **Linux** | {{< fa check >}}[^supported] | {{< fa check >}} | {{< fa check >}} | +| **macOS (x86_64)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | +| **macOS (arm64/M1/M2)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | +| **Windows** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | + +1. Install `Docker Desktop` for your platform. + +1. [Install `VS Code`](https://code.visualstudio.com/) + +1. [Install `VS Code Docker Extension`](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker) + +1. [Install `VS Code Dev Containers Extension`](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +1. If using an Apple Silicon Mac, virtualization may be fastest with Colima. + 1. [Install `Colima`](https://github.com/abiosoft/colima) + 2. To use colima for virtualization, `docker context use colima` or `export DOCKER_CONTEXT=colima`. + 3. Verify that the Colima context is in effect with `docker context ls`. + 4. Start Colima: `start colima`. + 5. If you encounter disk resource issues after building images, `colima prune` or `colima delete` may be needed. + + As an alternative to `Colima`, install Rosetta 2. + + ```sh + softwareupdate --install-rosetta + ``` + +1. Build the .devcontainer image from the project directory. (*name:tag* values are flexible.) + + ```sh + docker build -f .devcontainer/Dockerfile . -t ibisproj:1 + ``` + + This will build an image for the build (host) platform by default. + +1. Inspect the ID of the built image: `docker images` (see `IMAGE` column) + +1. Launch a development container with `docker run -itd $IMAGE_ID bash`. + +1. In VS Code, open the project directory. + +1. Menu options for working with devcontainers are available through the blue `'><'` button with tooltip `'Open a Remote Window'`, at the lower left corner of the project window. + + * Use `'Attach to Running Container'` then select the container. + * Other menu options, such as `'Reopen in Container'`, will give less control over naming and tagging. + * Containers continue to run until they are explicitly stopped. Use `docker ps -a` to see + their status. + +1. After pressing any button to close the automatically launched terminal, launch a new + `VS Code` terminal. The project will be in the container as an editable install, + with the working directory `/app`. + +1. Use `uv` commands, such `uv pip list` to show the install packages within the `uv` + .venv. Run non-`uv` commands within the virtual environment using `uv run`, for + example `uv run pytest -m core`. Standard `git` commands are available without + `uv run` because they do not need packages in the .venv to work. + +1. To exit a container, click the `'Dev Container'` button on the lower left of the + window and select the last menu option, `'Close Remote Connection'`. + +1. To ensure you have the latest dependencies from the main branch, based on `pyproject.toml`, + exit any running container, and from your local copy of the repo, + `git pull origin main`. Reopen the project in a new container. ## pip From 7e36fa6df84385758d63fd24f883d387b53d9ea2 Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 12 Dec 2024 04:53:30 +0000 Subject: [PATCH 5/8] Update docs --- docs/contribute/01_environment.qmd | 62 ++++++++++++++---------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/docs/contribute/01_environment.qmd b/docs/contribute/01_environment.qmd index d0bf2264903a..3c74a3fde165 100644 --- a/docs/contribute/01_environment.qmd +++ b/docs/contribute/01_environment.qmd @@ -187,12 +187,14 @@ for manager, params in managers.items(): | | Python 3.9 | Python 3.10 | Python 3.11 | | -----------------------------------------: | :--------------------------: | :-----------------------: | :-----------------------: | -| | | | | -| **Linux** | {{< fa check >}}[^supported] | {{< fa check >}} | {{< fa check >}} | +| | | | | +| **Linux** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | | **macOS (x86_64)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | | **macOS (arm64/M1/M2)** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | | **Windows** | {{< fa check >}} | {{< fa check >}} | {{< fa check >}} | +1. Git clone the project repository. + 1. Install `Docker Desktop` for your platform. 1. [Install `VS Code`](https://code.visualstudio.com/) @@ -203,10 +205,11 @@ for manager, params in managers.items(): 1. If using an Apple Silicon Mac, virtualization may be fastest with Colima. 1. [Install `Colima`](https://github.com/abiosoft/colima) - 2. To use colima for virtualization, `docker context use colima` or `export DOCKER_CONTEXT=colima`. - 3. Verify that the Colima context is in effect with `docker context ls`. - 4. Start Colima: `start colima`. - 5. If you encounter disk resource issues after building images, `colima prune` or `colima delete` may be needed. + 2. Verify that the disk allocation to Colima is satisfactory with `colima template --editor code`. + 3. To use Colima for virtualization, `docker context use colima` or `export DOCKER_CONTEXT=colima`. + 4. Verify that the Colima context is in effect with `docker context ls` (look for '*'). + 5. Start the Colima VM: `start colima`. + 6. If you encounter disk resource issues after building images, `colima prune` or `colima delete` may be needed. As an alternative to `Colima`, install Rosetta 2. @@ -214,42 +217,33 @@ for manager, params in managers.items(): softwareupdate --install-rosetta ``` -1. Build the .devcontainer image from the project directory. (*name:tag* values are flexible.) - - ```sh - docker build -f .devcontainer/Dockerfile . -t ibisproj:1 - ``` - - This will build an image for the build (host) platform by default. - -1. Inspect the ID of the built image: `docker images` (see `IMAGE` column) - -1. Launch a development container with `docker run -itd $IMAGE_ID bash`. +1. In `VS Code`, open the project directory. -1. In VS Code, open the project directory. +1. Menu options for working with devcontainers are available through the blue `'><'` + button, at the lower left corner of the project window. -1. Menu options for working with devcontainers are available through the blue `'><'` button with tooltip `'Open a Remote Window'`, at the lower left corner of the project window. + * Use `'Reopen the container'` to build an image and launch a container. + * Press any button to close the automatically launched terminal. + * Launch a new `VS Code` terminal from the main menu. + * The project will be in the container as an editable install with Ibis library, + dev and test dependencies installed, and with the working directory `/app`. - * Use `'Attach to Running Container'` then select the container. - * Other menu options, such as `'Reopen in Container'`, will give less control over naming and tagging. - * Containers continue to run until they are explicitly stopped. Use `docker ps -a` to see - their status. +1. Use `uv` commands such `uv pip list` to show the installed packages in the `uv` + .venv. -1. After pressing any button to close the automatically launched terminal, launch a new - `VS Code` terminal. The project will be in the container as an editable install, - with the working directory `/app`. - -1. Use `uv` commands, such `uv pip list` to show the install packages within the `uv` - .venv. Run non-`uv` commands within the virtual environment using `uv run`, for +1. Run non-`uv` commands in the virtual environment using `uv run`, for example `uv run pytest -m core`. Standard `git` commands are available without `uv run` because they do not need packages in the .venv to work. 1. To exit a container, click the `'Dev Container'` button on the lower left of the - window and select the last menu option, `'Close Remote Connection'`. - -1. To ensure you have the latest dependencies from the main branch, based on `pyproject.toml`, - exit any running container, and from your local copy of the repo, - `git pull origin main`. Reopen the project in a new container. + window and select the last menu option, `'Close Remote Connection'`. You can reopen + a container and any changes since the last session will persist. + +1. To ensure you have the latest dependencies from the main branch based on + `pyproject.toml`, exit any running container. From your local copy of the repo, + `git pull origin main`. Reopen the project in a new container, then + `'Rebuild the Container'` to copy files from the local git repo and have the build + run `uv sync`. ## pip From 0c80a1fd95a1b4e5646d8c28cb41c1dd2a733e79 Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 12 Dec 2024 05:06:52 +0000 Subject: [PATCH 6/8] Remove old chown code --- .devcontainer/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index bb1635090bd9..b95e39c56cd2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,12 +33,6 @@ RUN chown -R $USERNAME $VENV_DIR && chmod -R 755 $VENV_DIR ENV IBIS_PROJECT=. RUN chown -R $USERNAME $IBIS_PROJECT -# ENV HYPOTHESIS_EXAMPLE_DB_DIR=.hypothesis -# RUN chmod -R 755 $HYPOTHESIS_EXAMPLE_DB_DIR - -# ENV PYTEST_CACHE=.pytest_cache -# RUN chmod -R 755 $PYTEST_CACHE - # Place executables in the environment at the front of the path ENV PATH="/app/.venv/bin:$PATH" From 31cb178a778e68eb589bded608e3e607769ca4f9 Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 12 Dec 2024 05:08:05 +0000 Subject: [PATCH 7/8] Update docs --- docs/contribute/01_environment.qmd | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/contribute/01_environment.qmd b/docs/contribute/01_environment.qmd index 3c74a3fde165..5e35c6043c4f 100644 --- a/docs/contribute/01_environment.qmd +++ b/docs/contribute/01_environment.qmd @@ -236,14 +236,16 @@ for manager, params in managers.items(): `uv run` because they do not need packages in the .venv to work. 1. To exit a container, click the `'Dev Container'` button on the lower left of the - window and select the last menu option, `'Close Remote Connection'`. You can reopen - a container and any changes since the last session will persist. + window and select the last menu option, `'Close Remote Connection'`. 1. To ensure you have the latest dependencies from the main branch based on - `pyproject.toml`, exit any running container. From your local copy of the repo, - `git pull origin main`. Reopen the project in a new container, then - `'Rebuild the Container'` to copy files from the local git repo and have the build - run `uv sync`. + `pyproject.toml`: + + * Exit any running container. + * From your local Git repo, `git pull origin main`. + * Reopen the project in a new container + * `'Rebuild Container'` to copy files from the local Git repo and have the build + run `uv sync`. ## pip From a9b3257ce36244e8c55b5fd89e1733ca0849a4e4 Mon Sep 17 00:00:00 2001 From: "Nicholas A. Brown" Date: Thu, 12 Dec 2024 13:48:12 +0000 Subject: [PATCH 8/8] Fix trailing whitespace --- docs/contribute/01_environment.qmd | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/contribute/01_environment.qmd b/docs/contribute/01_environment.qmd index 5e35c6043c4f..fec107f54a8a 100644 --- a/docs/contribute/01_environment.qmd +++ b/docs/contribute/01_environment.qmd @@ -219,33 +219,33 @@ for manager, params in managers.items(): 1. In `VS Code`, open the project directory. -1. Menu options for working with devcontainers are available through the blue `'><'` +1. Menu options for working with devcontainers are available through the blue `><` button, at the lower left corner of the project window. - * Use `'Reopen the container'` to build an image and launch a container. + * Use `Reopen the container` to build an image and launch a container. * Press any button to close the automatically launched terminal. - * Launch a new `VS Code` terminal from the main menu. - * The project will be in the container as an editable install with Ibis library, - dev and test dependencies installed, and with the working directory `/app`. + * Launch a new `VS Code` terminal from the main menu. + * The project will be in the container as an editable install with Ibis library, + dev and test dependencies installed, and with the working directory `/app`. -1. Use `uv` commands such `uv pip list` to show the installed packages in the `uv` +1. Use `uv` commands such `uv pip list` to show the installed packages in the `uv` .venv. -1. Run non-`uv` commands in the virtual environment using `uv run`, for - example `uv run pytest -m core`. Standard `git` commands are available without +1. Run non-`uv` commands in the virtual environment using `uv run`, for + example `uv run pytest -m core`. Standard `git` commands are available without `uv run` because they do not need packages in the .venv to work. -1. To exit a container, click the `'Dev Container'` button on the lower left of the - window and select the last menu option, `'Close Remote Connection'`. +1. To exit a container, click the `Dev Container` button on the lower left of the + window and select the last menu option, `Close Remote Connection`. -1. To ensure you have the latest dependencies from the main branch based on +1. To ensure you have the latest dependencies from the main branch based on `pyproject.toml`: * Exit any running container. * From your local Git repo, `git pull origin main`. - * Reopen the project in a new container - * `'Rebuild Container'` to copy files from the local Git repo and have the build - run `uv sync`. + * Reopen the project in a new container. + * `Rebuild Container` to copy files from the local Git repo and have the build + run `uv sync`. ## pip