Merge pull request #77 from hotosm/feat/microservice

Migrate PDM --> uv & add a simple litestar microservice wrapper

.dockerignore

@@ -4,7 +4,8 @@
 # Allow files and directories
 !fmtm_splitter
 !pyproject.toml
-!pdm.lock
+!uv.lock
 !entrypoint.sh
 !README.md
 !LICENSE.md
+!api

.github/labeler.yml

@@ -15,7 +15,7 @@ dependency:
   - changed-files:
       - any-glob-to-any-file:
           - pyproject.toml
-          - pdm.lock
+          - uv.lock
 docs:
   - changed-files:
       - any-glob-to-any-file:

.gitignore

@@ -53,7 +53,7 @@ NEWS
 *.out
 *.app
 
-# PDM
+# PDM (old)
 pdm.toml
 .pdm-python
 __pypackages__

Dockerfile

@@ -1,148 +1,174 @@
-# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) Humanitarian OpenStreetMap Team
 # This file is part of fmtm-splitter.
 #
-#     This program is free software: you can redistribute it and/or modify
+#     fmtm-splitter is free software: you can redistribute it and/or modify
 #     it under the terms of the GNU General Public License as published by
 #     the Free Software Foundation, either version 3 of the License, or
 #     (at your option) any later version.
 #
-#     This program is distributed in the hope that it will be useful,
+#     fmtm-splitter is distributed in the hope that it will be useful,
 #     but WITHOUT ANY WARRANTY; without even the implied warranty of
 #     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #     GNU General Public License for more details.
 #
 #     You should have received a copy of the GNU General Public License
 #     along with fmtm-splitter.  If not, see <https:#www.gnu.org/licenses/>.
 #
-ARG PYTHON_IMG_TAG=3.10
+ARG PYTHON_IMG_TAG=3.12
+ARG UV_IMG_TAG=0.5.2
+FROM ghcr.io/astral-sh/uv:${UV_IMG_TAG} AS uv
 
 
+# Includes all labels and timezone info to extend from
 FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm AS base
+ARG APP_VERSION
 ARG COMMIT_REF
 ARG PYTHON_IMG_TAG
-ARG [email protected]
-LABEL org.hotosm.fmtm-splitter.python-img-tag="${PYTHON_IMG_TAG}" \
-      org.hotosm.fmtm-splitter.commit-ref="${COMMIT_REF}" \
-      org.hotosm.fmtm-splitter.maintainer="${MAINTAINER}"
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends "locales" "ca-certificates" \
+LABEL org.hotosm.fmtm.app-name="fmtm-splitter" \
+      org.hotosm.fmtm.app-version="${APP_VERSION}" \
+      org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \
+      org.hotosm.fmtm.python-img-tag="${PYTHON_IMG_TAG}" \
+      org.hotosm.fmtm.maintainer="[email protected]" \
+      org.hotosm.fmtm.api-port="8000"
+RUN apt-get update --quiet \
+    && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --quiet --no-install-recommends \
+        "locales" "ca-certificates" \
     && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
     && rm -rf /var/lib/apt/lists/* \
     && update-ca-certificates
-# Set locale
+# Set locale & env vars
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-
-
-FROM base AS extract-deps
-WORKDIR /opt/python
-COPY pyproject.toml pdm.lock /opt/python/
-RUN pip install --no-cache-dir --upgrade pip \
-    && pip install --no-cache-dir pdm==2.6.1
-RUN pdm export --prod > requirements.txt \
-    && pdm export -G debug -G test -G docs \
-        --no-default > requirements-ci.txt
-
-
-
-FROM base AS build-wheel
-WORKDIR /build
-COPY . .
-RUN pip install pdm==2.6.1 \
-    && pdm build
-
-
-
+# - Silence uv complaining about not being able to use hard links,
+# - tell uv to byte-compile packages for faster application startups,
+# - prevent uv from accidentally downloading isolated Python builds,
+# - use a temp dir instead of cache during install,
+# - select system python version,
+# - declare `/opt/python` as the target for `uv sync` (i.e. instead of .venv).
+ENV LANG=en_US.UTF-8 \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8 \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_NO_CACHE=1 \
+    UV_PYTHON="python$PYTHON_IMG_TAG" \
+    UV_PROJECT_ENVIRONMENT=/opt/python
+STOPSIGNAL SIGINT
+
+
+# Build stage will all dependencies required to build Python wheels
 FROM base AS build
-WORKDIR /opt/python
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends \
+ARG API
+RUN apt-get update --quiet \
+    && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --quiet --no-install-recommends \
         "build-essential" \
         "gcc" \
-        "libpcre3-dev" \
         "libpq-dev" \
-        "libspatialindex-dev" \
-        "libproj-dev" \
-        "libgeos-dev" \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=extract-deps \
-    /opt/python/requirements.txt /opt/python/
-RUN pip install --user --no-warn-script-location \
-    --no-cache-dir -r ./requirements.txt
-COPY --from=build-wheel \
-    "/build/dist/*-py3-none-any.whl" .
-RUN whl_file=$(find . -name '*-py3-none-any.whl' -type f) \
-    && pip install --user --no-warn-script-location \
-    --no-cache-dir "${whl_file}"
-
-
-
+COPY --from=uv /uv /usr/local/bin/uv
+COPY pyproject.toml uv.lock /_lock/
+# Ensure caching & install with or without api dependencies
+# FIXME add --locked & --no-dev flag to uv sync below
+RUN --mount=type=cache,target=/root/.cache <<EOT
+    uv sync \
+        --project /_lock \
+        --no-dev \
+    $(if [ -z "$API" ]; then \
+        echo ""; \
+    else \
+        echo "--group api"; \
+    fi)
+EOT
+
+
+# Run stage will minimal dependencies required to run Python libraries
 FROM base AS runtime
 ARG PYTHON_IMG_TAG
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONFAULTHANDLER=1 \
-    PATH="/root/.local/bin:$PATH" \
-    PYTHON_LIB="/usr/local/lib/python$PYTHON_IMG_TAG/site-packages" \
+    PATH="/opt/python/bin:$PATH" \
+    PYTHONPATH="/opt" \
+    PYTHON_LIB="/opt/python/lib/python$PYTHON_IMG_TAG/site-packages" \
     SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
     REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
     CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
-RUN set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends \
+RUN apt-get update --quiet \
+    && DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y --quiet --no-install-recommends \
         "nano" \
         "curl" \
-        "libpcre3" \
+        "mime-support" \
         "postgresql-client" \
-        "libglib2.0-0" \
-        "libspatialindex-c6" \
-        "libproj25" \
-        "libgeos-c1v5" \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=build \
-    /root/.local \
-    /root/.local
-WORKDIR /data
 COPY entrypoint.sh /container-entrypoint.sh
+ENTRYPOINT ["/container-entrypoint.sh"]
+WORKDIR /opt
+# Copy Python deps from build to runtime
+COPY --from=build /opt/python /opt/python
+# Add non-root user, permissions
+RUN useradd -u 1001 -m -c "user account" -d /home/appuser -s /bin/false appuser \
+    && chown -R appuser:appuser /opt /home/appuser \
+    && chmod +x /container-entrypoint.sh
 
 
-
-FROM runtime AS ci
-ARG PYTHON_IMG_TAG
-COPY --from=extract-deps \
-    /opt/python/requirements-ci.txt /opt/python/
-RUN cp -r /root/.local/bin/* /usr/local/bin/ \
-    && cp -r /root/.local/lib/python${PYTHON_IMG_TAG}/site-packages/* \
-    /usr/local/lib/python${PYTHON_IMG_TAG}/site-packages/ \
-    && set -ex \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install \
-    -y --no-install-recommends \
-        "git" \
-    && rm -rf /var/lib/apt/lists/* \
-    && pip install --upgrade --no-warn-script-location \
-    --no-cache-dir -r \
-    /opt/python/requirements-ci.txt \
-    && rm -r /opt/python && rm -r /root/.local \
-    # Pre-compile packages to .pyc (init speed gains)
-    && python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+# Stage to use during local development
+FROM runtime AS debug
+ARG API
+COPY --from=uv /uv /usr/local/bin/uv
+COPY pyproject.toml uv.lock /_lock/
+RUN --mount=type=cache,target=/root/.cache <<EOT
+    uv sync \
+        --project /_lock \
+        --group debug \
+        --group test \
+        --group docs \
+        --group dev \
+    $(if [ -z "$API" ]; then \
+        echo ""; \
+    else \
+        echo "--group api"; \
+    fi)
+EOT
+
+
+# Used during CI workflows (as root), with docs/test dependencies pre-installed
+FROM debug AS ci
 # Override entrypoint, as not possible in Github action
 ENTRYPOINT [""]
 CMD [""]
 
 
-
-FROM runtime AS prod
-# Pre-compile packages to .pyc (init speed gains)
-RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)" \
-    && chmod +x /container-entrypoint.sh
-ENTRYPOINT ["/container-entrypoint.sh"]
+# Override CMD for API debug
+FROM debug AS api-debug
+# Add API code & fmtm-splitter module
+COPY api/ /opt/api/
+COPY fmtm_splitter/ /opt/python/lib/python3.12/site-packages/fmtm_splitter/
+CMD ["python", "-Xfrozen_modules=off", "-m", "debugpy", \
+    "--listen", "0.0.0.0:5678", "-m", "uvicorn", "api.main:app", \
+    "--host", "0.0.0.0", "--port", "8000", "--workers", "1", \
+    "--reload", "--log-level", "critical", "--no-access-log"]
+
+
+# Final stage used during API deployment
+FROM runtime AS api-prod
+# Add API code & fmtm-splitter module
+COPY api/ /opt/api/
+COPY fmtm_splitter/ /opt/python/lib/python3.12/site-packages/fmtm_splitter/
+# Change to non-root user
+USER appuser
+# Sanity check to see if build succeeded
+RUN python -V \
+    && python -Im site \
+    && python -c 'import api.main'
+# Note: 1 worker (process) per container, behind load balancer
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000", \
+    "--workers", "1", "--log-level", "critical", "--no-access-log"]
+
+
+# Final stage to distribute fmtm-splitter in a container
+FROM api-prod AS prod
+# Change to non-root user
 CMD ["bash"]

README.md

@@ -196,4 +196,4 @@ fmtm-splitter
 ```
 
 > Note: the `output` directory in this repo is mounted in the container
-> to `/data/output`. To persist data, input and output should be placed here.
+> to `/opt/output`. To persist data, input and output should be placed here.

api/README.md

@@ -0,0 +1,5 @@
+# FMTM Splitter API
+
+A small microservice wrapping the functionality of fmtm-splitter.
+
+- This API uses LiteStar as an alternative to FastAPI.