Merge pull request #504 from JersyJ/503-migration-to-docker

Docker Deployment for Kelvin core

.dockerignore

@@ -1,4 +1,5 @@
 tasks/
 submits/
 submit_results/
-frontend/
+.venv/
+node_modules/

.env.example

@@ -1,9 +1,27 @@
 # PostgreSQL connection information
-DB_HOST=127.0.0.1
-DB_DATABASE=kelvin
-DB_USERNAME=kelvin
-DB_PASSWORD=kelvin
-
-# Path on the local disk where the data from a PostgreSQL instance running
-# inside Docker will be stored
-KELVIN_DB_PATH=./kelvin-data/db
+DATABASE__HOSTNAME=127.0.0.1
+DATABASE__PORT=5432
+DATABASE__DB=kelvin
+DATABASE__USERNAME=kelvin
+DATABASE__PASSWORD=kelvin
+
+# Redis connection information
+REDIS__HOSTNAME=127.0.0.1
+REDIS__PORT=6379
+
+# Data storage paths
+DATABASE__DATA_PATH=./kelvin_data/data
+REDIS__DATA_PATH=./kelvin_data/redis
+
+
+
+# Production only
+
+# Logs paths
+NGINX__LOGS_PATH=./kelvin_data/logs/nginx
+
+# Configuration paths
+NGINX__CERTS_PATH=./deploy/certs
+NGINX__CONFIG_PATH=./deploy/nginx.conf
+UWSGI__CONFIG_PATH=./deploy/uwsgi.ini
+KELVIN__LOCAL_SETTINGS_PATH=./deploy/local_settings.py

.gitignore

@@ -116,3 +116,7 @@ web/static/dolos
 
 survey/surveys/*.yaml
 exams/
+
+static/
+
+kelvin_data/

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+        args: [--allow-multiple-documents]
+      - id: check-json
+      - id: check-toml
+      - id: check-shebang-scripts-are-executable
+      - id: end-of-file-fixer
+        name: Makes sure files end in a newline and only a newline.
+        entry: end-of-file-fixer
+        types: [text]
+      - id: trailing-whitespace
+        name: Trims trailing whitespace.
+        entry: trailing-whitespace-fixer
+        types: [text]
+      - id: mixed-line-ending
+        args: [ --fix=lf ]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.7
+    hooks:
+      - id: ruff-format
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.7
+    hooks:
+      - id: ruff
+        args: [ --fix ]

Dockerfile

@@ -1,22 +1,73 @@
-FROM python:3.12
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm AS build-backend
 
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    libsasl2-dev \
+    libgraphviz-dev \
+    graphviz
 
-RUN apt-get update && apt-get install -y libsasl2-dev libgraphviz-dev graphviz
+WORKDIR /app
 
-COPY requirements.txt /kelvin/requirements.txt
-RUN pip install -r /kelvin/requirements.txt
+COPY pyproject.toml uv.lock ./
 
-RUN apt-get update \
-      && apt-get install -y ca-certificates curl gnupg lsb-release \
-      && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
-      && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list \
-      && apt-get update \
-      && apt-get install -y docker-ce docker-ce-cli containerd.io
+RUN uv sync --frozen --no-dev --no-install-project --compile-bytecode
 
-WORKDIR /kelvin
+FROM node:22.9.0-bookworm-slim AS build-frontend
 
-#RUN addgroup --gid 1000 user
-#RUN adduser --disabled-password --gecos '' --uid $HOST_USER_ID --gid 1000 user
-#USER user
+WORKDIR /frontend
+
+COPY frontend .
+
+RUN mkdir -p /web/static
+
+RUN npm ci
+
+RUN npm run build
+
+FROM python:3.12-bookworm AS runtime
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    graphviz && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /app
+
+# Create new user to run app process as unprivilaged user
+RUN groupadd --gid 102 webserver && \
+    useradd --uid 101 --gid 102 --shell /bin/false --system webserver
+
+RUN chown -R webserver:webserver /app
+
+COPY --from=build-backend --chown=webserver:webserver /app .
+ENV PATH="/app/.venv/bin:$PATH"
+
+COPY --chown=webserver:webserver web ./web
+COPY --from=build-frontend --chown=uvicorn:uvicorn /web/static/frontend.* ./web/static/
+COPY --from=build-frontend --chown=uvicorn:uvicorn /web/static/dolos ./web/static/dolos
+COPY --chown=webserver:webserver kelvin ./kelvin
+COPY --chown=webserver:webserver templates ./templates
+COPY --chown=webserver:webserver evaluator ./evaluator
+COPY --chown=webserver:webserver survey ./survey
+COPY --chown=webserver:webserver common ./common
+COPY --chown=webserver:webserver api ./api
+COPY --chown=webserver:webserver manage.py .
+
+RUN mkdir -p /socket && chown webserver:webserver /socket
+
+USER webserver
+
+RUN python manage.py collectstatic --no-input --clear
+
+EXPOSE 8000
+
+COPY --chown=webserver:webserver deploy/entrypoint.sh ./
+ENTRYPOINT [ "/app/entrypoint.sh" ]
+STOPSIGNAL SIGINT

deploy/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+python manage.py migrate
+
+exec uwsgi --ini uwsgi.ini

deploy/local_settings.py

@@ -0,0 +1,3 @@
+# This file can be used to override settings for deployment
+# e.g.
+# DEBUG=False

deploy/nginx.conf

@@ -0,0 +1,39 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+
+    upstream django {
+        server unix:///socket/kelvin.sock;
+    }
+
+    server {
+        listen 443 ssl;
+        server_name kelvin.cs.vsb.cz;
+        charset     utf-8;
+
+        ssl_certificate /etc/nginx/certs/fullchain.pem;
+        ssl_certificate_key /etc/nginx/certs/privkey.pem;
+
+        location / {
+            uwsgi_pass  django;
+            include uwsgi_params;
+        }
+
+        # Serve static files
+        location /static/ {
+            autoindex on;
+            alias /app/static/;
+            include /etc/nginx/mime.types;
+        }
+
+    }
+
+    # Redirect all HTTP requests to HTTPS
+    server {
+        listen 80;
+        server_name kelvin.cs.vsb.cz;
+        return 301 https://$host$request_uri;
+    }
+}

deploy/uwsgi.ini

@@ -0,0 +1,20 @@
+[uwsgi]
+
+# Django-related settings
+# the base directory (full path)
+chdir           = /app
+# Django's wsgi file
+module          = kelvin.wsgi
+
+# process-related settings
+# master
+master          = true
+# maximum number of worker processes
+processes       = 5
+# the socket (use the full path to be safe
+socket          = /socket/kelvin.sock
+chmod-socket    = 664
+# clear environment on exit
+vacuum          = true
+
+max-requests    = 5000

docker-compose.yml

@@ -1,18 +1,77 @@
-version: "3.9"
 services:
-  db:
-    image: postgres:14
-    ports:
-      - "5432:5432"
+  app:
+    container_name: kelvin_app
+    depends_on:
+      - db
+      - redis
+    profiles: [prod]
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: runtime
+    restart: unless-stopped
+    environment:
+      - DATABASE__HOSTNAME=db
+      - DATABASE__PORT=5432
+      - DATABASE__DB=${DATABASE__DB}
+      - DATABASE__USERNAME=${DATABASE__USERNAME}
+      - DATABASE__PASSWORD=${DATABASE__PASSWORD}
+      - REDIS__HOST=redis
+      - REDIS__PORT=6379
+    env_file:
+      - .env
     volumes:
-      - ${KELVIN_DB_PATH?Absolute path to Kelvin database storage}:/var/lib/postgresql/data
+      - app_static:/app/static
+      - app_socket:/socket
+      - ${KELVIN__LOCAL_SETTINGS_PATH}:/app/kelvin/local_settings.py:ro
+      - ${UWSGI__CONFIG_PATH}:/app/uwsgi.ini:ro
+    expose:
+      - "8000"
+
+  db:
     container_name: kelvin_db
+    image: postgres:14
+    restart: unless-stopped
     environment:
-      - POSTGRES_DB=${DB_DATABASE?Name of PostgreSQL database}
-      - POSTGRES_USER=${DB_USERNAME?PostgreSQL username}
-      - POSTGRES_PASSWORD=${DB_PASSWORD?PostgreSQL password}
-  redis:
-    image: redis
+      - POSTGRES_DB=${DATABASE__DB}
+      - POSTGRES_USER=${DATABASE__USERNAME}
+      - POSTGRES_PASSWORD=${DATABASE__PASSWORD}
+    env_file:
+      - .env
+    volumes:
+      - ${DATABASE__DATA_PATH}:/var/lib/postgresql/data
     ports:
-      - "6379:6379"
+      - "${DATABASE__HOSTNAME}:${DATABASE__PORT}:5432"
+
+  redis:
     container_name: kelvin_redis
+    image: redis:latest
+    restart: unless-stopped
+    env_file:
+      - .env
+    volumes:
+      - ${REDIS__DATA_PATH}:/data
+    ports:
+      - "${REDIS__HOSTNAME}:${REDIS__PORT}:6379"
+    command: "redis-server --save 60 1 --loglevel warning"
+
+  nginx:
+    image: nginx:stable
+    container_name: kelvin_nginx
+    depends_on:
+      - app
+    restart: unless-stopped
+    profiles: [prod]
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ${NGINX__CONFIG_PATH}:/etc/nginx/nginx.conf:ro
+      - ${NGINX__CERTS_PATH}:/etc/nginx/certs:ro
+      - ${NGINX__LOGS_PATH}:/var/log/nginx
+      - app_static:/app/static:ro
+      - app_socket:/socket
+
+volumes:
+  app_static:
+  app_socket:

docs/setup.md

@@ -22,6 +22,12 @@ Do not forget to activate the virtual environment if you want to work with Pytho
 $ source .venv/bin/activate
 ```
 
+## Git pre-commit hooks
+To ensure that the code is formatted correctly and linted, you can install pre-commit hooks:
+```bash
+$ pre-commit install
+```
+
 ## Building the frontend
 For building the frontend, you'll need `npm`:
 ```bash