Merge pull request #1238 from maykinmedia/feature/docker-celery

Some ❤️ for 🐳

Dockerfile

@@ -84,7 +84,7 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
 
 WORKDIR /app
 COPY ./bin/docker_start.sh /start.sh
-COPY ./bin/start_celery.sh /celery_worker.sh
+COPY ./bin/celery_worker.sh /celery_worker.sh
 COPY ./bin/check_celery_worker_liveness.py ./bin/
 RUN mkdir /app/log
 RUN mkdir /app/media

bin/celery_worker.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
+CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-1}
+
+QUEUE=${1:-${CELERY_WORKER_QUEUE:=celery}}
+WORKER_NAME=${2:-${CELERY_WORKER_NAME:="${QUEUE}"@%n}}
+
+_binary=$(which celery)
+
+if [[ "$ENABLE_COVERAGE" ]]; then
+    _binary="coverage run $_binary"
+fi
+
+echo "Starting celery worker $WORKER_NAME with queue $QUEUE"
+exec $_binary --workdir src -A "open_inwoner" events -l info --camera django_celery_monitor.camera.Camera --frequency=2.0 &
+exec $_binary --workdir src -A "open_inwoner" worker -l $LOGLEVEL -c $CONCURRENCY -Q $QUEUE -n $WORKER_NAME -E --max-tasks-per-child=50 -l info -B --scheduler django_celery_beat.schedulers:DatabaseScheduler

docker-compose.yml

@@ -3,14 +3,15 @@ version: '3'
 
 services:
   db:
-    image: postgis/postgis:12-2.5
+    image: postgis/postgis:${PG_VERSION:-14-master}
     container_name: open-inwoner-db
     environment:
-      - POSTGRES_USER=${DB_USER:-open_inwoner}
-      - POSTGRES_PASSWORD=${DB_PASSWORD:-open_inwoner}
+      - POSTGRES_HOST_AUTH_METHOD=trust
     volumes:
-      - ./docker-init-db.sql:/docker-entrypoint-initdb.d
-      - db_data:/var/lib/postgresql/data
+      - ./docker-init-db.sql:/docker-entrypoint-initdb.d/init_db.sql
+      - db:/var/lib/postgresql/data
+    networks:
+      - openinwoner-dev    
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2
@@ -19,32 +20,91 @@ services:
       - discovery.type=single-node
       - cluster.routing.allocation.disk.threshold_enabled=false
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    networks:
+      - openinwoner-dev         
     volumes:
       - es_data:/usr/share/elasticsearch/data
     ports:
       - ${ES_PORTS:-9201:9201}
 
   redis:
-    image: redis
+    image: redis:6
+    command: ["redis-server", "--appendonly", "yes"]
+    networks:
+      - openinwoner-dev
+    volumes:
+      - data:/data
+
+  busybox:
+    image: busybox
+    command: /bin/chown -R 1000 /private-media
+    volumes:
+      - private_media:/private-media
 
   web:
-    build: .
+    build: &web_build
+      context: .
     container_name: open-inwoner-web
-    environment:
+    image: maykinmedia/open-inwoner:${TAG:-latest}
+    environment: &web_env
       - DJANGO_SETTINGS_MODULE=open_inwoner.conf.docker
       - SECRET_KEY=${SECRET_KEY:-7bk)w=_%lnm#68rc!c)h@gy&5+%^fl=okq17bv!)yv!l0udu2y}
       - ALLOWED_HOSTS=*
+      - CORS_ALLOW_ALL_ORIGINS=${CORS_ALLOW_ALL_ORIGINS:-true}
+      - CSRF_TRUSTED_ORIGINS='http://localhost:9000'
       - CACHE_DEFAULT=redis:6379/0
       - CACHE_AXES=redis:6379/0
+      - CACHE_OIDC=redis:6379/0
+      - CACHE_PORTALOCKER=redis:6379/0
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - CELERY_LOGLEVEL=DEBUG      
       - ES_HOST=elasticsearch
       - DISABLE_2FA=${DISABLE_2FA:-True}
+      - DEBUG=True
+      - IS_HTTPS=no
+    volumes: &web_volumes
+      - media:/app/media
+      - private_media:/app/private_media
     ports:
       - 8000:8000
     depends_on:
       - db
       - redis
-      - elasticsearch
+    networks:
+      - openinwoner-dev
+
+  nginx:
+    image: nginx
+    volumes:
+      - ./docker-nginx-default.conf:/etc/nginx/conf.d/default.conf
+      - private_media:/private-media
+    ports:
+      - '9000:80'
+    depends_on:
+      - web
+    networks:
+      - openinwoner-dev
+
+  celery:
+    build: *web_build
+    image: maykinmedia/open-inwoner:${TAG:-latest}
+    environment: *web_env
+    command: /celery_worker.sh
+    volumes: *web_volumes
+    depends_on:
+      - db
+      - redis
+    networks:
+      - openinwoner-dev
 
 volumes:
-  db_data:
+  db:
+  media:
+  private_media:
+  data:
   es_data:
+
+networks:
+  openinwoner-dev:
+    name: openinwoner-dev

docker-init-db.sql

@@ -0,0 +1,7 @@
+CREATE USER open_inwoner;
+CREATE DATABASE open_inwoner;
+GRANT ALL PRIVILEGES ON DATABASE open_inwoner TO open_inwoner;
+\c open_inwoner;
+CREATE EXTENSION postgis;
+-- On Postgres 15+, connect to the database and grant schema permissions.
+-- GRANT USAGE, CREATE ON SCHEMA public TO openforms;

docker-nginx-default.conf

@@ -0,0 +1,14 @@
+server {
+    listen       80;
+    server_name  localhost;
+
+    location /private-media {
+        internal;
+        alias /private-media;
+    }
+
+    location / {
+        client_max_body_size 100M;
+        proxy_pass   http://web:8000;
+    }
+}

src/open_inwoner/conf/base.py

@@ -747,6 +747,13 @@
 CORS_ALLOWED_ORIGINS = []
 CORS_ALLOW_CREDENTIALS = True
 
+
+CSRF_TRUSTED_ORIGINS = config(
+    "CSRF_TRUSTED_ORIGINS",
+    split=True,
+    default=CORS_ALLOWED_ORIGINS,
+)
+
 ACCOUNT_AUTHENTICATION_METHOD = "email"
 ACCOUNT_USER_MODEL_USERNAME_FIELD = None
 ACCOUNT_EMAIL_REQUIRED = True

src/open_inwoner/conf/utils.py

@@ -2,6 +2,7 @@
 import os
 from shutil import which
 from subprocess import CalledProcessError, check_output
+from typing import Any
 
 from django.conf import settings
 
@@ -11,7 +12,7 @@
 logger = logging.getLogger(__name__)
 
 
-def config(option: str, default=undefined, *args, **kwargs):
+def config(option: str, default: Any = undefined, *args, **kwargs):
     """
     Pull a config parameter from the environment.
 
@@ -24,6 +25,8 @@ def config(option: str, default=undefined, *args, **kwargs):
     if "split" in kwargs:
         kwargs.pop("split")
         kwargs["cast"] = Csv()
+        if isinstance(default, list):
+            default = ",".join(default)
 
     if default is not undefined and default is not None:
         kwargs.setdefault("cast", type(default))