From 660d7cfa409552e21aac84bbf7cabba1f5fcb60a Mon Sep 17 00:00:00 2001 From: Sjoerd Schipper Date: Tue, 4 Jun 2024 15:59:39 +0200 Subject: [PATCH 1/2] Some :heart: for :whale: --- Dockerfile | 2 +- bin/celery_worker.sh | 19 ++++++++ docker-compose.yml | 80 +++++++++++++++++++++++++++++----- docker-init-db.sql | 7 +++ docker-nginx-default.conf | 14 ++++++ src/open_inwoner/conf/base.py | 7 +++ src/open_inwoner/conf/utils.py | 5 ++- 7 files changed, 122 insertions(+), 12 deletions(-) create mode 100755 bin/celery_worker.sh create mode 100644 docker-init-db.sql create mode 100644 docker-nginx-default.conf diff --git a/Dockerfile b/Dockerfile index c0703565b7..79df344d78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,7 +84,7 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco WORKDIR /app COPY ./bin/docker_start.sh /start.sh -COPY ./bin/start_celery.sh /celery_worker.sh +COPY ./bin/celery_worker.sh /celery_worker.sh COPY ./bin/check_celery_worker_liveness.py ./bin/ RUN mkdir /app/log RUN mkdir /app/media diff --git a/bin/celery_worker.sh b/bin/celery_worker.sh new file mode 100755 index 0000000000..570057f555 --- /dev/null +++ b/bin/celery_worker.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +LOGLEVEL=${CELERY_LOGLEVEL:-INFO} +CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-1} + +QUEUE=${1:-${CELERY_WORKER_QUEUE:=celery}} +WORKER_NAME=${2:-${CELERY_WORKER_NAME:="${QUEUE}"@%n}} + +_binary=$(which celery) + +if [[ "$ENABLE_COVERAGE" ]]; then + _binary="coverage run $_binary" +fi + +echo "Starting celery worker $WORKER_NAME with queue $QUEUE" +exec $_binary --workdir src -A "open_inwoner" events -l info --camera django_celery_monitor.camera.Camera --frequency=2.0 & +exec $_binary --workdir src -A "open_inwoner" worker -l $LOGLEVEL -c $CONCURRENCY -Q $QUEUE -n $WORKER_NAME -E --max-tasks-per-child=50 -l info -B --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/docker-compose.yml b/docker-compose.yml index 889235beba..14aa41b18e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,14 +3,15 @@ version: '3' services: db: - image: postgis/postgis:12-2.5 + image: postgis/postgis:${PG_VERSION:-14-master} container_name: open-inwoner-db environment: - - POSTGRES_USER=${DB_USER:-open_inwoner} - - POSTGRES_PASSWORD=${DB_PASSWORD:-open_inwoner} + - POSTGRES_HOST_AUTH_METHOD=trust volumes: - - ./docker-init-db.sql:/docker-entrypoint-initdb.d - - db_data:/var/lib/postgresql/data + - ./docker-init-db.sql:/docker-entrypoint-initdb.d/init_db.sql + - db:/var/lib/postgresql/data + networks: + - openinwoner-dev elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2 @@ -19,32 +20,91 @@ services: - discovery.type=single-node - cluster.routing.allocation.disk.threshold_enabled=false - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + networks: + - openinwoner-dev volumes: - es_data:/usr/share/elasticsearch/data ports: - ${ES_PORTS:-9201:9201} redis: - image: redis + image: redis:6 + command: ["redis-server", "--appendonly", "yes"] + networks: + - openinwoner-dev + volumes: + - data:/data + + busybox: + image: busybox + command: /bin/chown -R 1000 /private-media + volumes: + - private_media:/private-media web: - build: . + build: &web_build + context: . container_name: open-inwoner-web - environment: + image: maykinmedia/open-inwoner:${TAG:-latest} + environment: &web_env - DJANGO_SETTINGS_MODULE=open_inwoner.conf.docker - SECRET_KEY=${SECRET_KEY:-7bk)w=_%lnm#68rc!c)h@gy&5+%^fl=okq17bv!)yv!l0udu2y} - ALLOWED_HOSTS=* + - CORS_ALLOW_ALL_ORIGINS=${CORS_ALLOW_ALL_ORIGINS:-true} + - CSRF_TRUSTED_ORIGINS='http://localhost:9000' - CACHE_DEFAULT=redis:6379/0 - CACHE_AXES=redis:6379/0 + - CACHE_OIDC=redis:6379/0 + - CACHE_PORTALOCKER=redis:6379/0 + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - CELERY_LOGLEVEL=DEBUG - ES_HOST=elasticsearch - DISABLE_2FA=${DISABLE_2FA:-True} + - DEBUG=True + - IS_HTTPS=no + volumes: &web_volumes + - media:/app/media + - private_media:/app/private_media ports: - 8000:8000 depends_on: - db - redis - - elasticsearch + networks: + - openinwoner-dev + + nginx: + image: nginx + volumes: + - ./docker-nginx-default.conf:/etc/nginx/conf.d/default.conf + - private_media:/private-media + ports: + - '9000:80' + depends_on: + - web + networks: + - openinwoner-dev + + celery: + build: *web_build + image: maykinmedia/open-inwoner:${TAG:-latest} + environment: *web_env + command: /celery_worker.sh + volumes: *web_volumes + depends_on: + - db + - redis + networks: + - openinwoner-dev volumes: - db_data: + db: + media: + private_media: + data: es_data: + +networks: + openinwoner-dev: + name: openinwoner-dev \ No newline at end of file diff --git a/docker-init-db.sql b/docker-init-db.sql new file mode 100644 index 0000000000..925bc2dd60 --- /dev/null +++ b/docker-init-db.sql @@ -0,0 +1,7 @@ +CREATE USER open_inwoner; +CREATE DATABASE open_inwoner; +GRANT ALL PRIVILEGES ON DATABASE open_inwoner TO open_inwoner; +\c open_inwoner; +CREATE EXTENSION postgis; +-- On Postgres 15+, connect to the database and grant schema permissions. +-- GRANT USAGE, CREATE ON SCHEMA public TO openforms; \ No newline at end of file diff --git a/docker-nginx-default.conf b/docker-nginx-default.conf new file mode 100644 index 0000000000..187d4fb77c --- /dev/null +++ b/docker-nginx-default.conf @@ -0,0 +1,14 @@ +server { + listen 80; + server_name localhost; + + location /private-media { + internal; + alias /private-media; + } + + location / { + client_max_body_size 100M; + proxy_pass http://web:8000; + } +} \ No newline at end of file diff --git a/src/open_inwoner/conf/base.py b/src/open_inwoner/conf/base.py index 56a4fd41d2..78d67cdd3a 100644 --- a/src/open_inwoner/conf/base.py +++ b/src/open_inwoner/conf/base.py @@ -747,6 +747,13 @@ CORS_ALLOWED_ORIGINS = [] CORS_ALLOW_CREDENTIALS = True + +CSRF_TRUSTED_ORIGINS = config( + "CSRF_TRUSTED_ORIGINS", + split=True, + default=CORS_ALLOWED_ORIGINS, +) + ACCOUNT_AUTHENTICATION_METHOD = "email" ACCOUNT_USER_MODEL_USERNAME_FIELD = None ACCOUNT_EMAIL_REQUIRED = True diff --git a/src/open_inwoner/conf/utils.py b/src/open_inwoner/conf/utils.py index 956867d2a4..39b8402cfb 100644 --- a/src/open_inwoner/conf/utils.py +++ b/src/open_inwoner/conf/utils.py @@ -5,13 +5,14 @@ from django.conf import settings +from typing import Any from decouple import Csv, config as _config, undefined from sentry_sdk.integrations import DidNotEnable, django, redis logger = logging.getLogger(__name__) -def config(option: str, default=undefined, *args, **kwargs): +def config(option: str, default: Any = undefined, *args, **kwargs): """ Pull a config parameter from the environment. @@ -24,6 +25,8 @@ def config(option: str, default=undefined, *args, **kwargs): if "split" in kwargs: kwargs.pop("split") kwargs["cast"] = Csv() + if isinstance(default, list): + default = ",".join(default) if default is not undefined and default is not None: kwargs.setdefault("cast", type(default)) From 3c5c9d67e1ba9e95db47d5c12e231f943e667eb9 Mon Sep 17 00:00:00 2001 From: Sjoerd Schipper Date: Tue, 4 Jun 2024 17:05:00 +0200 Subject: [PATCH 2/2] Imports are incorrectly sorted and/or formatted. --- src/open_inwoner/conf/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/open_inwoner/conf/utils.py b/src/open_inwoner/conf/utils.py index 39b8402cfb..2356a4ad8e 100644 --- a/src/open_inwoner/conf/utils.py +++ b/src/open_inwoner/conf/utils.py @@ -2,10 +2,10 @@ import os from shutil import which from subprocess import CalledProcessError, check_output +from typing import Any from django.conf import settings -from typing import Any from decouple import Csv, config as _config, undefined from sentry_sdk.integrations import DidNotEnable, django, redis