From 1a92d39df1147be8d02f58f228a5ce1a2a417235 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 22 Mar 2019 14:12:42 +0100 Subject: [PATCH 1/4] Fix Celery worker CLI parameter name that was changed in Celery 4.x. --- bin/docker-entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index c7d0e473b6..40b7cfd658 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -6,7 +6,7 @@ worker() { QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas} echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair } scheduler() { @@ -16,7 +16,7 @@ scheduler() { echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair } server() { From 9ec75bc7b185bf1f3ce08991bd6c2ade51466ab6 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 22 Mar 2019 14:13:29 +0100 Subject: [PATCH 2/4] Set Celery worker --max-memory-per-child to 1/4th of total system memory. --- bin/docker-entrypoint | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 40b7cfd658..a4a9bca70f 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -4,9 +4,10 @@ set -e worker() { WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas} + MAX_MEMORY=$(($(/usr/bin/awk '/MemTotal/ {print $2}' /proc/meminfo)/4)) echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 --max-memory-per-child=$MAX_MEMORY -Ofair } scheduler() { From d0397f31267d50e09b40be757895233ba8779593 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 27 Mar 2019 15:28:44 +0100 Subject: [PATCH 3/4] Review fixes. --- bin/docker-entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index a4a9bca70f..dd0296c655 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -4,10 +4,10 @@ set -e worker() { WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas} - MAX_MEMORY=$(($(/usr/bin/awk '/MemTotal/ {print $2}' /proc/meminfo)/4)) + WORKER_EXTRA_OPTIONS=${WORKER_EXTRA_OPTIONS:} echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 --max-memory-per-child=$MAX_MEMORY -Ofair + exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair $WORKER_EXTRA_OPTIONS } scheduler() { From e2b5ea75648d1236b90d6b7315686e21349e573e Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 27 Mar 2019 18:01:55 +0100 Subject: [PATCH 4/4] Review fixes. --- bin/docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index dd0296c655..62d80e9526 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -4,7 +4,7 @@ set -e worker() { WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas} - WORKER_EXTRA_OPTIONS=${WORKER_EXTRA_OPTIONS:} + WORKER_EXTRA_OPTIONS=${WORKER_EXTRA_OPTIONS:-} echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair $WORKER_EXTRA_OPTIONS