Multiprocess RQ workers (using supervisor)

Dockerfile

@@ -36,7 +36,8 @@ RUN apt-get update && \
     libssl-dev \
     default-libmysqlclient-dev \
     freetds-dev \
-    libsasl2-dev && \
+    libsasl2-dev \
+    supervisor && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*
 

bin/docker-entrypoint

@@ -25,13 +25,22 @@ dev_scheduler() {
 worker() {
   echo "Starting RQ worker..."
 
-  exec /app/manage.py rq worker $QUEUES
+  start_worker
 }
 
 dev_worker() {
   echo "Starting dev RQ worker..."
 
-  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq worker $QUEUES
+  start_worker "watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- "
+}
+
+start_worker() {
+  export EXECUTION_PREFIX=$1
+  export WORKERS_COUNT=${WORKERS_COUNT:-2}
+  export QUEUES=${QUEUES:-}
+
+  touch /tmp/worker.log && tail -f /tmp/worker.log &
+  supervisord -c worker.conf
 }
 
 dev_celery_worker() {

docker-compose.yml

@@ -41,7 +41,6 @@ services:
         target: /app
     depends_on:
       - server
-    tty: true
     environment:
       PYTHONUNBUFFERED: 0
       REDASH_LOG_LEVEL: "INFO"

worker.conf

@@ -0,0 +1,15 @@
+[supervisord]
+logfile=/tmp/supervisord.log
+pidfile=/tmp/supervisord.pid
+nodaemon=true
+
+[program:worker]
+command=%(ENV_EXECUTION_PREFIX)s ./manage.py rq worker %(ENV_QUEUES)s
+process_name=%(program_name)s-%(process_num)s
+numprocs=%(ENV_WORKERS_COUNT)s
+directory=/app
+stopsignal=TERM
+autostart=true
+autorestart=true
+stdout_logfile=/tmp/worker.log
+redirect_stderr=true