Run supervisord and our rq worker in dev containers

To ensure asynchronous processes like submission deletion and hashing
happen in dev containers, run supervisord and our rq worker.

This adds a "run_supervisor" function in dev-deps that creates a
supervisor config under /tmp (which just runs the rq worker), and
starts supervisor.

That function is now invoked in securedrop/bin/run.

The Dockerfiles now install supervisor via pip or pip3.

Fixes freedomofpress#4328.

securedrop/bin/dev-deps

@@ -18,6 +18,12 @@ function run_redis() {
     setsid redis-server >& /tmp/redis.out || cat /tmp/redis.out
 }
 
+function run_supervisor() {
+    mkdir -p /tmp/supervisor/log /tmp/supervisor/run && printf "[unix_http_server]\nfile=/tmp/supervisor/run/supervisor.sock\nchmod=0700\n\n[supervisord]logfile=/tmp/supervisor/log/supervisord.log\npidfile=/tmp/supervisor/run/supervisord.pid\nchildlogdir=/tmp/supervisor/log\n[rpcinterface:supervisor]\nsupervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface\n\n[supervisorctl]\nserverurl=unix:///tmp/supervisor/run/supervisor.sock\n\n[program:securedrop_worker]\ncommand=/usr/local/bin/rqworker\ndirectory=$(pwd)\nautostart=true\nautorestart=true\nstartretries=3\nstderr_logfile=/tmp/supervisor/log/securedrop_worker.err\nstdout_logfile=/tmp/supervisor/log/securedrop_worker.out\nuser=%s\n" "$USER_NAME" > /tmp/supervisor/supervisor.conf
+
+    setsid supervisord -c /tmp/supervisor/supervisor.conf >& /tmp/supervisor/log/supervisor.out || cat /tmp/supervisor/log/supervisor.out
+}
+
 function setup_vncauth {
     x11vnc -storepasswd freedom /tmp/vncpasswd
 }

securedrop/bin/run

@@ -7,6 +7,7 @@ source "${BASH_SOURCE%/*}/dev-deps"
 
 run_xvfb &
 run_redis &
+run_supervisor &
 urandom
 run_sass --watch &
 maybe_create_config_py

securedrop/dockerfiles/trusty/python2/Dockerfile

@@ -34,6 +34,8 @@ RUN pip install -U setuptools==40.8.0 && \
     pip install -r requirements/securedrop-app-code-requirements.txt && \
     pip install -r requirements/test-requirements.txt
 
+RUN pip install supervisor
+
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi
 
 STOPSIGNAL SIGKILL

securedrop/dockerfiles/xenial/python2/Dockerfile

@@ -25,12 +25,15 @@ RUN curl -LO https://launchpad.net/~ubuntu-mozilla-security/+archive/ubuntu/ppa/
 RUN gem install sass -v 3.4.23
 
 COPY requirements requirements
+
 RUN pip install -r requirements/securedrop-app-code-requirements.txt && \
     pip install -r requirements/test-requirements.txt
 
 # Fixes #4036 pybabel requires latest version of setuptools
 RUN pip install --upgrade setuptools
 
+RUN pip install supervisor
+
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi
 
 STOPSIGNAL SIGKILL

securedrop/dockerfiles/xenial/python3/Dockerfile

@@ -32,6 +32,8 @@ RUN pip3 install -r requirements/securedrop-app-code-requirements.txt && \
 # Fixes #4036 pybabel requires latest version of setuptools
 RUN pip3 install --upgrade setuptools
 
+RUN pip3 install supervisor
+
 # Temporary workaround: Revert when python 3 is deployed to prod
 RUN sudo rm /usr/bin/python && sudo ln -s /usr/bin/python3 /usr/bin/python