Use redis as the Airflow broker (#49)

config/airflow.dev.cfg

@@ -379,7 +379,7 @@ worker_log_server_port = 8793
 # information.
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings
 # broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow
-broker_url = sqla+postgresql+psycopg2://postgres:example@db/airflow-db
+broker_url = redis://airflow-broker
 
 # The Celery result_backend. When a job finishes, it needs to update the
 # metadata of the job. Therefore it will post a message on a message bus,

docker-compose.yml

@@ -34,9 +34,6 @@ services:
       POSTGRES_DB: airflow-db
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 30s
-      timeout: 30s
-      retries: 3
 
 # only used to tag the base image ready to be uploaded to dockerhub
   app:
@@ -48,6 +45,11 @@ services:
     image: libero/jats-ingester:${IMAGE_TAG:-master}
     command: sh
 
+  airflow-broker:
+    image: redis:5.0.5-alpine
+    healthcheck:
+      test: ["CMD", "redis-cli", "PING"]
+
   airflow_initdb:
     &airflow
     build:
@@ -57,8 +59,9 @@ services:
     depends_on:
       - app
       - db
+      - airflow-broker
     environment:
-      WAIT_HOSTS: db:5432
+      WAIT_HOSTS: db:5432, airflow-broker:6379
       AIRFLOW_CONN_REMOTE_LOGS: http://s3:9000?host=http://s3:9000
     volumes:
       - ./dags:/airflow/dags
@@ -72,9 +75,6 @@ services:
       - 8080:8080
     healthcheck:
       test: ["CMD-SHELL", "python ./scripts/airflow_webserver_healthcheck.py"]
-      interval: 30s
-      timeout: 30s
-      retries: 3
 
   airflow_scheduler:
     <<: *airflow

requirements/base.txt

@@ -1,3 +1,3 @@
-apache-airflow[postgres, s3, celery]==1.10.3
+apache-airflow[redis, postgres, s3, celery]==1.10.3
 Wand==0.5.4
 Flask==1.0.3