diff --git a/config/airflow.dev.cfg b/config/airflow.dev.cfg index d2b4009..e99db08 100644 --- a/config/airflow.dev.cfg +++ b/config/airflow.dev.cfg @@ -379,7 +379,7 @@ worker_log_server_port = 8793 # information. # http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings # broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow -broker_url = sqla+postgresql+psycopg2://postgres:example@db/airflow-db +broker_url = redis://airflow-broker # The Celery result_backend. When a job finishes, it needs to update the # metadata of the job. Therefore it will post a message on a message bus, diff --git a/docker-compose.yml b/docker-compose.yml index 2a62447..0a5e4cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,9 +34,6 @@ services: POSTGRES_DB: airflow-db healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 30s - timeout: 30s - retries: 3 # only used to tag the base image ready to be uploaded to dockerhub app: @@ -48,6 +45,11 @@ services: image: libero/jats-ingester:${IMAGE_TAG:-master} command: sh + airflow-broker: + image: redis:5.0.5-alpine + healthcheck: + test: ["CMD", "redis-cli", "PING"] + airflow_initdb: &airflow build: @@ -57,8 +59,9 @@ services: depends_on: - app - db + - airflow-broker environment: - WAIT_HOSTS: db:5432 + WAIT_HOSTS: db:5432, airflow-broker:6379 AIRFLOW_CONN_REMOTE_LOGS: http://s3:9000?host=http://s3:9000 volumes: - ./dags:/airflow/dags @@ -72,9 +75,6 @@ services: - 8080:8080 healthcheck: test: ["CMD-SHELL", "python ./scripts/airflow_webserver_healthcheck.py"] - interval: 30s - timeout: 30s - retries: 3 airflow_scheduler: <<: *airflow diff --git a/requirements/base.txt b/requirements/base.txt index c134ee3..d60eba2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,3 +1,3 @@ -apache-airflow[postgres, s3, celery]==1.10.3 +apache-airflow[redis, postgres, s3, celery]==1.10.3 Wand==0.5.4 Flask==1.0.3