diff --git a/.gitignore b/.gitignore index 8fc771a33..1079999c6 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,5 @@ dmypy.json junit_report.xml venv_activation.sh infra/traefik/letsencrypt/acme.json +*.log +infra/airflow/logs/scheduler/latest diff --git a/Makefile b/Makefile index a63903a75..7876bc904 100644 --- a/Makefile +++ b/Makefile @@ -81,8 +81,7 @@ stop-server-services: | stop-traefik stop-portainer create-env-airflow: @ echo "$(BUILD_PRINT) Create Airflow env" @ mkdir -p infra/airflow/logs infra/airflow/plugins -# @ cd infra/airflow/ && ln -s -f ../../dags && ln -s -f ../../ted_sws - @ cd infra/airflow/ && ln -s -f ../../ted_sws + @ cd infra/airflow/ && ln -s -f ../../dags && ln -s -f ../../ted_sws @ echo -e "AIRFLOW_UID=$(CURRENT_UID)" >infra/airflow/.env diff --git a/dags/__init__.py b/dags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infra/airflow/docker-compose.yaml b/infra/airflow/docker-compose.yaml index e735e4e8b..9adf1feef 100644 --- a/infra/airflow/docker-compose.yaml +++ b/infra/airflow/docker-compose.yaml @@ -65,7 +65,7 @@ x-airflow-common: - ./dags:/opt/airflow/dags - ./logs:/opt/airflow/logs - ./plugins:/opt/airflow/plugins - - ./ted_xml_2_rdf:/opt/airflow/ted_sws + - ./ted_sws:/opt/airflow/ted_sws user: "${AIRFLOW_UID:-50000}:0" command: bash -c "export PYTHONPATH='/opt/airflow/'" depends_on: diff --git a/requirements.txt b/requirements.txt index c21b3b55b..b248bc99c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ deepdiff~=5.7.0 git+https://github.com/meaningfy-ws/mfy-data-core python-dotenv~=0.19.2 pymongo~=4.0.1 +apache-airflow==2.2.3 \ No newline at end of file