feat: process stock files before flux files

annuaire-entreprises-data-gouv-fr · Nov 2, 2023 · c1a0450 · c1a0450
1 parent 38fc9f9
commit c1a0450
Showing 1 changed file with 11 additions and 11 deletions.
diff --git a/data_pipelines/rne/database/DAG.py b/data_pipelines/rne/database/DAG.py
@@ -14,7 +14,7 @@
     process_stock_json_files,
     upload_db_to_minio,
     upload_latest_date_rne_minio,
-    notification_mattermost,
+    notification_tchap,
 )
 
 TMP_FOLDER = f"{AIRFLOW_DAG_TMP}rne/"
@@ -24,7 +24,7 @@
     start_date=days_ago(1),
     max_active_runs=1,
     catchup=False,
-    dagrun_timeout=timedelta(minutes=(60 * 8)),
+    dagrun_timeout=timedelta(minutes=(60 * 20)),
     tags=["data_processing", "rne", "dirigeants"],
     params={},
 ) as dag:
@@ -40,28 +40,28 @@
     get_latest_db = PythonOperator(
         task_id="get_latest_db", python_callable=get_latest_db
     )
-    process_flux_json_files = PythonOperator(
-        task_id="process_flux_json_files", python_callable=process_flux_json_files
-    )
     process_stock_json_files = PythonOperator(
         task_id="process_stock_json_files", python_callable=process_stock_json_files
     )
+    process_flux_json_files = PythonOperator(
+        task_id="process_flux_json_files", python_callable=process_flux_json_files
+    )
     upload_db_to_minio = PythonOperator(
         task_id="upload_db_to_minio", python_callable=upload_db_to_minio
     )
     upload_latest_date_rne_minio = PythonOperator(
         task_id="upload_latest_date_rne_minio",
         python_callable=upload_latest_date_rne_minio,
     )
-    notification_mattermost = PythonOperator(
-        task_id="notification_mattermost", python_callable=notification_mattermost
+    notification_tchap = PythonOperator(
+        task_id="notification_tchap", python_callable=notification_tchap
     )
 
     get_start_date.set_upstream(clean_previous_outputs)
     create_db.set_upstream(get_start_date)
     get_latest_db.set_upstream(create_db)
-    process_flux_json_files.set_upstream(get_latest_db)
-    process_stock_json_files.set_upstream(process_flux_json_files)
-    upload_db_to_minio.set_upstream(process_stock_json_files)
+    process_stock_json_files.set_upstream(get_latest_db)
+    process_flux_json_files.set_upstream(process_stock_json_files)
+    upload_db_to_minio.set_upstream(process_flux_json_files)
     upload_latest_date_rne_minio.set_upstream(upload_db_to_minio)
-    notification_mattermost.set_upstream(upload_latest_date_rne_minio)
+    notification_tchap.set_upstream(upload_latest_date_rne_minio)