From 76e2a0341e4cd7d8beba79488f8740dfc6928875 Mon Sep 17 00:00:00 2001 From: Hadrien Bossard Date: Mon, 23 Dec 2024 11:26:25 +0100 Subject: [PATCH] refactor: use consistent task name --- workflows/data_pipelines/finess/DAG.py | 4 ++-- .../data_pipelines/formation/formation_dag.py | 24 +++++++++---------- workflows/data_pipelines/uai/uai_dag.py | 24 +++++++++---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/workflows/data_pipelines/finess/DAG.py b/workflows/data_pipelines/finess/DAG.py index 8221b709..ffae8aa5 100644 --- a/workflows/data_pipelines/finess/DAG.py +++ b/workflows/data_pipelines/finess/DAG.py @@ -39,7 +39,7 @@ def clean_previous_outputs(): ) @task - def finess_download_data(): + def download_data(): return finess_processor.download_data() @task @@ -60,7 +60,7 @@ def compare_files_minio(): ( clean_previous_outputs() - >> finess_download_data() + >> download_data() >> preprocess_finess() >> save_date_last_modified() >> send_file_to_minio() diff --git a/workflows/data_pipelines/formation/formation_dag.py b/workflows/data_pipelines/formation/formation_dag.py index afe56895..250226ff 100644 --- a/workflows/data_pipelines/formation/formation_dag.py +++ b/workflows/data_pipelines/formation/formation_dag.py @@ -36,36 +36,36 @@ def data_processing_organisme_formation(): formation_processor = FormationProcessor() @task.bash - def formation_clean_previous_outputs(): + def clean_previous_outputs(): return f"rm -rf {FORMATION_CONFIG.tmp_folder} && mkdir -p {FORMATION_CONFIG.tmp_folder}" @task - def formation_download_data(): + def download_data(): return formation_processor.download_data() @task - def formation_preprocess_data(): + def preprocess_data(): return formation_processor.preprocess_data() @task - def formation_save_date_last_modified(): + def save_date_last_modified(): return formation_processor.save_date_last_modified() @task - def formation_send_file_to_minio(): + def send_file_to_minio(): return formation_processor.send_file_to_minio() @task - def formation_compare_files_minio(): + def compare_files_minio(): return formation_processor.compare_files_minio() ( - formation_clean_previous_outputs() - >> formation_download_data() - >> formation_preprocess_data() - >> formation_save_date_last_modified() - >> formation_send_file_to_minio() - >> formation_compare_files_minio() + clean_previous_outputs() + >> download_data() + >> preprocess_data() + >> save_date_last_modified() + >> send_file_to_minio() + >> compare_files_minio() ) diff --git a/workflows/data_pipelines/uai/uai_dag.py b/workflows/data_pipelines/uai/uai_dag.py index 4265d1d7..a6a89505 100644 --- a/workflows/data_pipelines/uai/uai_dag.py +++ b/workflows/data_pipelines/uai/uai_dag.py @@ -36,36 +36,36 @@ def data_processing_uai(): uai_processor = UAIProcessor() @task.bash - def uai_clean_previous_outputs(): + def clean_previous_outputs(): return f"rm -rf {UAI_CONFIG.tmp_folder} && mkdir -p {UAI_CONFIG.tmp_folder}" @task - def uai_download_data(): + def download_data(): return uai_processor.download_data() @task - def uai_preprocess_data(): + def preprocess_data(): return uai_processor.preprocess_data() @task - def uai_save_date_last_modified(): + def save_date_last_modified(): return uai_processor.save_date_last_modified() @task - def uai_send_file_to_minio(): + def send_file_to_minio(): return uai_processor.send_file_to_minio() @task - def uai_compare_files_minio(): + def compare_files_minio(): return uai_processor.compare_files_minio() ( - uai_clean_previous_outputs() - >> uai_download_data() - >> uai_preprocess_data() - >> uai_save_date_last_modified() - >> uai_send_file_to_minio() - >> uai_compare_files_minio() + clean_previous_outputs() + >> download_data() + >> preprocess_data() + >> save_date_last_modified() + >> send_file_to_minio() + >> compare_files_minio() )