From 2d0385116f6deb1adccb75096cc747ae2fb81ead Mon Sep 17 00:00:00 2001 From: Krystle Salazar Date: Thu, 11 Jul 2024 23:18:24 -0400 Subject: [PATCH] Cleanup --- .../database/catalog_cleaner/catalog_cleaner.py | 14 +------------- catalog/dags/database/catalog_cleaner/constants.py | 2 ++ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/catalog/dags/database/catalog_cleaner/catalog_cleaner.py b/catalog/dags/database/catalog_cleaner/catalog_cleaner.py index 9295654af3b..7cd665895cf 100644 --- a/catalog/dags/database/catalog_cleaner/catalog_cleaner.py +++ b/catalog/dags/database/catalog_cleaner/catalog_cleaner.py @@ -27,8 +27,6 @@ logger = logging.getLogger(__name__) -DAG_ID = "catalog_cleaner" - @task def count_dirty_rows(temp_table_name: str, task: AbstractOperator = None): @@ -45,16 +43,6 @@ def count_dirty_rows(temp_table_name: str, task: AbstractOperator = None): @task def get_batches(total_row_count: int, batch_size: int) -> list[tuple[int, int]]: - # batch_list = [] - # batch_start = 0 - # while batch_start <= total_row_count: - # batch_end = min(batch_start + batch_size, total_row_count) - # batch_list.append({ - # "batch_start": batch_start, - # "batch_end": batch_end - # }) - # batch_start += batch_size - # return batch_list return [(i, i + batch_size) for i in range(0, total_row_count, batch_size)] @@ -85,7 +73,7 @@ def update_batch( @dag( - dag_id=DAG_ID, + dag_id=constants.DAG_ID, default_args={ **DAG_DEFAULT_ARGS, "retries": 0, diff --git a/catalog/dags/database/catalog_cleaner/constants.py b/catalog/dags/database/catalog_cleaner/constants.py index 4623fac75a3..e4027772966 100644 --- a/catalog/dags/database/catalog_cleaner/constants.py +++ b/catalog/dags/database/catalog_cleaner/constants.py @@ -1,3 +1,5 @@ +DAG_ID = "catalog_cleaner" + CREATE_SQL = """ DROP TABLE IF EXISTS {temp_table_name}; CREATE UNLOGGED TABLE {temp_table_name} (