diff --git a/catalog/dags/common/ingestion_server.py b/catalog/dags/common/ingestion_server.py index 6bc3cb7f553..f3ac3eeff47 100644 --- a/catalog/dags/common/ingestion_server.py +++ b/catalog/dags/common/ingestion_server.py @@ -148,7 +148,7 @@ def trigger_and_wait_for_task( def index_readiness_check( media_type: str, index_suffix: str, - timeout: timedelta, + timeout: timedelta = timedelta(days=1), poke_interval: int = POKE_INTERVAL, ) -> HttpSensor: """ diff --git a/catalog/dags/data_refresh/create_filtered_index_dag.py b/catalog/dags/data_refresh/create_filtered_index_dag.py index b2978376b2c..4597540e9fe 100644 --- a/catalog/dags/data_refresh/create_filtered_index_dag.py +++ b/catalog/dags/data_refresh/create_filtered_index_dag.py @@ -206,6 +206,12 @@ def point_alias(destination_index_suffix: str): destination_index_suffix=destination_index_suffix, ) + # Await healthy results from the newly created elasticsearch index. + index_readiness_check = ingestion_server.index_readiness_check( + media_type=media_type, + index_suffix=destination_index_suffix, + ) + do_point_alias = point_alias(destination_index_suffix=destination_index_suffix) delete_old_index = ingestion_server.trigger_task( @@ -235,7 +241,7 @@ def point_alias(destination_index_suffix: str): ) get_current_index_if_exists >> continue_if_no_current_index >> do_create - await_create >> do_point_alias + await_create >> index_readiness_check >> do_point_alias [get_current_index_if_exists, do_point_alias] >> delete_old_index diff --git a/catalog/dags/data_refresh/data_refresh_types.py b/catalog/dags/data_refresh/data_refresh_types.py index 49643d31161..f54e7c4d33f 100644 --- a/catalog/dags/data_refresh/data_refresh_types.py +++ b/catalog/dags/data_refresh/data_refresh_types.py @@ -39,7 +39,7 @@ class DataRefresh: create_materialized_view_timeout: timedelta expressing amount of time the creation of the matview may take index_readiness_timeout: timedelta expressing amount of time it may take - to await a healthy ES cluster after reindexing + to await a healthy ES index after reindexing doc_md: str used for the DAG's documentation markdown """