Add index check to filtered index

WordPress · Jun 20, 2023 · 8ee2719 · 8ee2719
1 parent 288e069
commit 8ee2719
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/catalog/dags/common/ingestion_server.py b/catalog/dags/common/ingestion_server.py
@@ -148,7 +148,7 @@ def trigger_and_wait_for_task(
 def index_readiness_check(
     media_type: str,
     index_suffix: str,
-    timeout: timedelta,
+    timeout: timedelta = timedelta(days=1),
     poke_interval: int = POKE_INTERVAL,
 ) -> HttpSensor:
     """

diff --git a/catalog/dags/data_refresh/create_filtered_index_dag.py b/catalog/dags/data_refresh/create_filtered_index_dag.py
@@ -206,6 +206,12 @@ def point_alias(destination_index_suffix: str):
             destination_index_suffix=destination_index_suffix,
         )
 
+        # Await healthy results from the newly created elasticsearch index.
+        index_readiness_check = ingestion_server.index_readiness_check(
+            media_type=media_type,
+            index_suffix=destination_index_suffix,
+        )
+
         do_point_alias = point_alias(destination_index_suffix=destination_index_suffix)
 
         delete_old_index = ingestion_server.trigger_task(
@@ -235,7 +241,7 @@ def point_alias(destination_index_suffix: str):
         )
 
         get_current_index_if_exists >> continue_if_no_current_index >> do_create
-        await_create >> do_point_alias
+        await_create >> index_readiness_check >> do_point_alias
 
         [get_current_index_if_exists, do_point_alias] >> delete_old_index
 

diff --git a/catalog/dags/data_refresh/data_refresh_types.py b/catalog/dags/data_refresh/data_refresh_types.py
@@ -39,7 +39,7 @@ class DataRefresh:
     create_materialized_view_timeout:  timedelta expressing amount of time the
                                        creation of the matview may take
     index_readiness_timeout:           timedelta expressing amount of time it may take
-                                       to await a healthy ES cluster after reindexing
+                                       to await a healthy ES index after reindexing
     doc_md:                            str used for the DAG's documentation markdown
     """