Do not perform multiple source reindexes simultaneously

WordPress · Feb 14, 2024 · a5014aa · a5014aa
1 parent 547ace1
commit a5014aa
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/catalog/dags/common/elasticsearch.py b/catalog/dags/common/elasticsearch.py
@@ -96,8 +96,9 @@ def trigger_and_wait_for_reindex(
     es_host: str,
     max_docs: int | None = None,
     slices: Union[int, Literal["auto"]] = "auto",
+    max_active_tis_per_dagrun: int | None = None,
 ):
-    @task
+    @task(max_active_tis_per_dagrun=max_active_tis_per_dagrun)
     def trigger_reindex(
         es_host: str,
         destination_index: str,
@@ -162,6 +163,7 @@ def _wait_for_reindex(task_id: str, expected_docs: int, es_host: str):
             "expected_docs": max_docs,
             "es_host": es_host,
         },
+        max_active_tis_per_dagrun=max_active_tis_per_dagrun,
     )
 
     trigger_reindex_task >> wait_for_reindex

diff --git a/...e_proportional_by_source_staging_index/create_proportional_by_source_staging_index_dag.py b/...e_proportional_by_source_staging_index/create_proportional_by_source_staging_index_dag.py
@@ -164,6 +164,7 @@ def create_proportional_by_source_staging_index():
         # not work reliably and the final proportions may be incorrect.
         slices=None,
         es_host=es_host,
+        max_active_tis_per_dagrun=1,
     ).expand_kwargs(desired_source_counts)
 
     point_alias = es.point_alias(