From 243f274965a5c675f080cfe88a04e0efe32d2a6c Mon Sep 17 00:00:00 2001 From: Joey LeGrand Date: Fri, 4 Oct 2024 02:41:51 +0000 Subject: [PATCH] Max workers 3 --- airflow/dags/sagerx.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/airflow/dags/sagerx.py b/airflow/dags/sagerx.py index 49ba7b1..4ae5f55 100644 --- a/airflow/dags/sagerx.py +++ b/airflow/dags/sagerx.py @@ -208,7 +208,7 @@ def get_api(url): def parallel_api_calls(api_calls:list) -> list: from concurrent.futures import ThreadPoolExecutor, as_completed output = [] - with ThreadPoolExecutor(max_workers=32) as executor: + with ThreadPoolExecutor(max_workers=3) as executor: futures = {executor.submit(get_api, api_call):api_call for api_call in api_calls} for future in as_completed(futures): @@ -216,6 +216,8 @@ def parallel_api_calls(api_calls:list) -> list: response = future.result() if not len(response) == 0: output.append({"url":url,"response":response}) - else: - print(f"Empty response for url: {url}") + if len(output) % 1000 == 0: + print(f'MILESTONE {len(output)}') + # else: + # print(f"Empty response for url: {url}") return output \ No newline at end of file