From 003f5093588cf1a0b467784d2aca7ed6028524f7 Mon Sep 17 00:00:00 2001 From: Takuya ASADA Date: Sun, 17 Mar 2024 03:59:27 +0900 Subject: [PATCH] curl: raise exception when HTTP error code is client error When curl catched HTTP error 4xx = client error such as "404 Not Found" or "403 Forbidden", it likely our script bug not server side issue, we should re-raise exception immediately instead of retrying. related #505 --- lib/scylla_cloud.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/scylla_cloud.py b/lib/scylla_cloud.py index 29109b0d..3e9d1c96 100644 --- a/lib/scylla_cloud.py +++ b/lib/scylla_cloud.py @@ -73,7 +73,10 @@ def curl(url, headers=None, method=None, byte=False, timeout=3, max_retries=5, r while True: try: return _curl_one(url, headers, method, byte, timeout) - except (urllib.error.URLError, socket.timeout): + except (urllib.error.URLError, socket.timeout) as e: + # if HTTP error code is client error (400-499), skip retrying + if isinstance(e, urllib.error.HTTPError) and e.code in range(400,500): + raise time.sleep(retry_interval) retries += 1 if retries >= max_retries: @@ -84,7 +87,10 @@ async def aiocurl(url, headers=None, method=None, byte=False, timeout=3, max_ret while True: try: return _curl_one(url, headers, method, byte, timeout) - except (urllib.error.URLError, socket.timeout): + except (urllib.error.URLError, socket.timeout) as e: + # if HTTP error code is client error (400-499), skip retrying + if isinstance(e, urllib.error.HTTPError) and e.code in range(400,500): + raise await asyncio.sleep(retry_interval) retries += 1 if retries >= max_retries: