fix: rate limit retry with exponential backoff

azure.yaml

@@ -0,0 +1,8 @@
+azure_api_type: azure
+azure_api_base: https://synapseml-openai.openai.azure.com/
+azure_api_version: 2023-03-15-preview
+azure_model_map:
+    turbo_llm_model_deployment_id: gpt-35-turbo
+    smart_llm_model_deployment_id: gpt-4
+    large_llm_model_deployment_id: gpt-4-32k
+    embedding_model_deployment_id: text-embedding-ada-002

src/gpt_review/_openai.py

@@ -1,11 +1,11 @@
 """Open AI API Call Wrapper."""
 import logging
-import time
 
 import openai
 from openai.error import RateLimitError
 
 import gpt_review.constants as C
+from gpt_review.utils import retry_with_exponential_backoff
 from gpt_review.context import _load_azure_openai_context
 
 
@@ -96,12 +96,7 @@ def _call_gpt(
         return completion.choices[0].message.content  # type: ignore
     except RateLimitError as error:
         if retry < C.MAX_RETRIES:
-            logging.warning("Call to GPT failed due to rate limit, retry attempt %s of %s", retry, C.MAX_RETRIES)
-
-            wait_time = int(error.headers["Retry-After"]) if error.headers["Retry-After"] else retry * 10
-            logging.warning("Waiting for %s seconds before retrying.", wait_time)
-
-            time.sleep(wait_time)
+            retry_with_exponential_backoff(retry, error.headers["Retry-After"])
 
             return _call_gpt(prompt, temperature, max_tokens, top_p, frequency_penalty, presence_penalty, retry + 1)
         raise RateLimitError("Retry limit exceeded") from error

src/gpt_review/utils.py

@@ -0,0 +1,20 @@
+"""Utility functions"""
+import logging
+import time
+
+import gpt_review.constants as C
+
+
+def retry_with_exponential_backoff(retry_count, retry_after):
+    """Use exponential backoff to retry a request after specific time while staying under the retry count"""
+    logging.warning("Call to GPT failed due to rate limit, retry attempt %s of %s", retry_count, C.MAX_RETRIES)
+
+    wait_time = int(
+        int(retry_after) * 2 * (1 + retry_count / C.MAX_RETRIES)
+        if retry_after
+        else retry_count * 2 * (1 + retry_count / C.MAX_RETRIES)
+    )
+
+    logging.warning("Waiting for %s seconds before retrying.", wait_time)
+
+    time.sleep(wait_time)