Add more tests

deepset-ai · Mar 7, 2024 · b7842b7 · b7842b7
1 parent fed8475
commit b7842b7
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 7 deletions.
diff --git a/haystack/utils/openai_utils.py b/haystack/utils/openai_utils.py
@@ -89,8 +89,6 @@ def _openai_text_completion_tokenization_details(model_name: str):
             or model_name == "gpt-35-turbo-0613"
         ):
             max_tokens_limit = 4096
-        if model_name == "gpt-3.5-turbo-16k" or model_name == "gpt-35-turbo-16k":
-            max_tokens_limit = 16384
 
         # GPT-4 models that have a different token limit than 4096
         # Ref: https://platform.openai.com/docs/models/gpt-4

diff --git a/test/utils/test_openai_utils.py b/test/utils/test_openai_utils.py
@@ -20,21 +20,28 @@
         # Default
         ("not-recognized-name", "cl100k_base", 4096),
         # GPT-3.5
+        ("gpt-3.5-turbo-0125", "cl100k_base", 16385),
         ("gpt-3.5-turbo-instruct", "cl100k_base", 4096),
         ("gpt-3.5-turbo-0613", "cl100k_base", 4096),
         ("gpt-3.5-turbo", "cl100k_base", 16385),
         ("gpt-3.5-turbo-1106", "cl100k_base", 16385),
-        ("gpt-3.5-turbo-16k", "cl100k_base", 16384),
+        ("gpt-3.5-turbo-16k", "cl100k_base", 16385),
+        ("gpt-3.5-turbo-16k-0613", "cl100k_base", 16385),
         # GPT 4
-        ("gpt-4-32k", "cl100k_base", 32768),
-        ("gpt-4-1106", "cl100k_base", 128000),
-        ("gpt-4-turbo-preview", "cl100k_base", 128000),
         ("gpt-4-0125-preview", "cl100k_base", 128000),
+        ("gpt-4-turbo-preview", "cl100k_base", 128000),
+        ("gpt-4-1106-preview", "cl100k_base", 128000),
+        ("gpt-4-vision-preview", "cl100k_base", 128000),
+        ("gpt-4-1106-vision-preview", "cl100k_base", 128000),
         ("gpt-4", "cl100k_base", 8192),
+        ("gpt-4-0613", "cl100k_base", 8192),
+        ("gpt-4-32k", "cl100k_base", 32768),
+        ("gpt-4-32k-0613", "cl100k_base", 32768),
+        ("gpt-4-1106", "cl100k_base", 128000),
         # GPT-35 Azure
         ("gpt-35-turbo-instruct", "cl100k_base", 4096),
         ("gpt-35-turbo", "cl100k_base", 16385),
-        ("gpt-35-turbo-16k", "cl100k_base", 16384),
+        ("gpt-35-turbo-16k", "cl100k_base", 16385),
         # davinci and babbage
         ("davinci-002", "cl100k_base", 16384),
         ("babbage-002", "cl100k_base", 16384),