From c449fea34d59e7818c5d8a516e35e26269cc07cc Mon Sep 17 00:00:00 2001 From: Sam Goldman Date: Sun, 14 Jul 2024 15:44:59 -0400 Subject: [PATCH] fix: Update models to include whether they support streaming, and add validation to check whether a model supports streaming --- .changeset/short-seahorses-fix.md | 5 ++ docs/providers/ai21.md | 2 +- docs/providers/anthropic.md | 14 ++-- docs/providers/bedrock.md | 38 +++++------ docs/providers/cohere.md | 12 ++-- docs/providers/gemini.md | 6 +- docs/providers/groq.md | 10 +-- docs/providers/mistral.md | 28 ++++---- docs/providers/openai.md | 42 ++++++------ docs/providers/perplexity.md | 14 ++-- src/handlers/base.ts | 19 +++++- src/handlers/utils.ts | 27 +++++--- src/models.ts | 108 +++++++++++++++++++++++++++--- test/handlers/base.test.ts | 25 ++++++- test/handlers/gemini.test.ts | 31 ++------- test/handlers/mistral.test.ts | 11 +-- 16 files changed, 252 insertions(+), 140 deletions(-) create mode 100644 .changeset/short-seahorses-fix.md diff --git a/.changeset/short-seahorses-fix.md b/.changeset/short-seahorses-fix.md new file mode 100644 index 0000000..4ad265a --- /dev/null +++ b/.changeset/short-seahorses-fix.md @@ -0,0 +1,5 @@ +--- +'token.js': patch +--- + +Update models to include whether they support streaming, and add validation to check whether a model supports streaming diff --git a/docs/providers/ai21.md b/docs/providers/ai21.md index 8a51e60..0e8725d 100644 --- a/docs/providers/ai21.md +++ b/docs/providers/ai21.md @@ -40,7 +40,7 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | -------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| jamba-instruct | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | +| jamba-instruct | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | ### Legend | Symbol | Description | diff --git a/docs/providers/anthropic.md b/docs/providers/anthropic.md index 4d026b7..96aac0e 100644 --- a/docs/providers/anthropic.md +++ b/docs/providers/anthropic.md @@ -40,13 +40,13 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | -------------------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| claude-3-5-sonnet-20240620 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| claude-3-opus-20240229 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| claude-3-sonnet-20240229 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| claude-3-haiku-20240307 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| claude-2.1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| claude-2.0 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| claude-instant-1.2 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | +| claude-3-5-sonnet-20240620 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| claude-3-opus-20240229 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| claude-3-sonnet-20240229 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| claude-3-haiku-20240307 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| claude-2.1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| claude-2.0 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| claude-instant-1.2 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | ### Legend | Symbol | Description | diff --git a/docs/providers/bedrock.md b/docs/providers/bedrock.md index 7fb4259..5ad9581 100644 --- a/docs/providers/bedrock.md +++ b/docs/providers/bedrock.md @@ -42,25 +42,25 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | --------------------------------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| amazon.titan-text-lite-v1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| amazon.titan-text-express-v1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| anthropic.claude-3-opus-20240229-v1:0 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| anthropic.claude-3-sonnet-20240229-v1:0 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| anthropic.claude-3-haiku-20240307-v1:0 | ✅ | ➖ | ➖ | ✅ | ✅ | ➖ | -| anthropic.claude-v2:1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| anthropic.claude-v2 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| anthropic.claude-instant-v1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| cohere.command-r-plus-v1:0 | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| cohere.command-r-v1:0 | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| cohere.command-text-v14 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| cohere.command-light-text-v14 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| meta.llama3-8b-instruct-v1:0 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| meta.llama3-70b-instruct-v1:0 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| meta.llama2-13b-chat-v1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| meta.llama2-70b-chat-v1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mistral.mistral-7b-instruct-v0:2 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mistral.mixtral-8x7b-instruct-v0:1 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mistral.mistral-large-2402-v1:0 | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | +| amazon.titan-text-lite-v1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| amazon.titan-text-express-v1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| anthropic.claude-3-opus-20240229-v1:0 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| anthropic.claude-3-sonnet-20240229-v1:0 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| anthropic.claude-3-haiku-20240307-v1:0 | ✅ | ✅ | ➖ | ✅ | ✅ | ➖ | +| anthropic.claude-v2:1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| anthropic.claude-v2 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| anthropic.claude-instant-v1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| cohere.command-r-plus-v1:0 | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| cohere.command-r-v1:0 | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| cohere.command-text-v14 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| cohere.command-light-text-v14 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| meta.llama3-8b-instruct-v1:0 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| meta.llama3-70b-instruct-v1:0 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| meta.llama2-13b-chat-v1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| meta.llama2-70b-chat-v1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mistral.mistral-7b-instruct-v0:2 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mistral.mixtral-8x7b-instruct-v0:1 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mistral.mistral-large-2402-v1:0 | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | ### Legend | Symbol | Description | diff --git a/docs/providers/cohere.md b/docs/providers/cohere.md index 33f336c..cb12a35 100644 --- a/docs/providers/cohere.md +++ b/docs/providers/cohere.md @@ -40,12 +40,12 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | --------------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| command-r-plus | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| command-r | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| command | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| command-nightly | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| command-light | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| command-light-nightly | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | +| command-r-plus | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| command-r | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| command | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| command-nightly | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| command-light | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| command-light-nightly | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | ### Legend | Symbol | Description | diff --git a/docs/providers/gemini.md b/docs/providers/gemini.md index debc1a1..d643d11 100644 --- a/docs/providers/gemini.md +++ b/docs/providers/gemini.md @@ -40,9 +40,9 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | ---------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| gemini-1.5-pro | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gemini-1.5-flash | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gemini-1.0-pro | ✅ | ➖ | ➖ | ➖ | ✅ | ✅ | +| gemini-1.5-pro | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gemini-1.5-flash | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gemini-1.0-pro | ✅ | ✅ | ➖ | ➖ | ✅ | ✅ | ### Legend | Symbol | Description | diff --git a/docs/providers/groq.md b/docs/providers/groq.md index 2fbb437..616f592 100644 --- a/docs/providers/groq.md +++ b/docs/providers/groq.md @@ -40,11 +40,11 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | ------------------ | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| llama3-8b-8192 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| llama3-70b-8192 | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | -| mixtral-8x7b-32768 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| gemma-7b-it | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | -| gemma2-9b-it | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | +| llama3-8b-8192 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| llama3-70b-8192 | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | +| mixtral-8x7b-32768 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| gemma-7b-it | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | +| gemma2-9b-it | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | ### Legend | Symbol | Description | diff --git a/docs/providers/mistral.md b/docs/providers/mistral.md index a1796ce..e077d34 100644 --- a/docs/providers/mistral.md +++ b/docs/providers/mistral.md @@ -40,20 +40,20 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | ----------------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| open-mistral-7b | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | -| mistral-tiny-2312 | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | -| open-mixtral-8x7b | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mistral-small-2312 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| open-mixtral-8x22b | ✅ | ➖ | ✅ | ➖ | ✅ | ➖ | -| open-mixtral-8x22b-2404 | ✅ | ➖ | ✅ | ➖ | ✅ | ➖ | -| mistral-small-latest | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| mistral-small-2402 | ✅ | ➖ | ➖ | ➖ | ✅ | ➖ | -| mistral-medium-latest | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mistral-medium-2312 | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mistral-large-latest | ✅ | ➖ | ✅ | ➖ | ✅ | ➖ | -| mistral-large-2402 | ✅ | ➖ | ✅ | ➖ | ✅ | ➖ | -| codestral-latest | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | -| codestral-2405 | ✅ | ➖ | ✅ | ➖ | ➖ | ➖ | +| open-mistral-7b | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | +| mistral-tiny-2312 | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | +| open-mixtral-8x7b | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mistral-small-2312 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| open-mixtral-8x22b | ✅ | ✅ | ✅ | ➖ | ✅ | ➖ | +| open-mixtral-8x22b-2404 | ✅ | ✅ | ✅ | ➖ | ✅ | ➖ | +| mistral-small-latest | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| mistral-small-2402 | ✅ | ✅ | ➖ | ➖ | ✅ | ➖ | +| mistral-medium-latest | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mistral-medium-2312 | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mistral-large-latest | ✅ | ✅ | ✅ | ➖ | ✅ | ➖ | +| mistral-large-2402 | ✅ | ✅ | ✅ | ➖ | ✅ | ➖ | +| codestral-latest | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | +| codestral-2405 | ✅ | ✅ | ✅ | ➖ | ➖ | ➖ | ### Legend | Symbol | Description | diff --git a/docs/providers/openai.md b/docs/providers/openai.md index 9143348..0b7ad78 100644 --- a/docs/providers/openai.md +++ b/docs/providers/openai.md @@ -40,27 +40,27 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | ---------------------- | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| gpt-4o | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4o-2024-05-13 | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4-turbo | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4-turbo-2024-04-09 | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4-0125-preview | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4-turbo-preview | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4-1106-preview | ✅ | ➖ | ✅ | ✅ | ✅ | ✅ | -| gpt-4-vision-preview | ✅ | ➖ | ✅ | ✅ | ➖ | ✅ | -| gpt-4 | ✅ | ➖ | ➖ | ➖ | ✅ | ✅ | -| gpt-4-0314 | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | -| gpt-4-0613 | ✅ | ➖ | ➖ | ➖ | ✅ | ✅ | -| gpt-4-32k | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | -| gpt-4-32k-0314 | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | -| gpt-4-32k-0613 | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | -| gpt-3.5-turbo | ✅ | ➖ | ✅ | ➖ | ✅ | ✅ | -| gpt-3.5-turbo-16k | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | -| gpt-3.5-turbo-0301 | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | -| gpt-3.5-turbo-0613 | ✅ | ➖ | ➖ | ➖ | ✅ | ✅ | -| gpt-3.5-turbo-1106 | ✅ | ➖ | ✅ | ➖ | ✅ | ✅ | -| gpt-3.5-turbo-0125 | ✅ | ➖ | ✅ | ➖ | ✅ | ✅ | -| gpt-3.5-turbo-16k-0613 | ✅ | ➖ | ➖ | ➖ | ➖ | ✅ | +| gpt-4o | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4o-2024-05-13 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4-turbo | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4-turbo-2024-04-09 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4-0125-preview | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4-turbo-preview | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4-1106-preview | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| gpt-4-vision-preview | ✅ | ✅ | ✅ | ✅ | ➖ | ✅ | +| gpt-4 | ✅ | ✅ | ➖ | ➖ | ✅ | ✅ | +| gpt-4-0314 | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | +| gpt-4-0613 | ✅ | ✅ | ➖ | ➖ | ✅ | ✅ | +| gpt-4-32k | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | +| gpt-4-32k-0314 | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | +| gpt-4-32k-0613 | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | +| gpt-3.5-turbo | ✅ | ✅ | ✅ | ➖ | ✅ | ✅ | +| gpt-3.5-turbo-16k | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | +| gpt-3.5-turbo-0301 | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | +| gpt-3.5-turbo-0613 | ✅ | ✅ | ➖ | ➖ | ✅ | ✅ | +| gpt-3.5-turbo-1106 | ✅ | ✅ | ✅ | ➖ | ✅ | ✅ | +| gpt-3.5-turbo-0125 | ✅ | ✅ | ✅ | ➖ | ✅ | ✅ | +| gpt-3.5-turbo-16k-0613 | ✅ | ✅ | ➖ | ➖ | ➖ | ✅ | ### Legend | Symbol | Description | diff --git a/docs/providers/perplexity.md b/docs/providers/perplexity.md index fb5af3e..f1a1375 100644 --- a/docs/providers/perplexity.md +++ b/docs/providers/perplexity.md @@ -40,13 +40,13 @@ main() | Model | Chat Completion | Streaming | JSON Output | Image Input | Function Calling | N > 1 | | ------------------------------ | --------------- | --------- | ----------- | ----------- | ---------------- | ----- | -| llama-3-sonar-small-32k-chat | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| llama-3-sonar-small-32k-online | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| llama-3-sonar-large-32k-chat | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| llama-3-sonar-large-32k-online | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| llama-3-8b-instruct | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| llama-3-70b-instruct | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | -| mixtral-8x7b-instruct | ✅ | ➖ | ➖ | ➖ | ➖ | ➖ | +| llama-3-sonar-small-32k-chat | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| llama-3-sonar-small-32k-online | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| llama-3-sonar-large-32k-chat | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| llama-3-sonar-large-32k-online | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| llama-3-8b-instruct | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| llama-3-70b-instruct | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | +| mixtral-8x7b-instruct | ✅ | ✅ | ➖ | ➖ | ➖ | ➖ | ### Legend | Symbol | Description | diff --git a/src/handlers/base.ts b/src/handlers/base.ts index e1bbc43..f86a0fa 100644 --- a/src/handlers/base.ts +++ b/src/handlers/base.ts @@ -13,6 +13,7 @@ export abstract class BaseHandler { protected supportsImages: readonly T[] protected supportsToolCalls: readonly T[] protected supportsN: readonly T[] | boolean + protected supportsStreamingMessages: readonly T[] constructor( opts: ConfigOptions, @@ -20,7 +21,8 @@ export abstract class BaseHandler { supportsJSON: readonly T[], supportsImages: readonly T[], supportsToolCalls: readonly T[], - suportsN: readonly T[] | boolean + suportsN: readonly T[] | boolean, + supportsStreamingMessages: readonly T[] ) { this.opts = opts this.models = models @@ -28,6 +30,7 @@ export abstract class BaseHandler { this.supportsImages = supportsImages this.supportsToolCalls = supportsToolCalls this.supportsN = suportsN + this.supportsStreamingMessages = supportsStreamingMessages } abstract create( @@ -39,6 +42,12 @@ export abstract class BaseHandler { throw new InputError(`Invalid 'model' field: ${body.model}.`) } + if (!this.supportsStreaming(body.model)) { + throw new Error( + `Detected 'stream: true', but the following model does not support streaming: ${body.model}` + ) + } + if (body.tools !== undefined && !this.supportsTools(body.model)) { throw new InputError( `Detected a 'tools' parameter, but the following model does not support tools: ${body.model}` @@ -138,7 +147,9 @@ export abstract class BaseHandler { } } - protected isSupportedModel(model: LLMChatModel): model is T { + // We make this public so that we can mock it in tests, which is fine because the `BaseHandler` + // class isn't exposed to the user. + public isSupportedModel(model: LLMChatModel): model is T { return this.models.includes(model as T) } @@ -157,4 +168,8 @@ export abstract class BaseHandler { protected supportsTools(model: T): boolean { return this.isSupportedFeature(this.supportsToolCalls, model) } + + protected supportsStreaming(model: T): boolean { + return this.isSupportedFeature(this.supportsStreamingMessages, model) + } } diff --git a/src/handlers/utils.ts b/src/handlers/utils.ts index 1d2dedb..a4f44ab 100644 --- a/src/handlers/utils.ts +++ b/src/handlers/utils.ts @@ -25,7 +25,8 @@ export const Handlers: Record any> = { models.openai.supportsJSON, models.openai.supportsImages, models.openai.supportsToolCalls, - models.openai.supportsN + models.openai.supportsN, + models.openai.supportsStreaming ), ['anthropic']: (opts: ConfigOptions) => new AnthropicHandler( @@ -34,7 +35,8 @@ export const Handlers: Record any> = { models.anthropic.supportsJSON, models.anthropic.supportsImages, models.anthropic.supportsToolCalls, - models.anthropic.supportsN + models.anthropic.supportsN, + models.anthropic.supportsStreaming ), ['gemini']: (opts: ConfigOptions) => new GeminiHandler( @@ -43,7 +45,8 @@ export const Handlers: Record any> = { models.gemini.supportsJSON, models.gemini.supportsImages, models.gemini.supportsToolCalls, - models.gemini.supportsN + models.gemini.supportsN, + models.gemini.supportsStreaming ), ['cohere']: (opts: ConfigOptions) => new CohereHandler( @@ -52,7 +55,8 @@ export const Handlers: Record any> = { models.cohere.supportsJSON, models.cohere.supportsImages, models.cohere.supportsToolCalls, - models.cohere.supportsN + models.cohere.supportsN, + models.cohere.supportsStreaming ), ['bedrock']: (opts: ConfigOptions) => new BedrockHandler( @@ -61,7 +65,8 @@ export const Handlers: Record any> = { models.bedrock.supportsJSON, models.bedrock.supportsImages, models.bedrock.supportsToolCalls, - models.bedrock.supportsN + models.bedrock.supportsN, + models.bedrock.supportsStreaming ), ['mistral']: (opts: ConfigOptions) => new MistralHandler( @@ -70,7 +75,8 @@ export const Handlers: Record any> = { models.mistral.supportsJSON, models.mistral.supportsImages, models.mistral.supportsToolCalls, - models.mistral.supportsN + models.mistral.supportsN, + models.mistral.supportsStreaming ), ['groq']: (opts: ConfigOptions) => new GroqHandler( @@ -79,7 +85,8 @@ export const Handlers: Record any> = { models.groq.supportsJSON, models.groq.supportsImages, models.groq.supportsToolCalls, - models.groq.supportsN + models.groq.supportsN, + models.groq.supportsStreaming ), ['ai21']: (opts: ConfigOptions) => new AI21Handler( @@ -88,7 +95,8 @@ export const Handlers: Record any> = { models.ai21.supportsJSON, models.ai21.supportsImages, models.ai21.supportsToolCalls, - models.ai21.supportsN + models.ai21.supportsN, + models.ai21.supportsStreaming ), ['perplexity']: (opts: ConfigOptions) => new PerplexityHandler( @@ -97,7 +105,8 @@ export const Handlers: Record any> = { models.perplexity.supportsJSON, models.perplexity.supportsImages, models.perplexity.supportsToolCalls, - models.perplexity.supportsN + models.perplexity.supportsN, + models.perplexity.supportsStreaming ), } diff --git a/src/models.ts b/src/models.ts index b230b2f..704e5fe 100644 --- a/src/models.ts +++ b/src/models.ts @@ -34,7 +34,29 @@ export const models = { 'gpt-3.5-turbo-16k-0613', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'gpt-4o', + 'gpt-4o-2024-05-13', + 'gpt-4-turbo', + 'gpt-4-turbo-2024-04-09', + 'gpt-4-0125-preview', + 'gpt-4-turbo-preview', + 'gpt-4-1106-preview', + 'gpt-4-vision-preview', + 'gpt-4', + 'gpt-4-0314', + 'gpt-4-0613', + 'gpt-4-32k', + 'gpt-4-32k-0314', + 'gpt-4-32k-0613', + 'gpt-3.5-turbo', + 'gpt-3.5-turbo-16k', + 'gpt-3.5-turbo-0301', + 'gpt-3.5-turbo-0613', + 'gpt-3.5-turbo-1106', + 'gpt-3.5-turbo-0125', + 'gpt-3.5-turbo-16k-0613', + ] as const, supportsJSON: [ 'gpt-4o', 'gpt-4o-2024-05-13', @@ -78,7 +100,7 @@ export const models = { ai21: { models: ['jamba-instruct'] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: ['jamba-instruct'] as const, supportsJSON: [] as const, supportsImages: [] as const, supportsToolCalls: [] as const, @@ -95,7 +117,15 @@ export const models = { 'claude-instant-1.2', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'claude-3-5-sonnet-20240620', + 'claude-3-opus-20240229', + 'claude-3-sonnet-20240229', + 'claude-3-haiku-20240307', + 'claude-2.1', + 'claude-2.0', + 'claude-instant-1.2', + ] as const, supportsJSON: [] as const, supportsImages: [ 'claude-3-5-sonnet-20240620', @@ -114,7 +144,11 @@ export const models = { gemini: { models: ['gemini-1.5-pro', 'gemini-1.5-flash', 'gemini-1.0-pro'] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'gemini-1.5-pro', + 'gemini-1.5-flash', + 'gemini-1.0-pro', + ] as const, supportsJSON: ['gemini-1.5-pro', 'gemini-1.5-flash'] as const, supportsImages: ['gemini-1.5-pro', 'gemini-1.5-flash'] as const, supportsToolCalls: [ @@ -134,7 +168,14 @@ export const models = { 'command-light-nightly', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'command-r-plus', + 'command-r', + 'command', + 'command-nightly', + 'command-light', + 'command-light-nightly', + ] as const, supportsJSON: [] as const, supportsImages: [] as const, supportsToolCalls: [ @@ -167,7 +208,27 @@ export const models = { 'mistral.mistral-large-2402-v1:0', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'amazon.titan-text-lite-v1', + 'amazon.titan-text-express-v1', + 'anthropic.claude-3-opus-20240229-v1:0', + 'anthropic.claude-3-sonnet-20240229-v1:0', + 'anthropic.claude-3-haiku-20240307-v1:0', + 'anthropic.claude-v2:1', + 'anthropic.claude-v2', + 'anthropic.claude-instant-v1', + 'cohere.command-r-plus-v1:0', + 'cohere.command-r-v1:0', + 'cohere.command-text-v14', + 'cohere.command-light-text-v14', + 'meta.llama3-8b-instruct-v1:0', + 'meta.llama3-70b-instruct-v1:0', + 'meta.llama2-13b-chat-v1', + 'meta.llama2-70b-chat-v1', + 'mistral.mistral-7b-instruct-v0:2', + 'mistral.mixtral-8x7b-instruct-v0:1', + 'mistral.mistral-large-2402-v1:0', + ] as const, // At the time of writing, the only models that Bedrock supports which allow JSON are Mistral // models. However, Bedrock's `additionalModelRequestFields` field, which is designed to allow // us to pass arbitrary parameters to the model, does not appear to work for Mistral's @@ -206,7 +267,22 @@ export const models = { 'codestral-2405', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'open-mistral-7b', + 'mistral-tiny-2312', + 'open-mixtral-8x7b', + 'mistral-small-2312', + 'open-mixtral-8x22b', + 'open-mixtral-8x22b-2404', + 'mistral-small-latest', + 'mistral-small-2402', + 'mistral-medium-latest', + 'mistral-medium-2312', + 'mistral-large-latest', + 'mistral-large-2402', + 'codestral-latest', + 'codestral-2405', + ] as const, // Mistral claims that all of its models support JSON, but several of their weaker models either // fail to produce valid JSON or produce very low quality results for the following prompt: // "Generate a JSON that maps ten athletes to their jersey numbers". We removed these models @@ -241,7 +317,13 @@ export const models = { 'gemma2-9b-it', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'llama3-8b-8192', + 'llama3-70b-8192', + 'mixtral-8x7b-32768', + 'gemma-7b-it', + 'gemma2-9b-it', + ] as const, // Groq claims that all of its models support JSON, but some of the weaker models either fail to // produce valid JSON or produce very low quality results for the following prompt: "Generate a // JSON that maps ten athletes to their jersey numbers". We removed these models from the list @@ -262,7 +344,15 @@ export const models = { 'mixtral-8x7b-instruct', ] as const, supportsCompletion: true, - supportsStreaming: [] as const, + supportsStreaming: [ + 'llama-3-sonar-small-32k-chat', + 'llama-3-sonar-small-32k-online', + 'llama-3-sonar-large-32k-chat', + 'llama-3-sonar-large-32k-online', + 'llama-3-8b-instruct', + 'llama-3-70b-instruct', + 'mixtral-8x7b-instruct', + ] as const, supportsJSON: [] as const, supportsImages: [] as const, supportsToolCalls: [] as const, diff --git a/test/handlers/base.test.ts b/test/handlers/base.test.ts index 145d1c7..4b6f04a 100644 --- a/test/handlers/base.test.ts +++ b/test/handlers/base.test.ts @@ -1,6 +1,7 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { TokenJS } from '../../src' +import { BaseHandler } from '../../src/handlers/base' import { InputError } from '../../src/handlers/types' import { getDummyMessages, @@ -87,4 +88,26 @@ describe('Base Handler', () => { ) ) }) + + it('throws an error when the model does not support streaming', async () => { + const mockIsSupportedModel = vi + .spyOn(BaseHandler.prototype, 'isSupportedModel') + .mockReturnValue(true) + + const tokenjs = new TokenJS() + await expect( + tokenjs.chat.completions.create({ + provider: 'openai', + model: 'dummyModel' as any, + messages: getDummyMessages(), + }) + ).rejects.toThrow( + new InputError( + `Detected 'stream: true', but the following model does not support streaming: dummyModel` + ) + ) + + expect(mockIsSupportedModel).toHaveBeenCalled() + mockIsSupportedModel.mockRestore() + }) }) diff --git a/test/handlers/gemini.test.ts b/test/handlers/gemini.test.ts index 4cb3f4b..c02153a 100644 --- a/test/handlers/gemini.test.ts +++ b/test/handlers/gemini.test.ts @@ -16,7 +16,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CompletionParams } from '../../src/chat' import { - GeminiHandler, convertAssistantMessage, convertContentsToParts, convertFinishReason, @@ -31,8 +30,7 @@ import { convertUsageData, } from '../../src/handlers/gemini' import { InputError } from '../../src/handlers/types' -import { getTimestamp } from '../../src/handlers/utils' -import { models } from '../../src/models' +import { getHandler, getTimestamp } from '../../src/handlers/utils' import { StreamCompletionResponse } from '../../src/userTypes' import { MESSAGES_WITH_ASSISTANT_TOOL_CALLS_AND_TOOL_RESULTS } from './messages' @@ -1388,14 +1386,7 @@ describe('GeminiHandler', () => { it('should return a completion response', async () => { const handlerOptions = { apiKey: 'test-api-key' } - const handler = new GeminiHandler( - handlerOptions, - models.gemini.models, - models.gemini.supportsJSON, - models.gemini.supportsImages, - models.gemini.supportsToolCalls, - models.gemini.supportsN - ) + const handler = getHandler('gemini', handlerOptions) ;(GoogleGenerativeAI as any).mockImplementationOnce(() => ({ getGenerativeModel: vi.fn().mockReturnValue({ @@ -1452,14 +1443,7 @@ describe('GeminiHandler', () => { it('should return a tool completion response', async () => { const handlerOptions = { apiKey: 'test-api-key' } - const handler = new GeminiHandler( - handlerOptions, - models.gemini.models, - models.gemini.supportsJSON, - models.gemini.supportsImages, - models.gemini.supportsToolCalls, - models.gemini.supportsN - ) + const handler = getHandler('gemini', handlerOptions) ;(GoogleGenerativeAI as any).mockImplementationOnce(() => ({ getGenerativeModel: vi.fn().mockReturnValue({ @@ -1510,14 +1494,7 @@ describe('GeminiHandler', () => { it('should return a stream completion response', async () => { const handlerOptions = { apiKey: 'test-api-key' } - const handler = new GeminiHandler( - handlerOptions, - models.gemini.models, - models.gemini.supportsJSON, - models.gemini.supportsImages, - models.gemini.supportsToolCalls, - models.gemini.supportsN - ) + const handler = getHandler('gemini', handlerOptions) ;(GoogleGenerativeAI as any).mockImplementationOnce(() => ({ getGenerativeModel: vi.fn().mockReturnValue({ diff --git a/test/handlers/mistral.test.ts b/test/handlers/mistral.test.ts index e3026dd..0d7cdd3 100644 --- a/test/handlers/mistral.test.ts +++ b/test/handlers/mistral.test.ts @@ -12,7 +12,6 @@ import { describe, expect, it, vi } from 'vitest' import { CompletionParams } from '../../src/chat' import { - MistralHandler, convertMessages, convertStreamToolCalls, convertToolCalls, @@ -20,7 +19,7 @@ import { findLinkedToolCallName, } from '../../src/handlers/mistral' import { InputError } from '../../src/handlers/types' -import { models } from '../../src/models' +import { getHandler } from '../../src/handlers/utils' import { StreamCompletionResponse } from '../../src/userTypes' describe('findLinkedToolCallName', () => { @@ -743,13 +742,7 @@ describe('MistralHandler', () => { } const handlerOptions = { apiKey: 'test-api-key' } - const handler = new MistralHandler( - handlerOptions, - models.mistral.models, - models.mistral.supportsJSON, - models.mistral.supportsImages, - models.mistral.supportsToolCalls - ) + const handler = getHandler('mistral', handlerOptions) it('should return a completion response', async () => { ;(MistralClient as any).mockImplementationOnce(() => ({