use new anthropic token counting api (#16909)

run-llama · Nov 11, 2024 · 29c9035 · 29c9035
1 parent 35906a1
commit 29c9035
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 4 deletions.
diff --git a/docs/docs/examples/llm/anthropic.ipynb b/docs/docs/examples/llm/anthropic.ipynb
@@ -59,7 +59,7 @@
     "\n",
     "First we want to set the tokenizer, which is slightly different than TikToken.\n",
     "\n",
-    "**NOTE**: The Claude 3 tokenizer has not been updated yet; using the existing Anthropic tokenizer leads to context overflow errors for 200k tokens. We've temporarily set the max tokens for Claude 3 to 180k."
+    "**NOTE**: Anthropic recently updated their token counting API. Older models like claude-2.1 are no longer supported for token counting in the latest versions of the Anthropic python client."
    ]
   },
   {

diff --git a/llama-index-integrations/llms/llama-index-llms-anthropic/llama_index/llms/anthropic/base.py b/llama-index-integrations/llms/llama-index-llms-anthropic/llama_index/llms/anthropic/base.py
@@ -210,7 +210,13 @@ def metadata(self) -> LLMMetadata:
 
     @property
     def tokenizer(self) -> Tokenizer:
-        return self._client.get_tokenizer()
+        def _count_tokens(text: str) -> int:
+            return self._client.beta.messages.count_tokens(
+                messages=[{"role": "user", "content": text}],
+                model=self.model,
+            ).input_tokens
+
+        return _count_tokens
 
     @property
     def _model_kwargs(self) -> Dict[str, Any]:

diff --git a/llama-index-integrations/llms/llama-index-llms-anthropic/pyproject.toml b/llama-index-integrations/llms/llama-index-llms-anthropic/pyproject.toml
@@ -27,11 +27,11 @@ exclude = ["**/BUILD"]
 license = "MIT"
 name = "llama-index-llms-anthropic"
 readme = "README.md"
-version = "0.3.9"
+version = "0.4.0"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
-anthropic = {extras = ["bedrock", "vertex"], version = ">=0.34.2"}
+anthropic = {extras = ["bedrock", "vertex"], version = ">=0.39.0"}
 llama-index-core = "^0.11.0"
 
 [tool.poetry.group.dev.dependencies]