[Core][Bugfix] cache len of tokenizer (vllm-project#3741)

Temirulan · Mar 30, 2024 · eae1f07 · eae1f07
1 parent bf98867
commit eae1f07
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py
@@ -26,6 +26,7 @@ def get_cached_tokenizer(
     tokenizer_all_special_tokens_extended = (
         tokenizer.all_special_tokens_extended)
     tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
+    tokenizer_len = len(tokenizer)
 
     class CachedTokenizer(tokenizer.__class__):
 
@@ -41,6 +42,9 @@ def all_special_tokens(self):
         def all_special_tokens_extended(self):
             return tokenizer_all_special_tokens_extended
 
+        def __len__(self):
+            return tokenizer_len
+
     CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
 
     tokenizer.__class__ = CachedTokenizer