From beac8dd4615566f153d2d8e8b74e703a4df18ce4 Mon Sep 17 00:00:00 2001 From: Ricardo Lu <37237570+gesanqiu@users.noreply.github.com> Date: Sun, 29 Oct 2023 19:26:36 +0800 Subject: [PATCH] fix: don't skip first special token. (#1497) --- vllm/transformers_utils/tokenizer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 49e7007ae3a84..424c102c703d2 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -120,7 +120,11 @@ def detokenize_incrementally( # tokenizers (bigger = more conservative). # Subtract 1 extra to account for the generated token. prefix_offset = max(len(output_tokens) - 6, 0) - read_offset = max(len(output_tokens) - 1, 0) + # If the first new token is a special token, we can't skip 1 extra token + if skip_special_tokens and new_token_id in tokenizer.all_special_ids: + read_offset = max(len(output_tokens), 0) + else: + read_offset = max(len(output_tokens) - 1, 0) else: # Put new_token_id in a list so skip_special_tokens is respected new_tokens = tokenizer.convert_ids_to_tokens(