From d94c22d6c99f5505399d1bd191acf61e83842b76 Mon Sep 17 00:00:00 2001 From: Fei Date: Tue, 30 Jul 2024 05:05:14 +0000 Subject: [PATCH 1/4] fix logit processor excceed vocab size issue --- vllm/entrypoints/openai/logits_processors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/entrypoints/openai/logits_processors.py b/vllm/entrypoints/openai/logits_processors.py index 31eb5aa628c52..16fe6de5d720b 100644 --- a/vllm/entrypoints/openai/logits_processors.py +++ b/vllm/entrypoints/openai/logits_processors.py @@ -61,6 +61,9 @@ def get_logits_processors( def logit_bias_logits_processor(token_ids: List[int], logits: torch.Tensor) -> torch.Tensor: for token_id, bias in clamped_logit_bias.items(): + if token_id < 0 or token_id >= len(tokenizer.vocab_size): + raise ValueError("token_id in logit_bias contains " + "out-of-vocab token id") logits[token_id] += bias return logits From 26871dd3821de7c4c62437a4a553f8dd7f1c5701 Mon Sep 17 00:00:00 2001 From: Fei Date: Tue, 30 Jul 2024 22:44:28 +0000 Subject: [PATCH 2/4] refactor according to comments --- vllm/entrypoints/openai/logits_processors.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vllm/entrypoints/openai/logits_processors.py b/vllm/entrypoints/openai/logits_processors.py index 16fe6de5d720b..da22e873354c4 100644 --- a/vllm/entrypoints/openai/logits_processors.py +++ b/vllm/entrypoints/openai/logits_processors.py @@ -57,13 +57,16 @@ def get_logits_processors( raise ValueError( "Found token_id in logit_bias that is not " "an integer or string representing an integer") from exc + + # Check if token_id is within the vocab size + for token_id, bias in clamped_logit_bias.items(): + if token_id < 0 or token_id >= len(tokenizer.vocab_size): + raise ValueError("token_id in logit_bias contains " + "out-of-vocab token id") def logit_bias_logits_processor(token_ids: List[int], logits: torch.Tensor) -> torch.Tensor: for token_id, bias in clamped_logit_bias.items(): - if token_id < 0 or token_id >= len(tokenizer.vocab_size): - raise ValueError("token_id in logit_bias contains " - "out-of-vocab token id") logits[token_id] += bias return logits From bfdfefb7b3ae517e0c644359b2cbcd1786fc8b5f Mon Sep 17 00:00:00 2001 From: Fei Date: Wed, 31 Jul 2024 02:12:07 +0000 Subject: [PATCH 3/4] reformat --- vllm/entrypoints/openai/logits_processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/logits_processors.py b/vllm/entrypoints/openai/logits_processors.py index da22e873354c4..8a768e2c112b5 100644 --- a/vllm/entrypoints/openai/logits_processors.py +++ b/vllm/entrypoints/openai/logits_processors.py @@ -57,7 +57,7 @@ def get_logits_processors( raise ValueError( "Found token_id in logit_bias that is not " "an integer or string representing an integer") from exc - + # Check if token_id is within the vocab size for token_id, bias in clamped_logit_bias.items(): if token_id < 0 or token_id >= len(tokenizer.vocab_size): From caeadcd3f3dfce5e970dae1cd5b95870370ac226 Mon Sep 17 00:00:00 2001 From: Fei Date: Wed, 31 Jul 2024 04:52:40 +0000 Subject: [PATCH 4/4] fix failed gate --- vllm/entrypoints/openai/logits_processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/logits_processors.py b/vllm/entrypoints/openai/logits_processors.py index 8a768e2c112b5..f8e04e7f18e0f 100644 --- a/vllm/entrypoints/openai/logits_processors.py +++ b/vllm/entrypoints/openai/logits_processors.py @@ -60,7 +60,7 @@ def get_logits_processors( # Check if token_id is within the vocab size for token_id, bias in clamped_logit_bias.items(): - if token_id < 0 or token_id >= len(tokenizer.vocab_size): + if token_id < 0 or token_id >= tokenizer.vocab_size: raise ValueError("token_id in logit_bias contains " "out-of-vocab token id")