Skip to content

Commit

Permalink
Fix ignored sequence case
Browse files Browse the repository at this point in the history
  • Loading branch information
njhill committed Aug 10, 2024
1 parent 21ddcca commit d18d646
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions vllm/engine/llm_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -1236,9 +1236,19 @@ def _process_model_outputs(
if request_output:
request_outputs.append(request_output)
for seq_group in ignored_seq_groups:
if seq_group.sampling_params.output_kind == (
RequestOutputKind.CUMULATIVE):
request_output = RequestOutputFactory.create(seq_group)
params = seq_group.sampling_params
if params is not None and params.output_kind == (
RequestOutputKind.DELTA):
if not seq_group.is_finished():
continue
# Ignored seq groups have no delta, but we must still return
# an "empty" RequestOutput when finished
for seq in seq_group.seqs:
previous_output_lens[seq.seq_id] = (seq.get_output_len(),
seq.output_text)
request_output = RequestOutputFactory.create(
seq_group, previous_output_lens)
if request_output:
request_outputs.append(request_output)
return request_outputs

Expand Down

0 comments on commit d18d646

Please sign in to comment.