Skip to content

Commit

Permalink
Fixed endless generations in test_callback_one_string test
Browse files Browse the repository at this point in the history
  • Loading branch information
ilya-lavrenov committed Nov 11, 2024
1 parent 747c5d2 commit 80b622e
Showing 1 changed file with 4 additions and 1 deletion.
5 changes: 4 additions & 1 deletion src/cpp/src/continuous_batching_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,9 @@ ContinuousBatchingPipeline::ContinuousBatchingImpl::generate(const std::vector<o
}
}, streamer);

OPENVINO_ASSERT(streamer_ptr == nullptr || input_ids.size() == 1 && (sampling_params[0].is_greedy_decoding() || sampling_params[0].is_multinomial()),
"Currently streaming is possible only with batch size=1 and only for greedy or multinomial decoding");

std::vector<GenerationHandle> generations;
for (size_t request_id = 0; request_id < input_ids.size(); ++request_id) {
OPENVINO_ASSERT(1 == input_ids[request_id].get_shape().at(0), "Use multiple tensors to pass a batch.");
Expand All @@ -283,7 +286,7 @@ ContinuousBatchingPipeline::ContinuousBatchingImpl::generate(const std::vector<o
m_requests.clear();
};

bool continue_generation = true, step_throws_exception = false;
bool continue_generation = true;
while (has_non_finished_requests() && continue_generation) {
try {
step();
Expand Down

0 comments on commit 80b622e

Please sign in to comment.