[vllm] standardize input parameters (deepjavalibrary#1301)

maaquib · Nov 9, 2023 · 9015c08 · 9015c08
1 parent 31d7d92
commit 9015c08
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/engines/python/setup/djl_python/rolling_batch/vllm_rolling_batch.py b/engines/python/setup/djl_python/rolling_batch/vllm_rolling_batch.py
@@ -70,6 +70,7 @@ def inference(self, input_data, parameters):
         for request in new_requests:
             request_id = random_uuid()
             request.parameters.pop('seed', None)
+            request.parameters.pop('do_sample', None)
             if "max_new_tokens" in request.parameters.keys():
                 request.parameters["max_tokens"] = request.parameters.pop(
                     "max_new_tokens")