From 42fcd4d89dad52b27fdf65453018726b1055abf1 Mon Sep 17 00:00:00 2001
From: guozhong wang <guozhong.wang@intel.com>
Date: Mon, 15 Jul 2024 16:55:41 +0800
Subject: [PATCH] Add end_token_stopping option (#606)

CVS-146307
---
 llm_bench/python/benchmark.py         | 7 ++++++-
 llm_bench/python/utils/model_utils.py | 1 +
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/llm_bench/python/benchmark.py b/llm_bench/python/benchmark.py
index 0e2fcbdcc2..2e4a23da41 100644
--- a/llm_bench/python/benchmark.py
+++ b/llm_bench/python/benchmark.py
@@ -101,7 +101,7 @@ def run_text_generation(input_text, num, model, tokenizer, args, iter_data_list,
         mem_consumption.start_collect_memory_consumption()
     max_gen_tokens = DEFAULT_OUTPUT_TOKEN_SIZE if args['infer_count'] is None else args['infer_count']
     start = time.perf_counter()
-    if args['infer_count'] is not None:
+    if args['infer_count'] is not None and args['end_token_stopping'] is False:
         model.generation_config.eos_token_id = None
         model.config.eos_token_id = None
         result = model.generate(
@@ -693,6 +693,11 @@ def get_argprser():
     parser.add_argument('-od', '--output_dir', help='Save the input text and generated text, images to files')
     utils.model_utils.add_stateful_model_arguments(parser)
     parser.add_argument("--genai", action="store_true")
+    parser.add_argument(
+        '--end_token_stopping',
+        action='store_true',
+        help='Stop the generation even output token size does not achieve infer_count or max token size ({DEFAULT_OUTPUT_TOKEN_SIZE}}).'
+    )
 
     return parser.parse_args()
 
diff --git a/llm_bench/python/utils/model_utils.py b/llm_bench/python/utils/model_utils.py
index 8b6c1e95f5..19b53f3ad6 100644
--- a/llm_bench/python/utils/model_utils.py
+++ b/llm_bench/python/utils/model_utils.py
@@ -139,6 +139,7 @@ def analyze_args(args):
     if model_args['prompt_index'] is not None:
         # Deduplication
         [model_args['prompt_index'].append(i) for i in args.prompt_index if i not in model_args['prompt_index']]
+    model_args['end_token_stopping'] = args.end_token_stopping
 
     model_framework = args.framework
     model_path = Path(args.model)