From 5b4e0dae4e3b7ba414e343b97a27039e88b6f607 Mon Sep 17 00:00:00 2001 From: cranechu <1340390339@qq.com> Date: Wed, 4 Dec 2024 17:04:00 +0800 Subject: [PATCH 1/2] feat: add streaming --- .../LLM/baichuan2.py | 21 ++++++++++------ .../HF-Transformers-AutoModels/LLM/llama2.py | 21 ++++++++++------ .../HF-Transformers-AutoModels/LLM/llama3.py | 21 ++++++++++------ .../HF-Transformers-AutoModels/LLM/minicpm.py | 25 ++++++++++++------- .../HF-Transformers-AutoModels/LLM/qwen.py | 21 ++++++++++------ 5 files changed, 68 insertions(+), 41 deletions(-) diff --git a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/baichuan2.py b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/baichuan2.py index cdf26af179b..3aae11f8966 100644 --- a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/baichuan2.py +++ b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/baichuan2.py @@ -20,7 +20,7 @@ import argparse from ipex_llm.transformers.npu_model import AutoModelForCausalLM -from transformers import AutoTokenizer +from transformers import AutoTokenizer, TextStreamer from transformers.utils import logging @@ -56,6 +56,7 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]], parser.add_argument("--max-context-len", type=int, default=1024) parser.add_argument("--max-prompt-len", type=int, default=512) parser.add_argument("--disable-transpose-value-cache", action="store_true", default=False) + parser.add_argument("--disable-streaming", action="store_true", default=False) parser.add_argument("--save-directory", type=str, required=True, help="The path of folder to save converted model, " @@ -94,6 +95,10 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]], ) tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True) + if args.disable_streaming: + streamer = None + else: + streamer = TextStreamer(tokenizer=tokenizer, skip_special_tokens=True) DEFAULT_SYSTEM_PROMPT = """\ """ @@ -105,19 +110,19 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]], for i in range(5): prompt = get_prompt(args.prompt, [], system_prompt=DEFAULT_SYSTEM_PROMPT) _input_ids = tokenizer.encode(prompt, return_tensors="pt") + print("-" * 20, "Input", "-" * 20) print("input length:", len(_input_ids[0])) + print(prompt) + print("-" * 20, "Output", "-" * 20) st = time.time() output = model.generate( - _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict + _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict, streamer=streamer ) end = time.time() + if args.disable_streaming: + output_str = tokenizer.decode(output[0], skip_special_tokens=False) + print(output_str) print(f"Inference time: {end-st} s") - input_str = tokenizer.decode(_input_ids[0], skip_special_tokens=False) - print("-" * 20, "Input", "-" * 20) - print(input_str) - output_str = tokenizer.decode(output[0], skip_special_tokens=False) - print("-" * 20, "Output", "-" * 20) - print(output_str) print("-" * 80) print("done") diff --git a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama2.py b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama2.py index d981f39f97e..a2e0881b190 100644 --- a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama2.py +++ b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama2.py @@ -20,7 +20,7 @@ import argparse from ipex_llm.transformers.npu_model import AutoModelForCausalLM -from transformers import AutoTokenizer +from transformers import AutoTokenizer, TextStreamer from transformers.utils import logging @@ -56,6 +56,7 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]], parser.add_argument("--max-context-len", type=int, default=1024) parser.add_argument("--max-prompt-len", type=int, default=512) parser.add_argument("--disable-transpose-value-cache", action="store_true", default=False) + parser.add_argument("--disable-streaming", action="store_true", default=False) parser.add_argument("--save-directory", type=str, required=True, help="The path of folder to save converted model, " @@ -93,6 +94,10 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]], ) tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True) + if args.disable_streaming: + streamer = None + else: + streamer = TextStreamer(tokenizer=tokenizer, skip_special_tokens=True) DEFAULT_SYSTEM_PROMPT = """\ """ @@ -104,19 +109,19 @@ def get_prompt(message: str, chat_history: list[tuple[str, str]], for i in range(5): prompt = get_prompt(args.prompt, [], system_prompt=DEFAULT_SYSTEM_PROMPT) _input_ids = tokenizer.encode(prompt, return_tensors="pt") + print("-" * 20, "Input", "-" * 20) print("input length:", len(_input_ids[0])) + print(prompt) + print("-" * 20, "Output", "-" * 20) st = time.time() output = model.generate( - _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict + _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict, streamer=streamer ) end = time.time() + if args.disable_streaming: + output_str = tokenizer.decode(output[0], skip_special_tokens=False) + print(output_str) print(f"Inference time: {end-st} s") - input_str = tokenizer.decode(_input_ids[0], skip_special_tokens=False) - print("-" * 20, "Input", "-" * 20) - print(input_str) - output_str = tokenizer.decode(output[0], skip_special_tokens=False) - print("-" * 20, "Output", "-" * 20) - print(output_str) print("-" * 80) print("done") diff --git a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama3.py b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama3.py index 35ee4902246..9bc570411d5 100644 --- a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama3.py +++ b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/llama3.py @@ -20,7 +20,7 @@ import argparse from ipex_llm.transformers.npu_model import AutoModelForCausalLM -from transformers import AutoTokenizer +from transformers import AutoTokenizer, TextStreamer from transformers.utils import logging @@ -57,6 +57,7 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]], parser.add_argument("--max-context-len", type=int, default=1024) parser.add_argument("--max-prompt-len", type=int, default=512) parser.add_argument("--disable-transpose-value-cache", action="store_true", default=False) + parser.add_argument("--disable-streaming", action="store_true", default=False) parser.add_argument("--save-directory", type=str, required=True, help="The path of folder to save converted model, " @@ -94,6 +95,10 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]], ) tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True) + if args.disable_streaming: + streamer = None + else: + streamer = TextStreamer(tokenizer=tokenizer, skip_special_tokens=True) DEFAULT_SYSTEM_PROMPT = """\ """ @@ -105,19 +110,19 @@ def get_prompt(user_input: str, chat_history: list[tuple[str, str]], for i in range(5): prompt = get_prompt(args.prompt, [], system_prompt=DEFAULT_SYSTEM_PROMPT) _input_ids = tokenizer.encode(prompt, return_tensors="pt") + print("-" * 20, "Input", "-" * 20) print("input length:", len(_input_ids[0])) + print(prompt) + print("-" * 20, "Output", "-" * 20) st = time.time() output = model.generate( - _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict + _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict, streamer=streamer ) end = time.time() + if args.disable_streaming: + output_str = tokenizer.decode(output[0], skip_special_tokens=False) + print(output_str) print(f"Inference time: {end-st} s") - input_str = tokenizer.decode(_input_ids[0], skip_special_tokens=False) - print("-" * 20, "Input", "-" * 20) - print(input_str) - output_str = tokenizer.decode(output[0], skip_special_tokens=False) - print("-" * 20, "Output", "-" * 20) - print(output_str) print("-" * 80) print("done") diff --git a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/minicpm.py b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/minicpm.py index b177042cc2b..eb911ca0b6e 100644 --- a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/minicpm.py +++ b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/minicpm.py @@ -20,7 +20,7 @@ import argparse from ipex_llm.transformers.npu_model import AutoModelForCausalLM -from transformers import AutoTokenizer +from transformers import AutoTokenizer, TextStreamer from transformers.utils import logging @@ -43,6 +43,7 @@ parser.add_argument("--max-context-len", type=int, default=1024) parser.add_argument("--max-prompt-len", type=int, default=512) parser.add_argument("--disable-transpose-value-cache", action="store_true", default=False) + parser.add_argument("--disable-streaming", action="store_true", default=False) parser.add_argument("--save-directory", type=str, required=True, help="The path of folder to save converted model, " @@ -80,26 +81,32 @@ ) tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True) + if args.disable_streaming: + streamer = None + else: + streamer = TextStreamer(tokenizer=tokenizer, skip_special_tokens=True) + print("-" * 80) print("done") with torch.inference_mode(): print("finish to load") for i in range(5): - _input_ids = tokenizer.encode("<用户>{}".format(args.prompt), return_tensors="pt") + prompt = "<用户>{}".format(args.prompt) + _input_ids = tokenizer.encode(prompt, return_tensors="pt") + print("-" * 20, "Input", "-" * 20) print("input length:", len(_input_ids[0])) + print(prompt) + print("-" * 20, "Output", "-" * 20) st = time.time() output = model.generate( - _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict + _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict, streamer=streamer ) end = time.time() + if args.disable_streaming: + output_str = tokenizer.decode(output[0], skip_special_tokens=False) + print(output_str) print(f"Inference time: {end-st} s") - input_str = tokenizer.decode(_input_ids[0], skip_special_tokens=False) - print("-" * 20, "Input", "-" * 20) - print(input_str) - output_str = tokenizer.decode(output[0], skip_special_tokens=False) - print("-" * 20, "Output", "-" * 20) - print(output_str) print("-" * 80) print("done") diff --git a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/qwen.py b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/qwen.py index caf6d1b3e54..c3b1f534fd5 100644 --- a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/qwen.py +++ b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/qwen.py @@ -20,7 +20,7 @@ import argparse from ipex_llm.transformers.npu_model import AutoModelForCausalLM -from transformers import AutoTokenizer +from transformers import AutoTokenizer, TextStreamer from transformers.utils import logging @@ -45,6 +45,7 @@ parser.add_argument("--quantization_group_size", type=int, default=0) parser.add_argument('--low-bit', type=str, default="sym_int4", help='Load in low bit to use') + parser.add_argument("--disable-streaming", action="store_true", default=False) parser.add_argument("--disable-transpose-value-cache", action="store_true", default=False) parser.add_argument("--save-directory", type=str, required=True, @@ -85,6 +86,10 @@ ) tokenizer = AutoTokenizer.from_pretrained(args.save_directory, trust_remote_code=True) + if args.disable_streaming: + streamer = None + else: + streamer = TextStreamer(tokenizer=tokenizer, skip_special_tokens=True) print("-" * 80) print("done") @@ -97,19 +102,19 @@ print("finish to load") for i in range(3): _input_ids = tokenizer([text], return_tensors="pt").input_ids + print("-" * 20, "Input", "-" * 20) print("input length:", len(_input_ids[0])) + print(text) + print("-" * 20, "Output", "-" * 20) st = time.time() output = model.generate( - _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict + _input_ids, num_beams=1, do_sample=False, max_new_tokens=args.n_predict, streamer=streamer ) end = time.time() + if args.disable_streaming: + output_str = tokenizer.decode(output[0], skip_special_tokens=False) + print(output_str) print(f"Inference time: {end-st} s") - input_str = tokenizer.decode(_input_ids[0], skip_special_tokens=False) - print("-" * 20, "Input", "-" * 20) - print(input_str) - output_str = tokenizer.decode(output[0], skip_special_tokens=False) - print("-" * 20, "Output", "-" * 20) - print(output_str) print("-" * 80) print("done") From 31f741a335350cebda2b76207ad507538b5cd0a1 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Wed, 4 Dec 2024 17:41:47 +0800 Subject: [PATCH 2/2] Update readme accordingly --- python/llm/example/NPU/HF-Transformers-AutoModels/LLM/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/README.md b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/README.md index 246cc10e209..defed0bda8c 100644 --- a/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/README.md +++ b/python/llm/example/NPU/HF-Transformers-AutoModels/LLM/README.md @@ -136,6 +136,7 @@ Arguments info: - `--max-context-len MAX_CONTEXT_LEN`: Defines the maximum sequence length for both input and output tokens. It is default to be `1024`. - `--max-prompt-len MAX_PROMPT_LEN`: Defines the maximum number of tokens that the input prompt can contain. It is default to be `512`. - `--disable-transpose-value-cache`: Disable the optimization of transposing value cache. +- `--disable-streaming`: Disable streaming mode of generation. - `--save-directory SAVE_DIRECTORY`: argument defining the path to save converted model. If it is a non-existing path, the original pretrained model specified by `REPO_ID_OR_MODEL_PATH` will be loaded, otherwise the lowbit model in `SAVE_DIRECTORY` will be loaded. ### Troubleshooting