diff --git a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/README.md b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/README.md index 8d88fbb23a6..ee653b58136 100644 --- a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/README.md +++ b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/README.md @@ -5,7 +5,7 @@ In this directory, you will find examples on how you could apply IPEX-LLM INT4 o To run these examples with IPEX-LLM on Intel GPUs, we have some recommended requirements for your machine, please refer to [here](../../../README.md#requirements) for more information. ## Example: Predict Tokens using `chat()` API -In the example [generate.py](./generate.py), we show a basic use case for a MiniCPM-Llama3-V-2_5 model to predict the next N tokens using `chat()` API, with IPEX-LLM INT4 optimizations on Intel GPUs. +In the example [chat.py](./chat.py), we show a basic use case for a MiniCPM-Llama3-V-2_5 model to predict the next N tokens using `chat()` API, with IPEX-LLM INT4 optimizations on Intel GPUs. ### 1. Install #### 1.1 Installation on Linux We suggest using conda to manage environment: @@ -106,15 +106,20 @@ set SYCL_CACHE_PERSISTENT=1 > For the first time that each model runs on Intel iGPU/Intel Arc™ A300-Series or Pro A60, it may take several minutes to compile. ### 4. Running examples -``` -python ./generate.py --prompt 'What is in the image?' -``` +- chat without streaming mode: + ``` + python ./chat.py --prompt 'What is in the image?' + ``` +- chat in streaming mode: + ``` + python ./chat.py --prompt 'What is in the image?' --stream + ``` Arguments info: - `--repo-id-or-model-path REPO_ID_OR_MODEL_PATH`: argument defining the huggingface repo id for the MiniCPM-Llama3-V-2_5 (e.g. `openbmb/MiniCPM-Llama3-V-2_5`) to be downloaded, or the path to the huggingface checkpoint folder. It is default to be `'openbmb/MiniCPM-Llama3-V-2_5'`. - `--image-url-or-path IMAGE_URL_OR_PATH`: argument defining the image to be infered. It is default to be `'http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg'`. - `--prompt PROMPT`: argument defining the prompt to be infered (with integrated prompt format for chat). It is default to be `'What is in the image?'`. -- `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `32`. +- `--stream`: flag to chat in streaming mode #### Sample Output @@ -122,12 +127,21 @@ Arguments info: ```log Inference time: xxxx s --------------------- Input -------------------- +-------------------- Input Image -------------------- http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg --------------------- Prompt -------------------- +-------------------- Input Prompt -------------------- What is in the image? --------------------- Output -------------------- -The image features a young child holding a white teddy bear. The teddy bear is dressed in a pink outfit. The child appears to be outdoors, with a stone wall and some red flowers in the background. +-------------------- Chat Output -------------------- +The image features a young child holding a white teddy bear. The teddy bear is dressed in a pink dress with a ribbon on it. The child appears to be smiling and enjoying the moment. +``` +```log +Inference time: xxxx s +-------------------- Input Image -------------------- +http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg +-------------------- Input Prompt -------------------- +图片里有什么？ +-------------------- Chat Output -------------------- +图片中有一个小孩，手里拿着一个白色的玩具熊。这个孩子看起来很开心，正在微笑并与玩具互动。背景包括红色的花朵和石墙，为这个场景增添了色彩和质感。 ``` The sample input image is (which is fetched from [COCO dataset](https://cocodataset.org/#explore?id=264959)): diff --git a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/generate.py b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/chat.py similarity index 61% rename from python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/generate.py rename to python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/chat.py index fe5ab5e1014..66aa46304db 100644 --- a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/generate.py +++ b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-Llama3-V-2_5/chat.py @@ -14,10 +14,12 @@ # limitations under the License. # + import os import time import argparse import requests +import torch from PIL import Image from ipex_llm.transformers import AutoModel from transformers import AutoTokenizer @@ -33,8 +35,8 @@ help='The URL or path to the image to infer') parser.add_argument('--prompt', type=str, default="What is in the image?", help='Prompt to infer') - parser.add_argument('--n-predict', type=int, default=32, - help='Max tokens to predict') + parser.add_argument('--stream', action='store_true', + help='Whether to chat in streaming mode') args = parser.parse_args() model_path = args.repo_id_or_model_path @@ -45,11 +47,12 @@ # When running LLMs on Intel iGPUs for Windows users, we recommend setting `cpu_embedding=True` in the from_pretrained function. # This will allow the memory-intensive embedding layer to utilize the CPU instead of iGPU. model = AutoModel.from_pretrained(model_path, - load_in_4bit=True, - optimize_model=False, + load_in_low_bit="sym_int4", + optimize_model=True, trust_remote_code=True, - use_cache=True) - model = model.half().to(device='xpu') + use_cache=True, + modules_to_not_convert=["vpm", "resampler"]) + model = model.half().to('xpu') tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) model.eval() @@ -61,23 +64,45 @@ image = Image.open(requests.get(image_path, stream=True).raw).convert('RGB') # Generate predicted tokens - # here the prompt tuning refers to https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/blob/main/README.md - msgs = [{'role': 'user', 'content': args.prompt}] - st = time.time() - res = model.chat( - image=image, - msgs=msgs, - context=None, - tokenizer=tokenizer, - sampling=False, - temperature=0.7 + # here the prompt tuning refers to https://huggingface.co/openbmb/MiniCPM-V-2_6/blob/main/README.md + msgs = [{'role': 'user', 'content': [image, args.prompt]}] + + # ipex_llm model needs a warmup, then inference time can be accurate + model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, ) - end = time.time() - print(f'Inference time: {end-st} s') - print('-'*20, 'Input', '-'*20) - print(image_path) - print('-'*20, 'Prompt', '-'*20) - print(args.prompt) - output_str = res - print('-'*20, 'Output', '-'*20) - print(output_str) + + if args.stream: + res = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + stream=True + ) + + print('-'*20, 'Input Image', '-'*20) + print(image_path) + print('-'*20, 'Input Prompt', '-'*20) + print(args.prompt) + print('-'*20, 'Stream Chat Output', '-'*20) + for new_text in res: + print(new_text, flush=True, end='') + else: + st = time.time() + res = model.chat( + image=None, + msgs=msgs, + tokenizer=tokenizer, + ) + torch.xpu.synchronize() + end = time.time() + + print(f'Inference time: {end-st} s') + print('-'*20, 'Input Image', '-'*20) + print(image_path) + print('-'*20, 'Input Prompt', '-'*20) + print(args.prompt) + print('-'*20, 'Chat Output', '-'*20) + print(res) diff --git a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/README.md b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/README.md index da5f94007c9..aed936fb277 100644 --- a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/README.md +++ b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/README.md @@ -5,7 +5,7 @@ In this directory, you will find examples on how you could apply IPEX-LLM INT4 o To run these examples with IPEX-LLM on Intel GPUs, we have some recommended requirements for your machine, please refer to [here](../../../README.md#requirements) for more information. ## Example: Predict Tokens using `chat()` API -In the example [generate.py](./generate.py), we show a basic use case for a MiniCPM-V-2 model to predict the next N tokens using `chat()` API, with IPEX-LLM INT4 optimizations on Intel GPUs. +In the example [chat.py](./chat.py), we show a basic use case for a MiniCPM-V-2 model to predict the next N tokens using `chat()` API, with IPEX-LLM INT4 optimizations on Intel GPUs. ### 1. Install #### 1.1 Installation on Linux We suggest using conda to manage environment: @@ -106,15 +106,20 @@ set SYCL_CACHE_PERSISTENT=1 > For the first time that each model runs on Intel iGPU/Intel Arc™ A300-Series or Pro A60, it may take several minutes to compile. ### 4. Running examples -``` -python ./generate.py --prompt 'What is in the image?' -``` +- chat without streaming mode: + ``` + python ./chat.py --prompt 'What is in the image?' + ``` +- chat in streaming mode: + ``` + python ./chat.py --prompt 'What is in the image?' --stream + ``` Arguments info: - `--repo-id-or-model-path REPO_ID_OR_MODEL_PATH`: argument defining the huggingface repo id for the MiniCPM-V-2 (e.g. `openbmb/MiniCPM-V-2`) to be downloaded, or the path to the huggingface checkpoint folder. It is default to be `'openbmb/MiniCPM-V-2'`. - `--image-url-or-path IMAGE_URL_OR_PATH`: argument defining the image to be infered. It is default to be `'http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg'`. - `--prompt PROMPT`: argument defining the prompt to be infered (with integrated prompt format for chat). It is default to be `'What is in the image?'`. -- `--n-predict N_PREDICT`: argument defining the max number of tokens to predict. It is default to be `32`. +- `--stream`: flag to chat in streaming mode #### Sample Output @@ -122,12 +127,20 @@ Arguments info: ```log Inference time: xxxx s --------------------- Input -------------------- +-------------------- Input Image -------------------- http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg --------------------- Prompt -------------------- +-------------------- Input Prompt -------------------- What is in the image? --------------------- Output -------------------- -In the image, there is a young child holding a teddy bear. The teddy bear appears to be dressed in a pink tutu. The child is also wearing a red and white striped dress. The background of the image includes a stone wall and some red flowers. +-------------------- Chat Output -------------------- +In the image, there is a young child holding a teddy bear. The teddy bear is dressed in a pink tutu. The child is also wearing a red and white striped dress. The background of the image features a stone wall and some red flowers. +``` +```log +-------------------- Input Image -------------------- +http://farm6.staticflickr.com/5268/5602445367_3504763978_z.jpg +-------------------- Input Prompt -------------------- +图片里有什么？ +-------------------- Chat Output -------------------- +图中是一个小女孩，她手里拿着一只粉白相间的泰迪熊。 ``` The sample input image is (which is fetched from [COCO dataset](https://cocodataset.org/#explore?id=264959)): diff --git a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/generate.py b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/chat.py similarity index 82% rename from python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/generate.py rename to python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/chat.py index 91ae81d2a26..93441c84bbb 100644 --- a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/generate.py +++ b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2/chat.py @@ -15,6 +15,7 @@ # + from typing import List, Tuple, Optional, Union import math import timm @@ -110,6 +111,7 @@ def _pos_embed(self, x: torch.Tensor) -> torch.Tensor: import time import argparse import requests +import torch from PIL import Image from ipex_llm.transformers import AutoModel from transformers import AutoTokenizer @@ -125,8 +127,8 @@ def _pos_embed(self, x: torch.Tensor) -> torch.Tensor: help='The URL or path to the image to infer') parser.add_argument('--prompt', type=str, default="What is in the image?", help='Prompt to infer') - parser.add_argument('--n-predict', type=int, default=32, - help='Max tokens to predict') + parser.add_argument('--stream', action='store_true', + help='Whether to chat in streaming mode') args = parser.parse_args() model_path = args.repo_id_or_model_path @@ -140,9 +142,9 @@ def _pos_embed(self, x: torch.Tensor) -> torch.Tensor: load_in_low_bit="asym_int4", optimize_model=True, trust_remote_code=True, - modules_to_not_convert=["vpm", "resampler", "lm_head"], - use_cache=True) - model = model.half().to(device='xpu') + use_cache=True, + modules_to_not_convert=["vpm", "resampler"]) + model = model.half().to('xpu') tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) model.eval() @@ -156,7 +158,8 @@ def _pos_embed(self, x: torch.Tensor) -> torch.Tensor: # Generate predicted tokens # here the prompt tuning refers to https://huggingface.co/openbmb/MiniCPM-V-2/blob/main/README.md msgs = [{'role': 'user', 'content': args.prompt}] - st = time.time() + + # ipex_llm model needs a warmup, then inference time can be accurate res, context, _ = model.chat( image=image, msgs=msgs, @@ -165,12 +168,40 @@ def _pos_embed(self, x: torch.Tensor) -> torch.Tensor: sampling=False, temperature=0.7 ) - end = time.time() - print(f'Inference time: {end-st} s') - print('-'*20, 'Input', '-'*20) - print(image_path) - print('-'*20, 'Prompt', '-'*20) - print(args.prompt) - output_str = res - print('-'*20, 'Output', '-'*20) - print(output_str) + if args.stream: + res, context, _ = model.chat( + image=image, + msgs=msgs, + context=None, + tokenizer=tokenizer, + sampling=False, + temperature=0.7 + ) + + print('-'*20, 'Input Image', '-'*20) + print(image_path) + print('-'*20, 'Input Prompt', '-'*20) + print(args.prompt) + print('-'*20, 'Stream Chat Output', '-'*20) + for new_text in res: + print(new_text, flush=True, end='') + else: + st = time.time() + res, context, _ = model.chat( + image=image, + msgs=msgs, + context=None, + tokenizer=tokenizer, + sampling=False, + temperature=0.7 + ) + torch.xpu.synchronize() + end = time.time() + + print(f'Inference time: {end-st} s') + print('-'*20, 'Input Image', '-'*20) + print(image_path) + print('-'*20, 'Input Prompt', '-'*20) + print(args.prompt) + print('-'*20, 'Chat Output', '-'*20) + print(res) diff --git a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2_6/README.md b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2_6/README.md index 3a47448f6c2..6063a286b4a 100644 --- a/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2_6/README.md +++ b/python/llm/example/GPU/HuggingFace/Multimodal/MiniCPM-V-2_6/README.md @@ -108,11 +108,11 @@ set SYCL_CACHE_PERSISTENT=1 - chat without streaming mode: ``` - python ./generate.py --prompt 'What is in the image?' + python ./chat.py --prompt 'What is in the image?' ``` - chat in streaming mode: ``` - python ./generate.py --prompt 'What is in the image?' --stream + python ./chat.py --prompt 'What is in the image?' --stream ``` > [!TIP]