From 82f4f7dd35c89531dc257a3dd9bcec321d5c3333 Mon Sep 17 00:00:00 2001 From: sgwhat Date: Thu, 20 Jun 2024 22:08:30 +0800 Subject: [PATCH 1/7] add more model exampels for pipelien parallel inference --- .../run_codellama_arc_2_card.sh | 25 +++++++++++++++++++ .../run_mistral_arc_2_card.sh | 19 ++++++++++++++ .../run_solar_arc_2_card.sh | 17 +++++++++++++ .../transformers/pipeline_parallel.py | 3 ++- 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh create mode 100644 python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh create mode 100644 python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh new file mode 100644 index 00000000000..8ec32e976f4 --- /dev/null +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh @@ -0,0 +1,25 @@ +source /opt/intel/oneapi/setvars.sh +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=9090 +export FI_PROVIDER=tcp +export USE_XETLA=OFF +export OMP_NUM_THREADS=6 +export IPEX_LLM_QUANTIZE_KV_CACHE=1 +if [[ $KERNEL_VERSION != *"6.5"* ]]; then + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +fi +export TORCH_LLM_ALLREDUCE=0 + +NUM_GPUS=2 # number of used GPU + +# To run CodeLlama-7b-Instruct-hf +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'codellama/CodeLlama-7b-Instruct-hf' --gpu-num $NUM_GPUS + +# To run CodeLlama-13b-Instruct-hf +# CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ +# generate.py --repo-id-or-model-path 'codellama/CodeLlama-7b-Instruct-hf' --gpu-num $NUM_GPUS + +# To run CodeLlama-34b-Instruct-hf +# CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ +# generate.py --repo-id-or-model-path 'codellama/CodeLlama-34b-Instruct-hf' --gpu-num $NUM_GPUS diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh new file mode 100644 index 00000000000..35802549feb --- /dev/null +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh @@ -0,0 +1,19 @@ +source /opt/intel/oneapi/setvars.sh +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=9090 +export FI_PROVIDER=tcp +export USE_XETLA=OFF +export OMP_NUM_THREADS=6 +export IPEX_LLM_QUANTIZE_KV_CACHE=1 +if [[ $KERNEL_VERSION != *"6.5"* ]]; then + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +fi +export TORCH_LLM_ALLREDUCE=0 + +NUM_GPUS=2 # number of used GPU + +# To run Mistral-7B-v0.1 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'mistralai/Mistral-7B-v0.1' --gpu-num $NUM_GPUS + +# Todo: Mixtral-8x7B-Instruct-v0.1 diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh new file mode 100644 index 00000000000..077062234aa --- /dev/null +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh @@ -0,0 +1,17 @@ +source /opt/intel/oneapi/setvars.sh +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=9090 +export FI_PROVIDER=tcp +export USE_XETLA=OFF +export OMP_NUM_THREADS=6 +export IPEX_LLM_QUANTIZE_KV_CACHE=1 +if [[ $KERNEL_VERSION != *"6.5"* ]]; then + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +fi +export TORCH_LLM_ALLREDUCE=0 + +NUM_GPUS=2 # number of used GPU + +# To run SOLAR-10.7B-Instruct-v1.0 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'upstage/SOLAR-10.7B-Instruct-v1.0' --gpu-num $NUM_GPUS diff --git a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py index 92cb7d726a3..92c3756741d 100644 --- a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py +++ b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py @@ -234,7 +234,8 @@ def pipeline_parallel_generate(self, "make sure that `pad_token_id` is defined.") next_ids = next_ids * unfinished_sequences + pad_token_id * (1 - unfinished_sequences) - if isinstance(outputs.past_key_values, tuple) and local_rank != 0: + #if isinstance(outputs.past_key_values, tuple) and local_rank != 0: + if self.config.model_type == "baichuan" and local_rank != 0: value_placeholder = torch.empty_like((outputs.past_key_values)[-1][0]) past_key_values_placeholder = tuple( (value_placeholder, value_placeholder) for _ in range(layer_start) From 3f92e9f20dfcde6b969e07d97f5eee0ffab12c02 Mon Sep 17 00:00:00 2001 From: sgwhat Date: Fri, 21 Jun 2024 01:15:30 +0800 Subject: [PATCH 2/7] add mixtral and vicuna models --- .../run_mistral_arc_2_card.sh | 4 ++- .../run_vicuna_arc_2_card.sh | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh index 35802549feb..05b21467241 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh @@ -16,4 +16,6 @@ NUM_GPUS=2 # number of used GPU CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ generate.py --repo-id-or-model-path 'mistralai/Mistral-7B-v0.1' --gpu-num $NUM_GPUS -# Todo: Mixtral-8x7B-Instruct-v0.1 +# To run Mixtral-8x7B-Instruct-v0.1 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'mistralai/Mixtral-8x7B-Instruct-v0.1' --gpu-num $NUM_GPUS diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh new file mode 100644 index 00000000000..8d41e32315a --- /dev/null +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh @@ -0,0 +1,25 @@ +source /opt/intel/oneapi/setvars.sh +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=9090 +export FI_PROVIDER=tcp +export USE_XETLA=OFF +export OMP_NUM_THREADS=6 +export IPEX_LLM_QUANTIZE_KV_CACHE=1 +if [[ $KERNEL_VERSION != *"6.5"* ]]; then + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +fi +export TORCH_LLM_ALLREDUCE=0 + +NUM_GPUS=2 # number of used GPU + +# To run vicuna-7b-v1.3 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'lmsys/vicuna-7b-v1.3' --gpu-num $NUM_GPUS + +# To run vicuna-13b-v1.3 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'lmsys/vicuna-13b-v1.3' --gpu-num $NUM_GPUS + +# To run vicuna-33b-v1.3 +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path 'lmsys/vicuna-33b-v1.3' --gpu-num $NUM_GPUS From e3680c3aa3ca1aaa56ee8acb42e7ed2756119344 Mon Sep 17 00:00:00 2001 From: sgwhat Date: Sat, 22 Jun 2024 00:54:10 +0800 Subject: [PATCH 3/7] add yi model and past_kv supprot for chatglm family --- .../run_mistral_arc_2_card.sh | 4 ++-- .../run_vicuna_arc_2_card.sh | 8 +++---- .../run_yi_arc_2_card.sh | 21 +++++++++++++++++++ .../transformers/pipeline_parallel.py | 4 ++-- 4 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh index 05b21467241..0e742f26520 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh @@ -17,5 +17,5 @@ CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $N generate.py --repo-id-or-model-path 'mistralai/Mistral-7B-v0.1' --gpu-num $NUM_GPUS # To run Mixtral-8x7B-Instruct-v0.1 -CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ - generate.py --repo-id-or-model-path 'mistralai/Mixtral-8x7B-Instruct-v0.1' --gpu-num $NUM_GPUS +# CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ +# generate.py --repo-id-or-model-path 'mistralai/Mixtral-8x7B-Instruct-v0.1' --gpu-num $NUM_GPUS diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh index 8d41e32315a..bee8bcef9d2 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh @@ -17,9 +17,9 @@ CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $N generate.py --repo-id-or-model-path 'lmsys/vicuna-7b-v1.3' --gpu-num $NUM_GPUS # To run vicuna-13b-v1.3 -CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ - generate.py --repo-id-or-model-path 'lmsys/vicuna-13b-v1.3' --gpu-num $NUM_GPUS +# CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ +# generate.py --repo-id-or-model-path 'lmsys/vicuna-13b-v1.3' --gpu-num $NUM_GPUS # To run vicuna-33b-v1.3 -CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ - generate.py --repo-id-or-model-path 'lmsys/vicuna-33b-v1.3' --gpu-num $NUM_GPUS +# CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ +# generate.py --repo-id-or-model-path 'lmsys/vicuna-33b-v1.3' --gpu-num $NUM_GPUS diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh new file mode 100644 index 00000000000..8bf838f5014 --- /dev/null +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh @@ -0,0 +1,21 @@ +source /opt/intel/oneapi/setvars.sh +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=9090 +export FI_PROVIDER=tcp +export USE_XETLA=OFF +export OMP_NUM_THREADS=6 +export IPEX_LLM_QUANTIZE_KV_CACHE=1 +if [[ $KERNEL_VERSION != *"6.5"* ]]; then + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +fi +export TORCH_LLM_ALLREDUCE=0 + +NUM_GPUS=2 # number of used GPU + +# To run Yi-6B-Chat +CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ + generate.py --repo-id-or-model-path '01-ai/Yi-6B-Chat' --gpu-num $NUM_GPUS + +# To run Yi-34B-Chat +# CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ +# generate.py --repo-id-or-model-path '01-ai/Yi-34B-Chat' --gpu-num $NUM_GPUS diff --git a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py index 92c3756741d..31f6e3f6ee5 100644 --- a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py +++ b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py @@ -234,8 +234,8 @@ def pipeline_parallel_generate(self, "make sure that `pad_token_id` is defined.") next_ids = next_ids * unfinished_sequences + pad_token_id * (1 - unfinished_sequences) - #if isinstance(outputs.past_key_values, tuple) and local_rank != 0: - if self.config.model_type == "baichuan" and local_rank != 0: + # Temporarily specify as Baichuan and ChatGLM + if self.config.model_type in ["baichuan", "chatglm"] and local_rank != 0: value_placeholder = torch.empty_like((outputs.past_key_values)[-1][0]) past_key_values_placeholder = tuple( (value_placeholder, value_placeholder) for _ in range(layer_start) From e8fa2e2df1a160c32f766b9fadb1991a399fc2fd Mon Sep 17 00:00:00 2001 From: sgwhat Date: Sat, 22 Jun 2024 01:03:44 +0800 Subject: [PATCH 4/7] add docs --- .../GPU/Pipeline-Parallel-Inference/README.md | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md index a2efa1df992..2a8c23d4c4a 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md @@ -16,7 +16,17 @@ To run this example with IPEX-LLM on Intel GPUs, we have some recommended requir - [baichuan-inc/Baichuan2-13B-Chat](./run_baichuan2_arc_2_card.sh) - [microsoft/Phi-3-mini-4k-instruct](./run_phi3_arc_2_card.sh) - [microsoft/Phi-3-medium-4k-instruct](./run_phi3_arc_2_card.sh) - +- [mistralai/Mistral-7B-v0.1](./run_mistral_arc_2_card.sh) +- [mistralai/Mixtral-8x7B-Instruct-v0.1](./run_mistral_arc_2_card.sh) +- [mistralai/01-ai/Yi-6B-Chat](./run_yi_arc_2_card.sh) +- [mistralai/01-ai/Yi-34B-Chat](./run_yi_arc_2_card.sh) +- [codellama/CodeLlama-7b-Instruct-hf](./run_codellama_arc_2_card.sh) +- [codellama/CodeLlama-13b-Instruct-hf](./run_codellama_arc_2_card.sh) +- [codellama/CodeLlama-34b-Instruct-hf](./run_codellama_arc_2_card.sh) +- [upstage/SOLAR-10.7B-Instruct-v1.0](./run_solar_arc_2_card.sh) +- [lmsys/vicuna-7b-v1.3](./run_vicuna_arc_2_card.sh) +- [lmsys/vicuna-13b-v1.3](./run_vicuna_arc_2_card.sh) +- [lmsys/vicuna-33b-v1.3](./run_vicuna_arc_2_card.sh) ## Example: Run pipeline parallel inference on multiple GPUs @@ -101,6 +111,87 @@ bash run_phi3_arc_2_card.sh + + +
+ Show Mistral/Mixtral example + +#### Run Mistral-7B-v0.1 / Mixtral-8x7B-Instruct-v0.1 on two Intel Arc A770 + +You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Mistral/Mixtral to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. + +```bash +pip install transformers==4.37.0 +bash run_mistral_arc_2_card.sh +``` + +
+ + + +
+ Show Yi example + +#### Run Yi-6B-Chat / Yi-34B-Chat on two Intel Arc A770 + +You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Yi to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. + +```bash +pip install transformers==4.37.0 +bash run_yi_arc_2_card.sh +``` + +
+ + + +
+ Show Codellama example + +#### Run CodeLlama-7b-Instruct-hf / CodeLlama-13b-Instruct-hf / CodeLlama-34b-Instruct-hf on two Intel Arc A770 + +You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Codellama to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. + +```bash +pip install transformers==4.37.0 +bash run_codellama_arc_2_card.sh +``` + +
+ + + +
+ Show Solar example + +#### Run SOLAR-10.7B-Instruct-v1.0 on two Intel Arc A770 + +You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Solar to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. + +```bash +pip install transformers==4.37.0 +bash run_solar_arc_2_card.sh +``` + +
+ + + +
+ Show Vicuna example + +#### Run vicuna-7b-v1.3 / vicuna-13b-v1.3 / vicuna-33b-v1.3 on two Intel Arc A770 + +You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Vicuna to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. + +```bash +pip install transformers==4.37.0 +bash run_vicuna_arc_2_card.sh +``` + +
+ + ### 3. Sample Output #### [meta-llama/Llama-2-13b-chat-hf](https://huggingface.co/meta-llama/Llama-2-13b-chat-hf) ```log From 23602cb3bc552c1d63adadf5279a114c8d172822 Mon Sep 17 00:00:00 2001 From: sgwhat Date: Sat, 22 Jun 2024 01:05:35 +0800 Subject: [PATCH 5/7] doc update --- .../llm/example/GPU/Pipeline-Parallel-Inference/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md index 2a8c23d4c4a..c1a68e97a35 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md @@ -18,8 +18,8 @@ To run this example with IPEX-LLM on Intel GPUs, we have some recommended requir - [microsoft/Phi-3-medium-4k-instruct](./run_phi3_arc_2_card.sh) - [mistralai/Mistral-7B-v0.1](./run_mistral_arc_2_card.sh) - [mistralai/Mixtral-8x7B-Instruct-v0.1](./run_mistral_arc_2_card.sh) -- [mistralai/01-ai/Yi-6B-Chat](./run_yi_arc_2_card.sh) -- [mistralai/01-ai/Yi-34B-Chat](./run_yi_arc_2_card.sh) +- [01-ai/Yi-6B-Chat](./run_yi_arc_2_card.sh) +- [01-ai/Yi-34B-Chat](./run_yi_arc_2_card.sh) - [codellama/CodeLlama-7b-Instruct-hf](./run_codellama_arc_2_card.sh) - [codellama/CodeLlama-13b-Instruct-hf](./run_codellama_arc_2_card.sh) - [codellama/CodeLlama-34b-Instruct-hf](./run_codellama_arc_2_card.sh) @@ -118,7 +118,7 @@ bash run_phi3_arc_2_card.sh #### Run Mistral-7B-v0.1 / Mixtral-8x7B-Instruct-v0.1 on two Intel Arc A770 -You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Mistral/Mixtral to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. +You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Mistral / Mixtral to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash pip install transformers==4.37.0 From 18288f0654877070ade19686be0003f4131fb917 Mon Sep 17 00:00:00 2001 From: sgwhat Date: Sat, 22 Jun 2024 01:15:50 +0800 Subject: [PATCH 6/7] add license --- .../GPU/Pipeline-Parallel-Inference/README.md | 6 ------ .../run_codellama_arc_2_card.sh | 18 +++++++++++++++++- .../run_mistral_arc_2_card.sh | 16 ++++++++++++++++ .../run_solar_arc_2_card.sh | 16 ++++++++++++++++ .../run_vicuna_arc_2_card.sh | 16 ++++++++++++++++ .../run_yi_arc_2_card.sh | 16 ++++++++++++++++ 6 files changed, 81 insertions(+), 7 deletions(-) diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md index c1a68e97a35..6ac498634ef 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md @@ -89,7 +89,6 @@ bash run_qwen1.5_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Baichuan2 to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash -pip install transformers==4.37.0 bash run_baichuan2_arc_2_card.sh ``` @@ -105,7 +104,6 @@ bash run_baichuan2_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Phi3 to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash -pip install transformers==4.37.0 bash run_phi3_arc_2_card.sh ``` @@ -137,7 +135,6 @@ bash run_mistral_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Yi to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash -pip install transformers==4.37.0 bash run_yi_arc_2_card.sh ``` @@ -153,7 +150,6 @@ bash run_yi_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Codellama to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash -pip install transformers==4.37.0 bash run_codellama_arc_2_card.sh ``` @@ -169,7 +165,6 @@ bash run_codellama_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Solar to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash -pip install transformers==4.37.0 bash run_solar_arc_2_card.sh ``` @@ -185,7 +180,6 @@ bash run_solar_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Vicuna to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash -pip install transformers==4.37.0 bash run_vicuna_arc_2_card.sh ``` diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh index 8ec32e976f4..3de4c1ca197 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_codellama_arc_2_card.sh @@ -1,3 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + source /opt/intel/oneapi/setvars.sh export MASTER_ADDR=127.0.0.1 export MASTER_PORT=9090 @@ -18,7 +34,7 @@ CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $N # To run CodeLlama-13b-Instruct-hf # CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ -# generate.py --repo-id-or-model-path 'codellama/CodeLlama-7b-Instruct-hf' --gpu-num $NUM_GPUS +# generate.py --repo-id-or-model-path 'codellama/CodeLlama-13b-Instruct-hf' --gpu-num $NUM_GPUS # To run CodeLlama-34b-Instruct-hf # CCL_ZE_IPC_EXCHANGE=sockets torchrun --standalone --nnodes=1 --nproc-per-node $NUM_GPUS \ diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh index 0e742f26520..0eb36a42785 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_mistral_arc_2_card.sh @@ -1,3 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + source /opt/intel/oneapi/setvars.sh export MASTER_ADDR=127.0.0.1 export MASTER_PORT=9090 diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh index 077062234aa..76781745f14 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_solar_arc_2_card.sh @@ -1,3 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + source /opt/intel/oneapi/setvars.sh export MASTER_ADDR=127.0.0.1 export MASTER_PORT=9090 diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh index bee8bcef9d2..2f50893ed61 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_vicuna_arc_2_card.sh @@ -1,3 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + source /opt/intel/oneapi/setvars.sh export MASTER_ADDR=127.0.0.1 export MASTER_PORT=9090 diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh index 8bf838f5014..ff0a83eb84a 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/run_yi_arc_2_card.sh @@ -1,3 +1,19 @@ +# +# Copyright 2016 The BigDL Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + source /opt/intel/oneapi/setvars.sh export MASTER_ADDR=127.0.0.1 export MASTER_PORT=9090 From ed6998584b48bb33a2ee0922b84dec466dd60795 Mon Sep 17 00:00:00 2001 From: sgwhat Date: Sat, 22 Jun 2024 01:17:43 +0800 Subject: [PATCH 7/7] update --- python/llm/example/GPU/Pipeline-Parallel-Inference/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md index 6ac498634ef..0698674411c 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md @@ -104,6 +104,7 @@ bash run_baichuan2_arc_2_card.sh You could specify `--repo-id-or-model-path` in the test script to be the huggingface repo id for Phi3 to be downloaded, or the path to the huggingface checkpoint folder. Besides, you could change `NUM_GPUS` to the number of GPUs you have on your machine. ```bash +pip install transformers==4.37.0 bash run_phi3_arc_2_card.sh ```