Update performance test regarding updated default `transformers==4.37…

….0` (#11869) * Update igpu performance from transformers 4.36.2 to 4.37.0 (#11841) * upgrade arc perf test to transformers 4.37 (#11842) * fix load low bit com dtype (#11832) * feat: add mixed_precision argument on ppl longbench evaluation * fix: delete extra code * feat: upgrade arc perf test to transformers 4.37 * fix: add missing codes * fix: keep perf test for qwen-vl-chat in transformers 4.36 * fix: remove extra space * fix: resolve pr comment * fix: add empty line * fix: add pip install for spr and core test * fix: delete extra comments * fix: remove python -m for pip * Revert "fix load low bit com dtype (#11832)" This reverts commit 6841a9a. --------- Co-authored-by: Zhao Changmin <[email protected]> Co-authored-by: Jinhe Tang <[email protected]> * add transformers==4.36 for qwen vl in igpu-perf (#11846) * add transformers==4.36.2 for qwen-vl * Small update --------- Co-authored-by: Yuwen Hu <[email protected]> * fix: remove qwen-7b on core test (#11851) * fix: remove qwen-7b on core test * fix: change delete to comment --------- Co-authored-by: Jinhe Tang <[email protected]> * replce filename (#11854) * fix: remove qwen-7b on core test * fix: change delete to comment * fix: replace filename --------- Co-authored-by: Jinhe Tang <[email protected]> * fix: delete extra comments (#11863) * Remove transformers installation for temp test purposes * Small fix * Small update --------- Co-authored-by: Chu,Youcheng <[email protected]> Co-authored-by: Zhao Changmin <[email protected]> Co-authored-by: Jinhe Tang <[email protected]> Co-authored-by: Zijie Li <[email protected]> Co-authored-by: Chu,Youcheng <[email protected]>
intel-analytics · Aug 20, 2024 · 0d58c2f · 0d58c2f
1 parent 5e8286f
commit 0d58c2f
Show file tree

Hide file tree

Showing 25 changed files with 202 additions and 244 deletions.
diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
diff --git a/python/llm/test/benchmark/arc-perf-test-batch4.yaml b/python/llm/test/benchmark/arc-perf-test-batch4.yaml
diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
diff --git a/python/llm/test/benchmark/arc-perf-transformers-436-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-436-batch2.yaml
@@ -0,0 +1,16 @@
+repo_id:
+  - 'Qwen/Qwen-VL-Chat'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 2 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-436-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-436-batch4.yaml
@@ -0,0 +1,18 @@
+repo_id:
+  - 'Qwen/Qwen-VL-Chat'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 4 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+exclude:
+  - 'Qwen/Qwen-VL-Chat:2048'
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-436.yaml b/python/llm/test/benchmark/arc-perf-transformers-436.yaml
@@ -0,0 +1,16 @@
+repo_id:
+  - 'Qwen/Qwen-VL-Chat'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml
@@ -6,6 +6,18 @@ repo_id:
   - 'microsoft/phi-3-vision-128k-instruct'
   - 'Qwen/Qwen2-7B-Instruct'
   - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'meta-llama/Llama-2-7b-chat-hf'
+  - 'meta-llama/Llama-2-13b-chat-hf'
+  - 'THUDM/chatglm3-6b-4bit'
+  - 'baichuan-inc/Baichuan2-7B-Chat'
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
+  - 'THUDM/glm-4-9b-chat'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
+  #- 'SmerkyG/rwkv-5-world-7b' #this model only fp32 is supported for now, fp16 and bf16 are not supported
+  - '01-ai/Yi-6B-Chat'
+  - 'mistralai/Mistral-7B-Instruct-v0.2'
+  - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
+  - '01-ai/Yi-1.5-6B-Chat'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -19,4 +31,6 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+exclude:
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml
@@ -6,6 +6,18 @@ repo_id:
   - 'microsoft/phi-3-vision-128k-instruct'
   - 'Qwen/Qwen2-7B-Instruct'
   - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'meta-llama/Llama-2-7b-chat-hf'
+  - 'meta-llama/Llama-2-13b-chat-hf'
+  - 'THUDM/chatglm3-6b-4bit'
+  - 'baichuan-inc/Baichuan2-7B-Chat'
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
+  - 'THUDM/glm-4-9b-chat'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
+  #- 'SmerkyG/rwkv-5-world-7b' #this model only fp32 is supported for now, fp16 and bf16 are not supported
+  - '01-ai/Yi-6B-Chat'
+  - 'mistralai/Mistral-7B-Instruct-v0.2'
+  - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
+  - '01-ai/Yi-1.5-6B-Chat'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -22,4 +34,8 @@ cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu w
 exclude:
   - 'Qwen/Qwen1.5-7B-Chat:2048'
   - 'meta-llama/Meta-Llama-3-8B-Instruct:2048'
-task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
+  - 'meta-llama/Llama-2-13b-chat-hf:2048'
+  - 'baichuan-inc/Baichuan2-7B-Chat:2048'
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit:1024'
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/arc-perf-transformers-437.yaml b/python/llm/test/benchmark/arc-perf-transformers-437.yaml
@@ -6,6 +6,18 @@ repo_id:
   - 'microsoft/phi-3-vision-128k-instruct'
   - 'Qwen/Qwen2-7B-Instruct'
   - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'meta-llama/Llama-2-7b-chat-hf'
+  - 'meta-llama/Llama-2-13b-chat-hf'
+  - 'THUDM/chatglm3-6b-4bit'
+  - 'baichuan-inc/Baichuan2-7B-Chat'
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
+  - 'THUDM/glm-4-9b-chat'
+  - 'openbmb/MiniCPM-2B-sft-bf16'
+  #- 'SmerkyG/rwkv-5-world-7b' #this model only fp32 is supported for now, fp16 and bf16 are not supported
+  - '01-ai/Yi-6B-Chat'
+  - 'mistralai/Mistral-7B-Instruct-v0.2'
+  - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
+  - '01-ai/Yi-1.5-6B-Chat'
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -19,4 +31,6 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+exclude:
+  - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
diff --git a/python/llm/test/benchmark/core-perf-test.yaml b/python/llm/test/benchmark/core-perf-test.yaml
@@ -3,7 +3,7 @@ repo_id:
   - 'THUDM/chatglm3-6b'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'internlm/internlm-chat-7b'
-  - 'Qwen/Qwen-7B-Chat'
+ # - 'Qwen/Qwen-7B-Chat'  # requires transformers < 4.37.0
   - 'BAAI/AquilaChat2-7B'
   - 'meta-llama/Llama-2-7b-chat-hf'
   - 'WisdomShell/CodeShell-7B'

diff --git a/python/llm/test/benchmark/igpu-perf/1024-128.yaml b/python/llm/test/benchmark/igpu-perf/1024-128.yaml
@@ -10,9 +10,15 @@ repo_id:
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - 'RWKV/v5-Eagle-7B-HF'
   - '01-ai/Yi-6B-Chat'
-  - 'Qwen/Qwen-VL-Chat'
   - 'openbmb/MiniCPM-1B-sft-bf16'
   - 'openbmb/MiniCPM-2B-sft-bf16'
+  - 'Qwen/Qwen1.5-7B-Chat'
+  - 'Qwen/Qwen2-1.5B-Instruct'
+  - 'Qwen/Qwen2-7B-Instruct'
+  - 'microsoft/Phi-3-mini-4k-instruct'
+  - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'microsoft/phi-3-vision-128k-instruct'
+  - 'openbmb/MiniCPM-V-2_6'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/...est/benchmark/igpu-perf/1024-128_437.yaml → ...est/benchmark/igpu-perf/1024-128_436.yaml b/...est/benchmark/igpu-perf/1024-128_437.yaml → ...est/benchmark/igpu-perf/1024-128_436.yaml
@@ -1,11 +1,5 @@
 repo_id:
-  - 'Qwen/Qwen1.5-7B-Chat'
-  - 'Qwen/Qwen2-1.5B-Instruct'
-  - 'Qwen/Qwen2-7B-Instruct'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'microsoft/Phi-3-mini-128k-instruct'
-  - 'microsoft/phi-3-vision-128k-instruct'
-  - 'openbmb/MiniCPM-V-2_6'
+  - 'Qwen/Qwen-VL-Chat'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml
@@ -9,9 +9,15 @@ repo_id:
   - 'mistralai/Mistral-7B-Instruct-v0.2'
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
-  - 'Qwen/Qwen-VL-Chat'
   - 'openbmb/MiniCPM-1B-sft-bf16'
   - 'openbmb/MiniCPM-2B-sft-bf16'
+  - 'Qwen/Qwen1.5-7B-Chat'
+  - 'Qwen/Qwen2-1.5B-Instruct'
+  - 'Qwen/Qwen2-7B-Instruct'
+  - 'microsoft/Phi-3-mini-4k-instruct'
+  - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'microsoft/phi-3-vision-128k-instruct'
+  - 'openbmb/MiniCPM-V-2_6'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/...ark/igpu-perf/1024-128_int4_fp16_437.yaml → ...ark/igpu-perf/1024-128_int4_fp16_436.yaml b/...ark/igpu-perf/1024-128_int4_fp16_437.yaml → ...ark/igpu-perf/1024-128_int4_fp16_436.yaml
@@ -1,11 +1,5 @@
 repo_id:
-  - 'Qwen/Qwen1.5-7B-Chat'
-  - 'Qwen/Qwen2-1.5B-Instruct'
-  - 'Qwen/Qwen2-7B-Instruct'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'microsoft/Phi-3-mini-128k-instruct'
-  - 'microsoft/phi-3-vision-128k-instruct'
-  - 'openbmb/MiniCPM-V-2_6'
+  - 'Qwen/Qwen-VL-Chat'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml b/python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml
@@ -9,9 +9,14 @@ repo_id:
   - 'mistralai/Mistral-7B-Instruct-v0.2'
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
-  - 'Qwen/Qwen-VL-Chat'
   - 'openbmb/MiniCPM-1B-sft-bf16'
   - 'openbmb/MiniCPM-2B-sft-bf16'
+  - 'Qwen/Qwen1.5-7B-Chat'
+  - 'Qwen/Qwen2-1.5B-Instruct'
+  - 'Qwen/Qwen2-7B-Instruct'
+  - 'microsoft/Phi-3-mini-4k-instruct'
+  - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'microsoft/phi-3-vision-128k-instruct'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/...rf/1024-128_int4_fp16_loadlowbit_437.yaml → ...rf/1024-128_int4_fp16_loadlowbit_436.yaml b/...rf/1024-128_int4_fp16_loadlowbit_437.yaml → ...rf/1024-128_int4_fp16_loadlowbit_436.yaml
@@ -1,10 +1,5 @@
 repo_id:
-  - 'Qwen/Qwen1.5-7B-Chat'
-  - 'Qwen/Qwen2-1.5B-Instruct'
-  - 'Qwen/Qwen2-7B-Instruct'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'microsoft/Phi-3-mini-128k-instruct'
-  - 'microsoft/phi-3-vision-128k-instruct'
+  - 'Qwen/Qwen-VL-Chat'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml b/python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml
@@ -9,9 +9,15 @@ repo_id:
   - 'mistralai/Mistral-7B-Instruct-v0.2'
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
-  - 'Qwen/Qwen-VL-Chat'
   - 'openbmb/MiniCPM-1B-sft-bf16'
   - 'openbmb/MiniCPM-2B-sft-bf16'
+  - 'Qwen/Qwen1.5-7B-Chat'
+  - 'Qwen/Qwen2-1.5B-Instruct'
+  - 'Qwen/Qwen2-7B-Instruct'
+  - 'microsoft/Phi-3-mini-4k-instruct'
+  - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'microsoft/phi-3-vision-128k-instruct'
+  - 'openbmb/MiniCPM-V-2_6'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/...ark/igpu-perf/2048-256_int4_fp16_437.yaml → ...ark/igpu-perf/2048-256_int4_fp16_436.yaml b/...ark/igpu-perf/2048-256_int4_fp16_437.yaml → ...ark/igpu-perf/2048-256_int4_fp16_436.yaml
@@ -1,11 +1,5 @@
 repo_id:
-  - 'Qwen/Qwen1.5-7B-Chat'
-  - 'Qwen/Qwen2-1.5B-Instruct'
-  - 'Qwen/Qwen2-7B-Instruct'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'microsoft/Phi-3-mini-128k-instruct'
-  - 'microsoft/phi-3-vision-128k-instruct'
-  - 'openbmb/MiniCPM-V-2_6'
+  - 'Qwen/Qwen-VL-Chat'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml b/python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml
@@ -8,9 +8,15 @@ repo_id:
   - 'mistralai/Mistral-7B-Instruct-v0.2'
   - 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
   - '01-ai/Yi-6B-Chat'
-  - 'Qwen/Qwen-VL-Chat'
   - 'openbmb/MiniCPM-1B-sft-bf16'
   - 'openbmb/MiniCPM-2B-sft-bf16'
+  - 'Qwen/Qwen1.5-7B-Chat'
+  - 'Qwen/Qwen2-1.5B-Instruct'
+  - 'Qwen/Qwen2-7B-Instruct'
+  - 'microsoft/Phi-3-mini-4k-instruct'
+  - 'microsoft/Phi-3-mini-128k-instruct'
+  - 'microsoft/phi-3-vision-128k-instruct'
+  - 'openbmb/MiniCPM-V-2_6'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3

diff --git a/...ark/igpu-perf/3072-384_int4_fp16_437.yaml → ...ark/igpu-perf/3072-384_int4_fp16_436.yaml b/...ark/igpu-perf/3072-384_int4_fp16_437.yaml → ...ark/igpu-perf/3072-384_int4_fp16_436.yaml
@@ -1,11 +1,5 @@
 repo_id:
-  - 'Qwen/Qwen1.5-7B-Chat'
-  - 'Qwen/Qwen2-1.5B-Instruct'
-  - 'Qwen/Qwen2-7B-Instruct'
-  - 'microsoft/Phi-3-mini-4k-instruct'
-  - 'microsoft/Phi-3-mini-128k-instruct'
-  - 'microsoft/phi-3-vision-128k-instruct'
-  - 'openbmb/MiniCPM-V-2_6'
+  - 'Qwen/Qwen-VL-Chat'
 local_model_hub: 'path to your local model hub'
 warm_up: 1
 num_trials: 3
@@ -15,5 +9,5 @@ batch_size: 1 # default to 1
 in_out_pairs:
   - '3072-384'
 test_api:
-  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows (catch GPU peak memory)
+  - "transformer_int4_fp16_gpu_win" # on Intel GPU for Windows, use fp16 for non-linear layer
 cpu_embedding: True # whether put embedding to CPU (only avaiable now for gpu win related test_api)