From efcd95a0db0c46d2da2ad38105b476b6a9f4bd9f Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Fri, 2 Aug 2024 15:03:40 +0800
Subject: [PATCH 01/15] removed LLMs that are no longer needed

Removed:
mistralai/Mistral-7B-v0.1
deepseek-ai/deepseek-coder-6.7b-instruct
---
 python/llm/test/benchmark/arc-perf-test-batch2.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/python/llm/test/benchmark/arc-perf-test-batch2.yaml b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
index e5863465a44..70447fd7f59 100644
--- a/python/llm/test/benchmark/arc-perf-test-batch2.yaml
+++ b/python/llm/test/benchmark/arc-perf-test-batch2.yaml
@@ -4,8 +4,6 @@ repo_id:
   - 'THUDM/chatglm3-6b-4bit'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'mistralai/Mistral-7B-v0.1'
-  - 'deepseek-ai/deepseek-coder-6.7b-instruct'
   - 'THUDM/glm-4-9b-chat'
   - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'Qwen/Qwen-VL-Chat'
@@ -29,4 +27,4 @@ test_api:
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
 exclude:
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit:2048'
-task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'
\ No newline at end of file
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

From 2fec0ddec8dbf11a71150bc3b539ee3128e37305 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Fri, 2 Aug 2024 15:10:14 +0800
Subject: [PATCH 02/15] Update arc-perf-test-batch4.yaml

Removed:
deepseek-ai/deepseek-coder-6.7b-instruct
mistralai/Mistral-7B-v0.1
---
 python/llm/test/benchmark/arc-perf-test-batch4.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/llm/test/benchmark/arc-perf-test-batch4.yaml b/python/llm/test/benchmark/arc-perf-test-batch4.yaml
index 0c83cc9b398..3bfd47963a4 100644
--- a/python/llm/test/benchmark/arc-perf-test-batch4.yaml
+++ b/python/llm/test/benchmark/arc-perf-test-batch4.yaml
@@ -4,8 +4,6 @@ repo_id:
   - 'THUDM/chatglm3-6b-4bit'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'mistralai/Mistral-7B-v0.1' #mwj: need to check
-  - 'deepseek-ai/deepseek-coder-6.7b-instruct'
   - 'THUDM/glm-4-9b-chat'
   - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'Qwen/Qwen-VL-Chat'

From 7224eaff0a801df1bbf60e6e7e6261a07a1f2b4c Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Fri, 2 Aug 2024 15:11:07 +0800
Subject: [PATCH 03/15] Update arc-perf-test.yaml

Removed:
deepseek-ai/deepseek-coder-6.7b-instruct
mistralai/Mistral-7B-v0.1
---
 python/llm/test/benchmark/arc-perf-test.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/llm/test/benchmark/arc-perf-test.yaml b/python/llm/test/benchmark/arc-perf-test.yaml
index 5298514bb09..890b8dbf470 100644
--- a/python/llm/test/benchmark/arc-perf-test.yaml
+++ b/python/llm/test/benchmark/arc-perf-test.yaml
@@ -4,8 +4,6 @@ repo_id:
   - 'THUDM/chatglm3-6b-4bit'
   - 'baichuan-inc/Baichuan2-7B-Chat'
   - 'baichuan-inc/Baichuan2-13B-Chat-4bit'
-  - 'mistralai/Mistral-7B-v0.1'
-  - 'deepseek-ai/deepseek-coder-6.7b-instruct'
   - 'THUDM/glm-4-9b-chat'
   - 'openbmb/MiniCPM-2B-sft-bf16'
   - 'Qwen/Qwen-VL-Chat'

From f76f062971fc9d67430acd0afd25e840619b5252 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Mon, 5 Aug 2024 16:39:58 +0800
Subject: [PATCH 04/15] Create arc-perf-transformers-438.yaml

---
 .../benchmark/arc-perf-transformers-438.yaml   | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 python/llm/test/benchmark/python/llm/test/benchmark/arc-perf-transformers-438.yaml

diff --git a/python/llm/test/benchmark/python/llm/test/benchmark/arc-perf-transformers-438.yaml b/python/llm/test/benchmark/python/llm/test/benchmark/arc-perf-transformers-438.yaml
new file mode 100644
index 00000000000..eda2815e9df
--- /dev/null
+++ b/python/llm/test/benchmark/python/llm/test/benchmark/arc-perf-transformers-438.yaml
@@ -0,0 +1,18 @@
+# For the models that require transformers 4.38.2
+repo_id:
+  - 'stablelm/stablelm-zephyr-3b'
+  - 'Gemma/gemma-7b-it'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 1 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

From ed190cf270150b2c0e9fe3c8db218a731f43bb32 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Mon, 5 Aug 2024 16:48:28 +0800
Subject: [PATCH 05/15] Moved arc-perf-transformers-438.yaml location

---
 .../llm/test/benchmark => }/arc-perf-transformers-438.yaml        | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename python/llm/test/benchmark/{python/llm/test/benchmark => }/arc-perf-transformers-438.yaml (100%)

diff --git a/python/llm/test/benchmark/python/llm/test/benchmark/arc-perf-transformers-438.yaml b/python/llm/test/benchmark/arc-perf-transformers-438.yaml
similarity index 100%
rename from python/llm/test/benchmark/python/llm/test/benchmark/arc-perf-transformers-438.yaml
rename to python/llm/test/benchmark/arc-perf-transformers-438.yaml

From fcc12ae948e86a98bcc72bf454f0f3559f1050ca Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Mon, 5 Aug 2024 16:58:58 +0800
Subject: [PATCH 06/15] Create arc-perf-transformers-438-batch2.yaml

---
 .../arc-perf-transformers-438-batch2.yaml      | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
new file mode 100644
index 00000000000..3c9fa68bd10
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
@@ -0,0 +1,18 @@
+# For the models that require transformers 4.38.2
+repo_id:
+  - 'stablelm/stablelm-zephyr-3b'
+  - 'Gemma/gemma-7b-it'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 2 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

From 2e629fc824910377cf44b13bec360cbb2a4de285 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Mon, 5 Aug 2024 17:03:44 +0800
Subject: [PATCH 07/15] Create arc-perf-transformers-438-batch4.yaml

---
 .../arc-perf-transformers-438-batch4.yaml     | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
new file mode 100644
index 00000000000..f7499fe8806
--- /dev/null
+++ b/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
@@ -0,0 +1,20 @@
+# For the models that require transformers 4.38.2
+repo_id:
+  - 'stablelm/stablelm-zephyr-3b'
+  - 'Gemma/gemma-7b-it'
+local_model_hub: '/mnt/disk1/models'
+warm_up: 1
+num_trials: 3
+num_beams: 1 # default to greedy search
+low_bit: 'sym_int4' # default to use 'sym_int4' (i.e. symmetric int4)
+batch_size: 4 # default to 1
+in_out_pairs:
+  - '32-32'
+  - '1024-128'
+  - '2048-256'
+test_api:
+  - "transformer_int4_fp16_gpu"  # on Intel GPU
+cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
+exclude:
+  - 'Gemma/gemma-7b-it:2048'
+task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

From 4a818a161dc1cacea309ef19310287f3775702b1 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Tue, 6 Aug 2024 16:49:53 +0800
Subject: [PATCH 08/15] Update llm_performance_tests.yml to include transformer
 4.38.2

---
 .github/workflows/llm_performance_tests.yml | 32 +++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
index 9185e4282d5..00ce9bc07e7 100644
--- a/.github/workflows/llm_performance_tests.yml
+++ b/.github/workflows/llm_performance_tests.yml
@@ -213,6 +213,38 @@ jobs:
           sed -i 's/batch2/batch4/g' run.py
           python run.py
           mv *.csv test_batch4
+
+  - name: Test on xpu(transformers==4.38.2)
+        shell: bash
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          export USE_XETLA=OFF
+          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
+          # upgrade transformers for model stablelm/stablelm-zephyr-3b & Gemma/gemma-7b-it
+          python -m pip install transformers==4.38.2
+          # batch_size 1
+          cp python/llm/test/benchmark/arc-perf-transformers-438.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name
+          sed -i 's/test1_batch4/test2_batch1/g' run.py
+          python run.py
+          mv *.csv test_batch1
+          # batch_size 2
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch1/batch2/g' run.py
+          python run.py
+          mv *.csv test_batch2
+          # batch_size 4
+          cd ../../../../../ 
+          cp python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
+          # change csv name 
+          sed -i 's/batch2/batch4/g' run.py
+          python run.py
+          mv *.csv test_batch4
       
       - name: Test on xpu(transformers==4.40.0)
         shell: bash

From 420d2136624aa81367e11e5e1dc4eb8d4cb8982e Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:42:05 +0800
Subject: [PATCH 09/15] Update arc-perf-transformers-438-batch2.yaml

Disabled Gemma/gemma-7b-it for now
---
 python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
index 3c9fa68bd10..41e54326b4e 100644
--- a/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
+++ b/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
@@ -1,7 +1,7 @@
 # For the models that require transformers 4.38.2
 repo_id:
   - 'stablelm/stablelm-zephyr-3b'
-  - 'Gemma/gemma-7b-it'
+  #- 'Gemma/gemma-7b-it' #Temporarily removed due to performance issues
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3

From 727a97e99cbb784a246edb1a1f34a85334ed4f37 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:43:05 +0800
Subject: [PATCH 10/15] Update arc-perf-transformers-438-batch4.yaml

Disabled Gemma/gemma-7b-it for now
---
 .../test/benchmark/arc-perf-transformers-438-batch4.yaml    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
index f7499fe8806..82393e67355 100644
--- a/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
+++ b/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
@@ -1,7 +1,7 @@
 # For the models that require transformers 4.38.2
 repo_id:
   - 'stablelm/stablelm-zephyr-3b'
-  - 'Gemma/gemma-7b-it'
+  #- 'Gemma/gemma-7b-it' #disabled due to performance issues
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -15,6 +15,6 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
-exclude:
-  - 'Gemma/gemma-7b-it:2048'
+#exclude:
+  #- 'Gemma/gemma-7b-it:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

From a68e7c9095db3db8e7302c8d470bd7356bdc3910 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:43:40 +0800
Subject: [PATCH 11/15] Update arc-perf-transformers-438.yaml

Disabled Gemma/gemma-7b-it for now
---
 python/llm/test/benchmark/arc-perf-transformers-438.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438.yaml b/python/llm/test/benchmark/arc-perf-transformers-438.yaml
index eda2815e9df..a8b31cfe327 100644
--- a/python/llm/test/benchmark/arc-perf-transformers-438.yaml
+++ b/python/llm/test/benchmark/arc-perf-transformers-438.yaml
@@ -1,7 +1,7 @@
 # For the models that require transformers 4.38.2
 repo_id:
   - 'stablelm/stablelm-zephyr-3b'
-  - 'Gemma/gemma-7b-it'
+  #- 'Gemma/gemma-7b-it' #disabled due to performance issues
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3

From 95a4fc014b4a8130bd216737d2b061188af1bfc4 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:52:48 +0800
Subject: [PATCH 12/15] Update llm_performance_tests.yml

---
 .github/workflows/llm_performance_tests.yml | 64 ++++++++++-----------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
index 00ce9bc07e7..e7b1b890c93 100644
--- a/.github/workflows/llm_performance_tests.yml
+++ b/.github/workflows/llm_performance_tests.yml
@@ -145,85 +145,85 @@ jobs:
           python -m pip install --upgrade expecttest
           bash python/llm/test/run-llm-install-tests.sh
 
-      - name: Test on xpu(transformers==4.36.2)
+      - name: Test on xpu(transformers==4.38.2)
         shell: bash
         run: |
-          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
-          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
           source /opt/intel/oneapi/setvars.sh
           export USE_XETLA=OFF
           export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
-          cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
-          cd python/llm/dev/benchmark/all-in-one
-          mkdir test_batch1
-          mkdir test_batch2
-          mkdir test_batch4
+          # upgrade transformers for model stablelm/stablelm-zephyr-3b & Gemma/gemma-7b-it
+          python -m pip install transformers==4.38.2
           # batch_size 1
-          # hide time info
-          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
+          cp python/llm/test/benchmark/arc-perf-transformers-438.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cd python/llm/dev/benchmark/all-in-one
           # change csv name
-          sed -i 's/{today}/{today}_test1_batch1/g' run.py
+          sed -i 's/test1_batch4/test2_batch1/g' run.py
           python run.py
           mv *.csv test_batch1
           # batch_size 2
           cd ../../../../../ 
-          cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
-          # change csv name
+          # change csv name 
           sed -i 's/batch1/batch2/g' run.py
           python run.py
           mv *.csv test_batch2
           # batch_size 4
           cd ../../../../../ 
-          cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
-          # change csv name
+          # change csv name 
           sed -i 's/batch2/batch4/g' run.py
           python run.py
           mv *.csv test_batch4
-
-      - name: Test on xpu(transformers==4.37.0)
+          
+      - name: Test on xpu(transformers==4.36.2)
         shell: bash
         run: |
+          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
+          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
           source /opt/intel/oneapi/setvars.sh
           export USE_XETLA=OFF
           export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
-          # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
-          python -m pip install transformers==4.37.0
-          # batch_size 1
-          cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
+          mkdir test_batch1
+          mkdir test_batch2
+          mkdir test_batch4
+          # batch_size 1
+          # hide time info
+          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
           # change csv name
-          sed -i 's/test1_batch4/test2_batch1/g' run.py
+          sed -i 's/{today}/{today}_test1_batch1/g' run.py
           python run.py
           mv *.csv test_batch1
           # batch_size 2
           cd ../../../../../ 
-          cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
-          # change csv name 
+          # change csv name
           sed -i 's/batch1/batch2/g' run.py
           python run.py
           mv *.csv test_batch2
           # batch_size 4
           cd ../../../../../ 
-          cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
-          # change csv name 
+          # change csv name
           sed -i 's/batch2/batch4/g' run.py
           python run.py
           mv *.csv test_batch4
 
-  - name: Test on xpu(transformers==4.38.2)
+      - name: Test on xpu(transformers==4.37.0)
         shell: bash
         run: |
           source /opt/intel/oneapi/setvars.sh
           export USE_XETLA=OFF
           export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
-          # upgrade transformers for model stablelm/stablelm-zephyr-3b & Gemma/gemma-7b-it
-          python -m pip install transformers==4.38.2
+          # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
+          python -m pip install transformers==4.37.0
           # batch_size 1
-          cp python/llm/test/benchmark/arc-perf-transformers-438.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
           # change csv name
           sed -i 's/test1_batch4/test2_batch1/g' run.py
@@ -231,7 +231,7 @@ jobs:
           mv *.csv test_batch1
           # batch_size 2
           cd ../../../../../ 
-          cp python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
           # change csv name 
           sed -i 's/batch1/batch2/g' run.py
@@ -239,7 +239,7 @@ jobs:
           mv *.csv test_batch2
           # batch_size 4
           cd ../../../../../ 
-          cp python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
+          cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
           cd python/llm/dev/benchmark/all-in-one
           # change csv name 
           sed -i 's/batch2/batch4/g' run.py

From 917a2d2e3c35bf018d195e083fc00332c3a72d47 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:59:41 +0800
Subject: [PATCH 13/15] Update arc-perf-transformers-438-batch2.yaml

---
 python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml b/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
index 41e54326b4e..46c3905782c 100644
--- a/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
+++ b/python/llm/test/benchmark/arc-perf-transformers-438-batch2.yaml
@@ -1,7 +1,6 @@
 # For the models that require transformers 4.38.2
 repo_id:
   - 'stablelm/stablelm-zephyr-3b'
-  #- 'Gemma/gemma-7b-it' #Temporarily removed due to performance issues
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3

From 805c556671e5828d4dd0f96dcf26b90828636702 Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:00:10 +0800
Subject: [PATCH 14/15] Update arc-perf-transformers-438-batch4.yaml

---
 .../llm/test/benchmark/arc-perf-transformers-438-batch4.yaml   | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml b/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
index 82393e67355..598b5ce1771 100644
--- a/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
+++ b/python/llm/test/benchmark/arc-perf-transformers-438-batch4.yaml
@@ -1,7 +1,6 @@
 # For the models that require transformers 4.38.2
 repo_id:
   - 'stablelm/stablelm-zephyr-3b'
-  #- 'Gemma/gemma-7b-it' #disabled due to performance issues
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3
@@ -15,6 +14,4 @@ in_out_pairs:
 test_api:
   - "transformer_int4_fp16_gpu"  # on Intel GPU
 cpu_embedding: False # whether put embedding to CPU (only avaiable now for gpu win related test_api)
-#exclude:
-  #- 'Gemma/gemma-7b-it:2048'
 task: 'continuation' # task can be 'continuation', 'QA' and 'summarize'

From bf40d9538f09f47f5f4732e3cb95f1f0b6fa084f Mon Sep 17 00:00:00 2001
From: SichengStevenLi <144295301+SichengStevenLi@users.noreply.github.com>
Date: Wed, 7 Aug 2024 18:00:25 +0800
Subject: [PATCH 15/15] Update arc-perf-transformers-438.yaml

---
 python/llm/test/benchmark/arc-perf-transformers-438.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/llm/test/benchmark/arc-perf-transformers-438.yaml b/python/llm/test/benchmark/arc-perf-transformers-438.yaml
index a8b31cfe327..02065068962 100644
--- a/python/llm/test/benchmark/arc-perf-transformers-438.yaml
+++ b/python/llm/test/benchmark/arc-perf-transformers-438.yaml
@@ -1,7 +1,6 @@
 # For the models that require transformers 4.38.2
 repo_id:
   - 'stablelm/stablelm-zephyr-3b'
-  #- 'Gemma/gemma-7b-it' #disabled due to performance issues
 local_model_hub: '/mnt/disk1/models'
 warm_up: 1
 num_trials: 3