Skip to content

Commit

Permalink
only test batch-4
Browse files Browse the repository at this point in the history
  • Loading branch information
@MeouSker77
MeouSker77 committed Jun 19, 2024
1 parent ca23261 commit 2d949f9
Showing 1 changed file with 131 additions and 124 deletions.
255 changes: 131 additions & 124 deletions .github/workflows/llm_performance_tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,90 +118,94 @@ jobs:
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
mkdir test_batch1
mkdir test_batch2
# mkdir test_batch1
# mkdir test_batch2
mkdir test_batch4
# batch_size 1
# hide time info
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
# change csv name
sed -i 's/{today}/{today}_test1_batch1/g' run.py
python run.py
mv *.csv test_batch1
# batch_size 2
cd ../../../../../
cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/batch1/batch2/g' run.py
python run.py
mv *.csv test_batch2
# # batch_size 1
# # hide time info
# sed -i 's/str(end - st)/"xxxxxx"/g' run.py
# # change csv name
# sed -i 's/{today}/{today}_test1_batch1/g' run.py
# python run.py
# mv *.csv test_batch1
# # batch_size 2
# cd ../../../../../
# cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
# cd python/llm/dev/benchmark/all-in-one
# # change csv name
# sed -i 's/batch1/batch2/g' run.py
# python run.py
# mv *.csv test_batch2
# batch_size 4
cd ../../../../../
cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/batch2/batch4/g' run.py
######## mwj add
sed -i 's/str(end - st)/"xxxxxx"/g' run.py
sed -i 's/{today}/{today}_test1_batch4/g' run.py
#######
# # change csv name
# sed -i 's/batch2/batch4/g' run.py
python run.py
mv *.csv test_batch4
- name: Test on xpu(transformers==4.37.0)
shell: bash
run: |
source /opt/intel/oneapi/setvars.sh
export USE_XETLA=OFF
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
# upgrade transformers for model Qwen/Qwen1.5-7B-Chat
python -m pip install transformers==4.37.0
# batch_size 1
cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/test1_batch2/test2_batch1/g' run.py
python run.py
mv *.csv test_batch1
# batch_size 2
cd ../../../../../
cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/batch1/batch2/g' run.py
python run.py
mv *.csv test_batch2
# batch_size 4
cd ../../../../../
cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
cd python/llm/dev/benchmark/all-in-one
# change csv name
sed -i 's/batch2/batch4/g' run.py
python run.py
mv *.csv test_batch4
# - name: Test on xpu(transformers==4.37.0)
# shell: bash
# run: |
# source /opt/intel/oneapi/setvars.sh
# export USE_XETLA=OFF
# export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
# # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
# python -m pip install transformers==4.37.0
# # batch_size 1
# cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
# cd python/llm/dev/benchmark/all-in-one
# # change csv name
# sed -i 's/test1_batch2/test2_batch1/g' run.py
# python run.py
# mv *.csv test_batch1
# # batch_size 2
# cd ../../../../../
# cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
# cd python/llm/dev/benchmark/all-in-one
# # change csv name
# sed -i 's/batch1/batch2/g' run.py
# python run.py
# mv *.csv test_batch2
# # batch_size 4
# cd ../../../../../
# cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
# cd python/llm/dev/benchmark/all-in-one
# # change csv name
# sed -i 's/batch2/batch4/g' run.py
# python run.py
# mv *.csv test_batch4

- name: Concat csv and generate html
shell: bash
run: |
# batch_size 1
# # batch_size 1
cd python/llm/dev/benchmark/all-in-one/test_batch1
python ../../../../test/benchmark/concat_csv.py
for file in *.csv; do
if [[ $file != *test* ]]; then
cp "$file" $CSV_SAVE_PATH/batch_size_1
fi
done
python -m pip install pandas==1.5.3
cd ../../../../test/benchmark
python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1
# batch_size 2
cd ../../../../
cd python/llm/dev/benchmark/all-in-one/test_batch2
python ../../../../test/benchmark/concat_csv.py
for file in *.csv; do
if [[ $file != *test* ]]; then
cp "$file" $CSV_SAVE_PATH/batch_size_2
fi
done
# python ../../../../test/benchmark/concat_csv.py
# for file in *.csv; do
# if [[ $file != *test* ]]; then
# cp "$file" $CSV_SAVE_PATH/batch_size_1
# fi
# done
# python -m pip install pandas==1.5.3
cd ../../../../test/benchmark
python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2
# python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1
# # batch_size 2
# cd ../../../../
# cd python/llm/dev/benchmark/all-in-one/test_batch2
# python ../../../../test/benchmark/concat_csv.py
# for file in *.csv; do
# if [[ $file != *test* ]]; then
# cp "$file" $CSV_SAVE_PATH/batch_size_2
# fi
# done
# cd ../../../../test/benchmark
# python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2
# batch_size 4
cd ../../../../
cd python/llm/dev/benchmark/all-in-one/test_batch4
Expand All @@ -214,40 +218,40 @@ jobs:
cd ../../../../test/benchmark
python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_4
- name: Merge and sort csv files of multiple batches and generate html
shell: bash
run: |
cd python/llm/test/benchmark
mkdir merged_temp
# go through all the files and go to merged_temp
cd ../../dev/benchmark/all-in-one/test_batch1
for file in *.csv; do
if [[ $file != *test* ]]; then
cp "$file" ../../../../test/benchmark/merged_temp
fi
done
cd ../test_batch2
for file in *.csv; do
if [[ $file != *test* ]]; then
cp "$file" ../../../../test/benchmark/merged_temp
fi
done
cd ../test_batch4
for file in *.csv; do
if [[ $file != *test* ]]; then
cp "$file" ../../../../test/benchmark/merged_temp
fi
done
cd ../../../../test/benchmark
python merge_csv_batch.py -f ./merged_temp
cd merged_temp
find . -name "*batch*.csv" -delete
for file in *.csv; do
cp "$file" $CSV_SAVE_PATH/merged
done
cd ..
python csv_to_html.py -f $CSV_SAVE_PATH/merged
rm -r merged_temp
# - name: Merge and sort csv files of multiple batches and generate html
# shell: bash
# run: |
# cd python/llm/test/benchmark
# mkdir merged_temp
# # go through all the files and go to merged_temp
# cd ../../dev/benchmark/all-in-one/test_batch1
# for file in *.csv; do
# if [[ $file != *test* ]]; then
# cp "$file" ../../../../test/benchmark/merged_temp
# fi
# done
# cd ../test_batch2
# for file in *.csv; do
# if [[ $file != *test* ]]; then
# cp "$file" ../../../../test/benchmark/merged_temp
# fi
# done
# cd ../test_batch4
# for file in *.csv; do
# if [[ $file != *test* ]]; then
# cp "$file" ../../../../test/benchmark/merged_temp
# fi
# done
# cd ../../../../test/benchmark
# python merge_csv_batch.py -f ./merged_temp
# cd merged_temp
# find . -name "*batch*.csv" -delete
# for file in *.csv; do
# cp "$file" $CSV_SAVE_PATH/merged
# done
# cd ..
# python csv_to_html.py -f $CSV_SAVE_PATH/merged
# rm -r merged_temp

- name: Update html in parent folder
shell: bash
Expand All @@ -259,29 +263,32 @@ jobs:
shell: bash
run: |
# batch_size 1
cd python/llm/dev/benchmark/all-in-one/test_batch1
python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml
python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml
find . -name "*test*.csv" -delete
if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
fi
cd ../
rm -r test_batch1
# batch_size 2
cd test_batch2
python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml
python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml
find . -name "*test*.csv" -delete
if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
fi
cd ../
rm -r test_batch2
# cd python/llm/dev/benchmark/all-in-one/test_batch1
# python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml
# python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml
# find . -name "*test*.csv" -delete
# if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
# curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
# fi
# cd ../
# rm -r test_batch1
# # batch_size 2
# cd test_batch2
# python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml
# # python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml
# find . -name "*test*.csv" -delete
# if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
# curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
# fi
# cd ../
# rm -r test_batch2
# batch_size 4
### mwj add
cd python/llm/dev/benchmark/all-in-one/
####
cd test_batch4
python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch4.yaml
python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml
# python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml
find . -name "*test*.csv" -delete
if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
Expand Down

0 comments on commit 2d949f9

Please sign in to comment.