diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index a546e017a16..139ef1d54da 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -118,90 +118,94 @@ jobs: export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml cd python/llm/dev/benchmark/all-in-one - mkdir test_batch1 - mkdir test_batch2 + # mkdir test_batch1 + # mkdir test_batch2 mkdir test_batch4 - # batch_size 1 - # hide time info - sed -i 's/str(end - st)/"xxxxxx"/g' run.py - # change csv name - sed -i 's/{today}/{today}_test1_batch1/g' run.py - python run.py - mv *.csv test_batch1 - # batch_size 2 - cd ../../../../../ - cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml - cd python/llm/dev/benchmark/all-in-one - # change csv name - sed -i 's/batch1/batch2/g' run.py - python run.py - mv *.csv test_batch2 + # # batch_size 1 + # # hide time info + # sed -i 's/str(end - st)/"xxxxxx"/g' run.py + # # change csv name + # sed -i 's/{today}/{today}_test1_batch1/g' run.py + # python run.py + # mv *.csv test_batch1 + # # batch_size 2 + # cd ../../../../../ + # cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml + # cd python/llm/dev/benchmark/all-in-one + # # change csv name + # sed -i 's/batch1/batch2/g' run.py + # python run.py + # mv *.csv test_batch2 # batch_size 4 cd ../../../../../ cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml cd python/llm/dev/benchmark/all-in-one - # change csv name - sed -i 's/batch2/batch4/g' run.py + ######## mwj add + sed -i 's/str(end - st)/"xxxxxx"/g' run.py + sed -i 's/{today}/{today}_test1_batch4/g' run.py + ####### + # # change csv name + # sed -i 's/batch2/batch4/g' run.py python run.py mv *.csv test_batch4 - - name: Test on xpu(transformers==4.37.0) - shell: bash - run: | - source /opt/intel/oneapi/setvars.sh - export USE_XETLA=OFF - export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 - # upgrade transformers for model Qwen/Qwen1.5-7B-Chat - python -m pip install transformers==4.37.0 - # batch_size 1 - cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml - cd python/llm/dev/benchmark/all-in-one - # change csv name - sed -i 's/test1_batch2/test2_batch1/g' run.py - python run.py - mv *.csv test_batch1 - # batch_size 2 - cd ../../../../../ - cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml - cd python/llm/dev/benchmark/all-in-one - # change csv name - sed -i 's/batch1/batch2/g' run.py - python run.py - mv *.csv test_batch2 - # batch_size 4 - cd ../../../../../ - cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml - cd python/llm/dev/benchmark/all-in-one - # change csv name - sed -i 's/batch2/batch4/g' run.py - python run.py - mv *.csv test_batch4 + # - name: Test on xpu(transformers==4.37.0) + # shell: bash + # run: | + # source /opt/intel/oneapi/setvars.sh + # export USE_XETLA=OFF + # export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + # # upgrade transformers for model Qwen/Qwen1.5-7B-Chat + # python -m pip install transformers==4.37.0 + # # batch_size 1 + # cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml + # cd python/llm/dev/benchmark/all-in-one + # # change csv name + # sed -i 's/test1_batch2/test2_batch1/g' run.py + # python run.py + # mv *.csv test_batch1 + # # batch_size 2 + # cd ../../../../../ + # cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml + # cd python/llm/dev/benchmark/all-in-one + # # change csv name + # sed -i 's/batch1/batch2/g' run.py + # python run.py + # mv *.csv test_batch2 + # # batch_size 4 + # cd ../../../../../ + # cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml + # cd python/llm/dev/benchmark/all-in-one + # # change csv name + # sed -i 's/batch2/batch4/g' run.py + # python run.py + # mv *.csv test_batch4 - name: Concat csv and generate html shell: bash run: | - # batch_size 1 + # # batch_size 1 cd python/llm/dev/benchmark/all-in-one/test_batch1 - python ../../../../test/benchmark/concat_csv.py - for file in *.csv; do - if [[ $file != *test* ]]; then - cp "$file" $CSV_SAVE_PATH/batch_size_1 - fi - done - python -m pip install pandas==1.5.3 - cd ../../../../test/benchmark - python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1 - # batch_size 2 - cd ../../../../ - cd python/llm/dev/benchmark/all-in-one/test_batch2 - python ../../../../test/benchmark/concat_csv.py - for file in *.csv; do - if [[ $file != *test* ]]; then - cp "$file" $CSV_SAVE_PATH/batch_size_2 - fi - done + # python ../../../../test/benchmark/concat_csv.py + # for file in *.csv; do + # if [[ $file != *test* ]]; then + # cp "$file" $CSV_SAVE_PATH/batch_size_1 + # fi + # done + # python -m pip install pandas==1.5.3 cd ../../../../test/benchmark - python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2 + # python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1 + # # batch_size 2 + # cd ../../../../ + # cd python/llm/dev/benchmark/all-in-one/test_batch2 + # python ../../../../test/benchmark/concat_csv.py + # for file in *.csv; do + # if [[ $file != *test* ]]; then + # cp "$file" $CSV_SAVE_PATH/batch_size_2 + # fi + # done + # cd ../../../../test/benchmark + # python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2 # batch_size 4 cd ../../../../ cd python/llm/dev/benchmark/all-in-one/test_batch4 @@ -214,40 +218,40 @@ jobs: cd ../../../../test/benchmark python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_4 - - name: Merge and sort csv files of multiple batches and generate html - shell: bash - run: | - cd python/llm/test/benchmark - mkdir merged_temp - # go through all the files and go to merged_temp - cd ../../dev/benchmark/all-in-one/test_batch1 - for file in *.csv; do - if [[ $file != *test* ]]; then - cp "$file" ../../../../test/benchmark/merged_temp - fi - done - cd ../test_batch2 - for file in *.csv; do - if [[ $file != *test* ]]; then - cp "$file" ../../../../test/benchmark/merged_temp - fi - done - cd ../test_batch4 - for file in *.csv; do - if [[ $file != *test* ]]; then - cp "$file" ../../../../test/benchmark/merged_temp - fi - done - cd ../../../../test/benchmark - python merge_csv_batch.py -f ./merged_temp - cd merged_temp - find . -name "*batch*.csv" -delete - for file in *.csv; do - cp "$file" $CSV_SAVE_PATH/merged - done - cd .. - python csv_to_html.py -f $CSV_SAVE_PATH/merged - rm -r merged_temp + # - name: Merge and sort csv files of multiple batches and generate html + # shell: bash + # run: | + # cd python/llm/test/benchmark + # mkdir merged_temp + # # go through all the files and go to merged_temp + # cd ../../dev/benchmark/all-in-one/test_batch1 + # for file in *.csv; do + # if [[ $file != *test* ]]; then + # cp "$file" ../../../../test/benchmark/merged_temp + # fi + # done + # cd ../test_batch2 + # for file in *.csv; do + # if [[ $file != *test* ]]; then + # cp "$file" ../../../../test/benchmark/merged_temp + # fi + # done + # cd ../test_batch4 + # for file in *.csv; do + # if [[ $file != *test* ]]; then + # cp "$file" ../../../../test/benchmark/merged_temp + # fi + # done + # cd ../../../../test/benchmark + # python merge_csv_batch.py -f ./merged_temp + # cd merged_temp + # find . -name "*batch*.csv" -delete + # for file in *.csv; do + # cp "$file" $CSV_SAVE_PATH/merged + # done + # cd .. + # python csv_to_html.py -f $CSV_SAVE_PATH/merged + # rm -r merged_temp - name: Update html in parent folder shell: bash @@ -259,29 +263,32 @@ jobs: shell: bash run: | # batch_size 1 - cd python/llm/dev/benchmark/all-in-one/test_batch1 - python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml - python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml - find . -name "*test*.csv" -delete - if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then - curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/ - fi - cd ../ - rm -r test_batch1 - # batch_size 2 - cd test_batch2 - python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml - python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml - find . -name "*test*.csv" -delete - if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then - curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/ - fi - cd ../ - rm -r test_batch2 + # cd python/llm/dev/benchmark/all-in-one/test_batch1 + # python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml + # python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml + # find . -name "*test*.csv" -delete + # if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then + # curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/ + # fi + # cd ../ + # rm -r test_batch1 + # # batch_size 2 + # cd test_batch2 + # python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml + # # python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml + # find . -name "*test*.csv" -delete + # if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then + # curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/ + # fi + # cd ../ + # rm -r test_batch2 # batch_size 4 + ### mwj add + cd python/llm/dev/benchmark/all-in-one/ + #### cd test_batch4 python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch4.yaml - python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml + # python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml find . -name "*test*.csv" -delete if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/