.github/workflows/llm_performance_tests.yml

separate prefill into a process #921

Workflow file for this run

.github/workflows/llm_performance_tests.yml at fdfb094

	name: LLM Performance Test

	# Cancel previous runs in the PR when you push new commits
	concurrency:
	group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number \|\| github.run_id }}
	cancel-in-progress: true

	permissions:
	contents: read

	# Controls when the action will run.
	on:
	# schedule:
	# - cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China
	# please uncomment it for PR tests
	# pull_request:
	# branches: [main]
	# paths:
	# - ".github/workflows/llm_performance_tests.yml"
	# - "python/llm/test/benchmark/**"
	# - "python/llm/dev/benchmark/all-in-one/**"
	# workflow_dispatch:
	# inputs:
	# arc:
	# description: "If trigger performance test on Arc"
	# required: false
	# type: boolean
	# default: true
	# spr:
	# description: "If trigger performance test on SPR"
	# required: false
	# type: boolean
	# default: true
	# core:
	# description: "If trigger performance test on Core"
	# required: false
	# type: boolean
	# default: true
	# igpu:
	# description: "If trigger performance test on iGPU"
	# required: false
	# type: boolean
	# default: true
	workflow_call:
	inputs:
	checkout-ref:
	description: 'ref for checking out, including branch, tag or SHA'
	required: true
	type: string
	arc:
	description: "If trigger performance test on Arc"
	required: false
	type: boolean
	default: true
	spr:
	description: "If trigger performance test on SPR"
	required: false
	type: boolean
	default: true
	core:
	description: "If trigger performance test on Core"
	required: false
	type: boolean
	default: true
	igpu:
	description: "If trigger performance test on iGPU"
	required: false
	type: boolean
	default: true

	# A workflow run is made up of one or more jobs that can run sequentially or in parallel
	jobs:
	llm-cpp-build:
	uses: ./.github/workflows/llm-binary-build.yml
	with:
	platform: ${{ (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main')) && 'Windows,Linux' \|\| 'Dummy' }}

	llm-performance-test-on-arc:
	if: ${{ github.event.schedule \|\| ( github.event_name == 'workflow_dispatch' && inputs.arc ) }} # please comment it for PR tests
	needs: llm-cpp-build
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11"]
	runs-on: [self-hosted, llm, perf]
	env:
	OMP_NUM_THREADS: 16
	THREAD_NUM: 16
	ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
	CSV_SAVE_PATH: ${{ (github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main'))) && '/mnt/disk1/nightly_perf_gpu/' \|\| '/mnt/disk1/pr_perf_gpu/' }}

	steps:
	- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
	with:
	repository: 'intel-analytics/ipex-llm'
	ref: ${{ inputs.checkout-ref }}

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	shell: bash
	# pip install transformers_stream_generator for model internlm-chat-7b-8k
	# pip install tiktoken for model Qwen-7B-Chat-10-12
	# pip install matplotlib for model Qwen-VL-Chat
	run: \|
	python -m pip install --upgrade pip
	python -m pip install --upgrade wheel
	python -m pip install --upgrade omegaconf
	python -m pip install --upgrade pandas
	python -m pip install --upgrade einops
	python -m pip install --upgrade transformers_stream_generator
	python -m pip install --upgrade tiktoken
	python -m pip install --upgrade matplotlib

	# specific for test on certain commits
	- name: Download llm binary
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/download-llm-binary

	- name: Install IPEX-LLM from source
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/setup-llm-env
	with:
	extra-dependency: "xpu_2.1"

	- name: Install IPEX-LLM from Pypi
	if: ${{ github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
	shell: bash
	run: \|
	pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
	test_version_date=`date -d 'yesterday' '+%Y%m%d'`
	if ! pip show ipex-llm \| grep $test_version_date; then
	echo "Did not install ipex-llm with excepted version $test_version_date"
	exit 1
	fi

	- name: Test installed xpu version
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	python -m pip install --upgrade pytest
	python -m pip install --upgrade expecttest
	bash python/llm/test/run-llm-install-tests.sh

	- name: Test on xpu(transformers==4.36.2)
	shell: bash
	run: \|
	date_for_test_version=$(date -d yesterday +%Y-%m-%d)
	sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	mkdir test_batch1
	mkdir test_batch2
	mkdir test_batch4
	# batch_size 1
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run.py
	# change csv name
	sed -i 's/{today}/{today}_test1_batch1/g' run.py
	python run.py
	mv *.csv test_batch1
	# batch_size 2
	cd ../../../../../
	cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# change csv name
	sed -i 's/batch1/batch2/g' run.py
	python run.py
	mv *.csv test_batch2
	# batch_size 4
	cd ../../../../../
	cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# change csv name
	sed -i 's/batch2/batch4/g' run.py
	python run.py
	mv *.csv test_batch4

	- name: Test on xpu(transformers==4.37.0)
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	# upgrade transformers for model Qwen/Qwen1.5-7B-Chat
	python -m pip install transformers==4.37.0
	# batch_size 1
	cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# change csv name
	sed -i 's/test1_batch4/test2_batch1/g' run.py
	python run.py
	mv *.csv test_batch1
	# batch_size 2
	cd ../../../../../
	cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# change csv name
	sed -i 's/batch1/batch2/g' run.py
	python run.py
	mv *.csv test_batch2
	# batch_size 4
	cd ../../../../../
	cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# change csv name
	sed -i 's/batch2/batch4/g' run.py
	python run.py
	mv *.csv test_batch4

	- name: Test on xpu(transformers==4.40.0)
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	# upgrade transformers for model Qwen/Qwen1.5-MoE-A2.7B-Chat
	python -m pip install transformers==4.40.0
	python -m pip install trl
	# batch_size 1
	cp python/llm/test/benchmark/arc-perf-transformers-440.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# change csv name
	sed -i 's/test2_batch4/test3_batch1/g' run.py
	python run.py
	mv *.csv test_batch1

	- name: Concat csv and generate html
	shell: bash
	run: \|
	# batch_size 1
	cd python/llm/dev/benchmark/all-in-one/test_batch1
	python ../../../../test/benchmark/concat_csv.py
	for file in *.csv; do
	if [[ $file != test ]]; then
	cp "$file" $CSV_SAVE_PATH/batch_size_1
	fi
	done
	python -m pip install pandas==1.5.3
	cd ../../../../test/benchmark
	python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1
	# batch_size 2
	cd ../../../../
	cd python/llm/dev/benchmark/all-in-one/test_batch2
	python ../../../../test/benchmark/concat_csv.py
	for file in *.csv; do
	if [[ $file != test ]]; then
	cp "$file" $CSV_SAVE_PATH/batch_size_2
	fi
	done
	cd ../../../../test/benchmark
	python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2
	# batch_size 4
	cd ../../../../
	cd python/llm/dev/benchmark/all-in-one/test_batch4
	python ../../../../test/benchmark/concat_csv.py
	for file in *.csv; do
	if [[ $file != test ]]; then
	cp "$file" $CSV_SAVE_PATH/batch_size_4
	fi
	done
	cd ../../../../test/benchmark
	python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_4

	- name: Merge and sort csv files of multiple batches and generate html
	shell: bash
	run: \|
	cd python/llm/test/benchmark
	mkdir merged_temp
	# go through all the files and go to merged_temp
	cd ../../dev/benchmark/all-in-one/test_batch1
	for file in *.csv; do
	if [[ $file != test ]]; then
	cp "$file" ../../../../test/benchmark/merged_temp
	fi
	done
	cd ../test_batch2
	for file in *.csv; do
	if [[ $file != test ]]; then
	cp "$file" ../../../../test/benchmark/merged_temp
	fi
	done
	cd ../test_batch4
	for file in *.csv; do
	if [[ $file != test ]]; then
	cp "$file" ../../../../test/benchmark/merged_temp
	fi
	done
	cd ../../../../test/benchmark
	python merge_csv_batch.py -f ./merged_temp
	cd merged_temp
	find . -name "batch.csv" -delete
	for file in *.csv; do
	cp "$file" $CSV_SAVE_PATH/merged
	done
	cd ..
	python csv_to_html.py -f $CSV_SAVE_PATH/merged
	rm -r merged_temp

	- name: Update html in parent folder
	shell: bash
	run: \|
	cd python/llm/test/benchmark
	python update_html_in_parent_folder.py -f $CSV_SAVE_PATH

	- name: Check and upload results to ftp
	shell: bash
	run: \|
	# batch_size 1
	cd python/llm/dev/benchmark/all-in-one/test_batch1
	python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml
	python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml
	python ../../../../test/benchmark/check_results.py -c test3 -y ../../../../test/benchmark/arc-perf-transformers-440.yaml
	find . -name "test.csv" -delete
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
	elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
	fi
	cd ../
	rm -r test_batch1
	# batch_size 2
	cd test_batch2
	python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml
	python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml
	find . -name "test.csv" -delete
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
	elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
	fi
	cd ../
	rm -r test_batch2
	# batch_size 4
	cd test_batch4
	python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch4.yaml
	python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml
	find . -name "test.csv" -delete
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
	elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
	fi
	cd ../
	rm -r test_batch4


	llm-performance-test-on-spr:
	if: ${{ github.event.schedule \|\| ( github.event_name == 'workflow_dispatch' && inputs.spr ) }} # please comment it for PR tests
	needs: llm-cpp-build
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11"]
	runs-on: [self-hosted, llm, spr-perf]
	env:
	OMP_NUM_THREADS: 16
	THREAD_NUM: 16
	ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
	steps:
	- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
	with:
	repository: 'intel-analytics/ipex-llm'
	ref: ${{ inputs.checkout-ref }}

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	shell: bash
	run: \|
	python -m pip install --upgrade pip
	python -m pip install --upgrade wheel
	python -m pip install --upgrade omegaconf
	python -m pip install --upgrade pandas
	python -m pip install --upgrade einops
	python -m pip install --upgrade tiktoken
	python -m pip install --upgrade transformers_stream_generator

	# specific for test on certain commits
	- name: Download llm binary
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/download-llm-binary

	- name: Install IPEX-LLM from source
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/setup-llm-env

	- name: Install IPEX-LLM from Pypi
	if: ${{ github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
	shell: bash
	run: \|
	pip install --pre --upgrade ipex-llm[all] --extra-index-url https://download.pytorch.org/whl/cpu
	test_version_date=`date -d 'yesterday' '+%Y%m%d'`
	if ! pip show ipex-llm \| grep $test_version_date; then
	echo "Did not install ipex-llm with excepted version $test_version_date"
	exit 1
	fi

	- name: Test on cpu
	shell: bash
	run: \|
	date_for_test_version=$(date -d yesterday +%Y-%m-%d)
	sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

	mv python/llm/test/benchmark/cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	export http_proxy=${HTTP_PROXY}
	export https_proxy=${HTTPS_PROXY}
	source ipex-llm-init -t
	export OMP_NUM_THREADS=48
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run.py
	python run.py
	cp ./*.csv /mnt/disk1/models/nightly_perf_cpu
	cd ../../../test/benchmark
	python -m pip install pandas==1.5.3
	python csv_to_html.py -f /mnt/disk1/models/nightly_perf_cpu
	cd /mnt/disk1/models/nightly_perf_cpu
	for f in *.html; do
	curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
	done

	llm-performance-test-on-core:
	if: ${{ github.event.schedule \|\| ( github.event_name == 'workflow_dispatch' && inputs.core ) }} # please comment it for PR tests
	needs: llm-cpp-build
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: windows
	platform: dp
	python-version: "3.11"
	# - os: windows
	# platform: lp
	# python-version: "3.11"
	runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"]
	env:
	ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
	CSV_SAVE_PATH: ${{ (github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main'))) && 'D:/action-runners/nightly_perf_core_' \|\| 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/
	steps:
	- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
	with:
	repository: 'intel-analytics/ipex-llm'
	ref: ${{ inputs.checkout-ref }}

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	shell: bash
	run: \|
	python -m pip install --upgrade pip
	python -m pip install --upgrade wheel
	python -m pip install --upgrade omegaconf pandas
	python -m pip install --upgrade tiktoken einops transformers_stream_generator

	# specific for test on certain commits
	- name: Download llm binary
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/download-llm-binary

	- name: Install IPEX-LLM from source
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/setup-llm-env

	- name: Install IPEX-LLM from Pypi
	if: ${{ github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
	shell: bash
	run: \|
	pip install --pre --upgrade ipex-llm[all]
	test_version_date=`date -d 'yesterday' '+%Y%m%d'`
	if ! pip show ipex-llm \| grep $test_version_date; then
	echo "Did not install ipex-llm with excepted version $test_version_date"
	exit 1
	fi


	- name: Test on core ${{ matrix.platform }}
	shell: bash
	run: \|
	date_for_test_version=$(date -d yesterday +%Y-%m-%d)
	sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

	mv python/llm/test/benchmark/core-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	export http_proxy=${HTTP_PROXY}
	export https_proxy=${HTTPS_PROXY}
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run.py
	python run.py
	cp ./*.csv $CSV_SAVE_PATH
	cd ../../../test/benchmark
	python -m pip install pandas==1.5.3
	python csv_to_html.py -f $CSV_SAVE_PATH
	cd ../../dev/benchmark/all-in-one/
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
	elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
	curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
	fi

	llm-performance-test-on-igpu:
	if: ${{ github.event.schedule \|\| ( github.event_name == 'workflow_dispatch' && inputs.igpu ) }} # please comment it for PR tests
	needs: llm-cpp-build
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: windows
	python-version: "3.11"
	runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-igpu]
	env:
	ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
	steps:
Check failure on line 525 in .github/workflows/llm_performance_tests.yml View workflow run for this annotation GitHub Actions / .github/workflows/llm_performance_tests.yml Invalid workflow file `You have an error in your yaml syntax on line 525`
	- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
	with:
	repository: 'intel-analytics/ipex-llm'
	ref: ${{ inputs.checkout-ref }}

	# TODO: Put the ipex-llm related install process for win gpu into a action function

	# specific for test on certain commits
	- name: Download llm binary
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	uses: ./.github/actions/llm/download-llm-binary

	- name: Prepare for install ipex-llm from source
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	shell: bash
	run: \|
	sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
	sed -i 's/"bigdl-core-xe-batch-21==" + CORE_XE_VERSION/"bigdl-core-xe-batch-21"/g' python/llm/setup.py
	sed -i 's/"bigdl-core-xe-addons-21==" + CORE_XE_VERSION/"bigdl-core-xe-addons-21"/g' python/llm/setup.py
	sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION/"bigdl-core-xe-esimd-21"/g' python/llm/setup.py

	- name: Install ipex-llm and other related packages (install from source)
	if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
	shell: cmd
	run: \|
	call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
	call conda activate igpu-perf

	pip install --upgrade pip
	pip install --upgrade wheel
	pip install --upgrade omegaconf pandas
	pip install --upgrade tiktoken einops transformers_stream_generator matplotlib

	cd python\llm
	python setup.py clean --all bdist_wheel --win
	if not exist dist\ipex_llm*.whl (exit /b 1)
	for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i

	pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
	if %ERRORLEVEL% neq 0 (exit /b 1)
	pip list

	call conda deactivate

	- name: Determine desired ipex-llm version
	if: ${{ github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
	shell: bash
	run: \|
	test_version_date=`date -d 'yesterday' '+%Y%m%d'`
	echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV"

	- name: Install ipex-llm and other related packages (install from pypi)
	if: ${{ github.event.schedule \|\| (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
	shell: cmd
	run: \|
	call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
	call conda activate igpu-perf

	pip install --upgrade pip
	pip install --upgrade wheel
	pip install --upgrade omegaconf pandas
	pip install --upgrade tiktoken einops transformers_stream_generator matplotlib

	pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
	pip show ipex-llm \| findstr %TEST_VERSION_DATE%
	if %ERRORLEVEL% neq 0 (
	echo "Did not install ipex-llm with excepted version %TEST_VERSION_DATE%"
	exit /b 1
	)
	pip list

	call conda deactivate

	- name: Create env for html generation
	shell: cmd
	run: \|
	call conda create -n html-gen python=3.11 -y
	call conda activate html-gen

	pip install pandas==1.5.3
	pip install Jinja2
	pip install "numpy<2.0.0"

	call conda deactivate

	- name: Set directory envs & and fix generated csv date name
	shell: bash
	run: \|
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
	elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
	echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
	else
	echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV"
	fi
	date_for_test_version=$(date -d yesterday +%Y-%m-%d)
	echo "LOG_FILE=${date_for_test_version}_output.txt" >> "$GITHUB_ENV"

	sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

	- name: Add extra warmup for chatglm3-6b int4+fp32 & MiniCPM int4+fp16 int4+fp32 for more stable results
	shell: bash
	run: \|
	sed -i '/^\s*result = run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
	if repo_id in ["THUDM/chatglm3-6b", "THUDM/glm-4-9b-chat", "openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
	run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
	' python/llm/dev/benchmark/all-in-one/run.py

	sed -i '/^\s*result = run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
	if repo_id in ["openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
	run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
	' python/llm/dev/benchmark/all-in-one/run.py

	# 32-32 int4+fp16
	- name: Prepare igpu perf test (32-32 int4+fp16)
	shell: bash
	run: \|
	# hide time info
	# sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml

	- name: Test on igpu (32-32 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (32-32 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_437.yaml

	- name: Test on igpu for transformers 4.37 (32-32 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (32-32 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_438.yaml

	- name: Test on igpu for transformers 4.38 (32-32 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.43 (32-32 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_443.yaml

	- name: Test on igpu for transformers 4.43 (32-32 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.43.1
	pip install trl

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_443.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	if %ERRORLEVEL% neq 0 (exit /b 1)

	pip uninstall trl -y
	call conda deactivate

	- name: Concat csv and generate html (32-32 int4+fp16)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\32-32_int4_fp16\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\32-32_int4_fp16\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\32-32_int4_fp16\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# TODO: create a action function here for different input
	# 1024-128 int4+fp16
	- name: Prepare igpu perf test (1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml

	- name: Test on igpu (1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.36.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml

	- name: Test on igpu for transformers 4.37 (1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_438.yaml

	- name: Test on igpu for transformers 4.38 (1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.43 (1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_443.yaml

	- name: Test on igpu for transformers 4.43 (1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.43.1
	pip install trl

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_443.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	if %ERRORLEVEL% neq 0 (exit /b 1)

	pip uninstall trl -y
	call conda deactivate

	- name: Concat csv and generate html (1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# 2048-256 int4+fp16
	- name: Prepare igpu perf test (2048-256 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml

	- name: Test on igpu (2048-256 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.36.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (2048-256 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_437.yaml

	- name: Test on igpu for transformers 4.37 (2048-256 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (2048-256 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_438.yaml

	- name: Test on igpu for transformers 4.38 (2048-256 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.43 (2048-256 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_443.yaml

	- name: Test on igpu for transformers 4.43 (2048-256 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.43.1
	pip install trl

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_443.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	if %ERRORLEVEL% neq 0 (exit /b 1)

	pip uninstall trl -y
	call conda deactivate

	- name: Concat csv and generate html (2048-256 int4+fp16)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\2048-256_int4_fp16\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256_int4_fp16\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\2048-256_int4_fp16\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# 3072-384 int4+fp16
	- name: Prepare igpu perf test (3072-384 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/2048-256/3072-384/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml

	- name: Test on igpu (3072-384 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.36.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (3072-384 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_437.yaml

	- name: Test on igpu for transformers 4.37 (3072-384 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (3072-384 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_438.yaml

	- name: Test on igpu for transformers 4.38 (3072-384 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.43 (3072-384 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_443.yaml

	- name: Test on igpu for transformers 4.43 (3072-384 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.43.1
	pip install trl

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_443.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	if %ERRORLEVEL% neq 0 (exit /b 1)

	pip uninstall trl -y
	call conda deactivate

	- name: Concat csv and generate html (3072-384 int4+fp16)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\3072-384_int4_fp16\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\3072-384_int4_fp16\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\3072-384_int4_fp16\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# 4096-512 int4+fp16
	- name: Prepare igpu perf test (4096-512 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/3072-384/4096-512/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16.yaml

	- name: Test on igpu (4096-512 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.36.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (4096-512 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_437.yaml

	- name: Test on igpu for transformers 4.37 (4096-512 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (4096-512 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_438.yaml

	- name: Test on igpu for transformers 4.38 (4096-512 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	# NOTE: Gemma2 not working for 4096-512.
	# When it works, uncomment this section and remember to change "'s/{today}_test3/{today}_test1/g'" in next section.

	#- name: Prepare igpu perf test for transformers 4.43 (4096-512 int4+fp16)
	# shell: bash
	# run: \|
	# sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	# sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_443.yaml

	#- name: Test on igpu for transformers 4.43 (4096-512 int4+fp16)
	# shell: cmd
	# run: \|
	# call conda activate igpu-perf
	# pip install transformers==4.43.1
	# pip install trl
	#
	# set SYCL_CACHE_PERSISTENT=1
	# set BIGDL_LLM_XMX_DISABLED=1
	#
	# cd python\llm\dev\benchmark\all-in-one
	# move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_443.yaml config.yaml
	# set PYTHONIOENCODING=utf-8
	# python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
	# if %ERRORLEVEL% neq 0 (exit /b 1)
	# python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	# if %ERRORLEVEL% neq 0 (exit /b 1)
	#
	# pip uninstall trl -y
	# call conda deactivate

	- name: Concat csv and generate html (4096-512 int4+fp16)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\4096-512_int4_fp16\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\4096-512_int4_fp16\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\4096-512_int4_fp16\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# load_low_bit 1024-128 int4+fp16
	- name: Prepare igpu perf test (load_low_bit 1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/4096-512/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml

	- name: Test on igpu (load_low_bit 1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.36.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_437.yaml

	- name: Test on igpu for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (load_low_bit 1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_438.yaml

	- name: Test on igpu for transformers 4.38 (load_low_bit 1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.43 (load_low_bit 1024-128 int4+fp16)
	shell: bash
	run: \|
	sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_443.yaml

	- name: Test on igpu for transformers 4.43 (load_low_bit 1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.43.1
	pip install trl

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_443.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	if %ERRORLEVEL% neq 0 (exit /b 1)

	pip uninstall trl -y
	call conda deactivate

	- name: Concat csv and generate html (load_low_bit 1024-128 int4+fp16)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# 1024-128
	- name: Prepare igpu perf test (1024-128)
	shell: bash
	run: \|
	sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml

	- name: Test on igpu (1024-128)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.36.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1
	REM for llava
	set TRANSFORMERS_OFFLINE=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.37 (1024-128)
	shell: bash
	run: \|
	sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml

	- name: Test on igpu for transformers 4.37 (1024-128)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.37.0

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.38 (1024-128)
	shell: bash
	run: \|
	sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_438.yaml

	- name: Test on igpu for transformers 4.38 (1024-128)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.38.2

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
	if %ERRORLEVEL% neq 0 (exit /b 1)

	call conda deactivate

	- name: Prepare igpu perf test for transformers 4.43 (1024-128)
	shell: bash
	run: \|
	sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
	sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_443.yaml

	- name: Test on igpu for transformers 4.43 (1024-128)
	shell: cmd
	run: \|
	call conda activate igpu-perf
	pip install transformers==4.43.1
	pip install trl

	set SYCL_CACHE_PERSISTENT=1
	set BIGDL_LLM_XMX_DISABLED=1

	cd python\llm\dev\benchmark\all-in-one
	move ..\..\..\test\benchmark\igpu-perf\1024-128_443.yaml config.yaml
	set PYTHONIOENCODING=utf-8
	python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
	if %ERRORLEVEL% neq 0 (exit /b 1)
	python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
	if %ERRORLEVEL% neq 0 (exit /b 1)

	pip uninstall trl -y
	call conda deactivate

	- name: Concat csv and generate html (1024-128)
	shell: cmd
	run: \|
	call conda activate html-gen

	cd python\llm\dev\benchmark\all-in-one
	python ..\..\..\test\benchmark\concat_csv.py
	if %ERRORLEVEL% neq 0 (exit /b 1)
	del /q test.csv
	move *.csv %CSV_SAVE_PATH%\1024-128\
	cd ..\..\..\test\benchmark
	python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\
	if %ERRORLEVEL% neq 0 (exit /b 1)
	move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH%

	call conda deactivate

	# TODO: avoid duplicated code
	- name: Upload results to ftp
	if: ${{ always() }}
	shell: cmd
	run: \|
	cd %CSV_SAVE_PATH%
	IF "${{ github.event_name }}"=="schedule" (
	for %%f in (*.html) do (
	curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
	)
	)
	IF "${{ github.event_name }}"=="workflow_dispatch" (
	IF "${{ inputs.checkout-ref }}"=="main" (
	for %%f in (*.html) do (
	curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
	)
	)
	)
	# for test on machine when encountering error
	# - name: Remove conda env
	# if: ${{ always() }}
	# shell: cmd
	# run: \|
	# call conda env remove -n igpu-perf -y

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

separate prefill into a process #921

Workflow file

separate prefill into a process #921

Jobs

Run details

Workflow file for this run

GitHub Actions / .github/workflows/llm_performance_tests.yml