llm_performance_tests.yml

name: LLM Performance Test

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

# Controls when the action will run.
on:
  # schedule:
  #   - cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China
  # please uncomment it for PR tests
  # pull_request:
  #   branches: [main]
  #   paths:
  #     - ".github/workflows/llm_performance_tests.yml"
  #     - "python/llm/test/benchmark/**"
  #     - "python/llm/dev/benchmark/all-in-one/**"
  # workflow_dispatch:
  #   inputs:
  #     arc:
  #       description: "If trigger performance test on Arc"
  #       required: false
  #       type: boolean
  #       default: true
  #     spr:
  #       description: "If trigger performance test on SPR"
  #       required: false
  #       type: boolean
  #       default: true
  #     core:
  #       description: "If trigger performance test on Core"
  #       required: false
  #       type: boolean
  #       default: true
  #     igpu:
  #       description: "If trigger performance test on iGPU"
  #       required: false
  #       type: boolean
  #       default: true
  workflow_call:
    inputs:
      checkout-ref:
        description: 'ref for checking out, including branch, tag or SHA'
        required: true
        type: string
      arc:
        description: "If trigger performance test on Arc"
        required: false
        type: boolean
        default: true
      spr:
        description: "If trigger performance test on SPR"
        required: false
        type: boolean
        default: true
      core:
        description: "If trigger performance test on Core"
        required: false
        type: boolean
        default: true
      igpu:
        description: "If trigger performance test on iGPU"
        required: false
        type: boolean
        default: true

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  llm-cpp-build:
    uses: ./.github/workflows/llm-binary-build.yml
    with:
      platform: ${{ (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main')) && 'Windows,Linux' || 'Dummy' }}

  llm-performance-test-on-arc:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.arc ) }} # please comment it for PR tests
    needs: llm-cpp-build
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
    runs-on: [self-hosted, llm, perf]
    env:
      OMP_NUM_THREADS: 16
      THREAD_NUM: 16
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
      CSV_SAVE_PATH: ${{ (github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main'))) && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }}

    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: 'intel-analytics/ipex-llm'
          ref: ${{ inputs.checkout-ref }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        # pip install transformers_stream_generator for model internlm-chat-7b-8k
        # pip install tiktoken for model Qwen-7B-Chat-10-12
        # pip install matplotlib for model Qwen-VL-Chat
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade wheel
          python -m pip install --upgrade omegaconf
          python -m pip install --upgrade pandas
          python -m pip install --upgrade einops
          python -m pip install --upgrade transformers_stream_generator
          python -m pip install --upgrade tiktoken
          python -m pip install --upgrade matplotlib

      # specific for test on certain commits
      - name: Download llm binary
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/download-llm-binary

      - name: Install IPEX-LLM from source
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/setup-llm-env
        with:
          extra-dependency: "xpu_2.1"

      - name: Install IPEX-LLM from Pypi
        if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
        shell: bash
        run: |
          pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          if ! pip show ipex-llm | grep $test_version_date; then
            echo "Did not install ipex-llm with excepted version $test_version_date"
            exit 1
          fi

      - name: Test installed xpu version
        shell: bash
        run: |
          source /opt/intel/oneapi/setvars.sh
          python -m pip install --upgrade pytest
          python -m pip install --upgrade expecttest
          bash python/llm/test/run-llm-install-tests.sh

      - name: Test on xpu(transformers==4.36.2)
        shell: bash
        run: |
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
          source /opt/intel/oneapi/setvars.sh
          export USE_XETLA=OFF
          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
          cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          mkdir test_batch1
          mkdir test_batch2
          mkdir test_batch4
          # batch_size 1
          # hide time info
          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
          # change csv name
          sed -i 's/{today}/{today}_test1_batch1/g' run.py
          python run.py
          mv *.csv test_batch1
          # batch_size 2
          cd ../../../../../ 
          cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/batch1/batch2/g' run.py
          python run.py
          mv *.csv test_batch2
          # batch_size 4
          cd ../../../../../ 
          cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/batch2/batch4/g' run.py
          python run.py
          mv *.csv test_batch4

      - name: Test on xpu(transformers==4.37.0)
        shell: bash
        run: |
          source /opt/intel/oneapi/setvars.sh
          export USE_XETLA=OFF
          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
          # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
          python -m pip install transformers==4.37.0
          # batch_size 1
          cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/test1_batch4/test2_batch1/g' run.py
          python run.py
          mv *.csv test_batch1
          # batch_size 2
          cd ../../../../../ 
          cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name 
          sed -i 's/batch1/batch2/g' run.py
          python run.py
          mv *.csv test_batch2
          # batch_size 4
          cd ../../../../../ 
          cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name 
          sed -i 's/batch2/batch4/g' run.py
          python run.py
          mv *.csv test_batch4
      
      - name: Test on xpu(transformers==4.40.0)
        shell: bash
        run: |
          source /opt/intel/oneapi/setvars.sh
          export USE_XETLA=OFF
          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
          # upgrade transformers for model Qwen/Qwen1.5-MoE-A2.7B-Chat
          python -m pip install transformers==4.40.0
          python -m pip install trl
          # batch_size 1
          cp python/llm/test/benchmark/arc-perf-transformers-440.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/test2_batch4/test3_batch1/g' run.py
          python run.py
          mv *.csv test_batch1

      - name: Concat csv and generate html
        shell: bash
        run: |
          # batch_size 1 
          cd python/llm/dev/benchmark/all-in-one/test_batch1
          python ../../../../test/benchmark/concat_csv.py
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" $CSV_SAVE_PATH/batch_size_1
              fi
          done
          python -m pip install pandas==1.5.3
          cd ../../../../test/benchmark
          python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1
          # batch_size 2
          cd ../../../../
          cd python/llm/dev/benchmark/all-in-one/test_batch2
          python ../../../../test/benchmark/concat_csv.py
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" $CSV_SAVE_PATH/batch_size_2
              fi
          done
          cd ../../../../test/benchmark
          python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2
          # batch_size 4
          cd ../../../../
          cd python/llm/dev/benchmark/all-in-one/test_batch4
          python ../../../../test/benchmark/concat_csv.py
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" $CSV_SAVE_PATH/batch_size_4
              fi
          done
          cd ../../../../test/benchmark
          python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_4
      
      - name: Merge and sort csv files of multiple batches and generate html 
        shell: bash
        run: |
          cd python/llm/test/benchmark
          mkdir merged_temp
          # go through all the files and go to merged_temp
          cd ../../dev/benchmark/all-in-one/test_batch1
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" ../../../../test/benchmark/merged_temp
              fi
          done
          cd ../test_batch2
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" ../../../../test/benchmark/merged_temp
              fi
          done
          cd ../test_batch4
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" ../../../../test/benchmark/merged_temp
              fi
          done
          cd ../../../../test/benchmark
          python merge_csv_batch.py -f ./merged_temp
          cd merged_temp
          find . -name "*batch*.csv" -delete
          for file in *.csv; do
              cp "$file" $CSV_SAVE_PATH/merged
          done
          cd ..
          python csv_to_html.py -f $CSV_SAVE_PATH/merged
          rm -r merged_temp
      
      - name: Update html in parent folder
        shell: bash
        run: |
          cd python/llm/test/benchmark
          python update_html_in_parent_folder.py -f $CSV_SAVE_PATH

      - name: Check and upload results to ftp
        shell: bash
        run: |
          # batch_size 1
          cd python/llm/dev/benchmark/all-in-one/test_batch1
          python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml
          python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml
          python ../../../../test/benchmark/check_results.py -c test3 -y ../../../../test/benchmark/arc-perf-transformers-440.yaml
          find . -name "*test*.csv" -delete
          if [[ ${{ github.event_name }} == "schedule" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          fi
          cd ../
          rm -r test_batch1
          # batch_size 2
          cd test_batch2
          python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml
          python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml
          find . -name "*test*.csv" -delete
          if [[ ${{ github.event_name }} == "schedule" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          fi
          cd ../
          rm -r test_batch2
          # batch_size 4
          cd test_batch4
          python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch4.yaml
          python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml
          find . -name "*test*.csv" -delete
          if [[ ${{ github.event_name }} == "schedule" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          fi
          cd ../
          rm -r test_batch4
          
          
  llm-performance-test-on-spr:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.spr ) }} # please comment it for PR tests
    needs: llm-cpp-build
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
    runs-on: [self-hosted, llm, spr-perf]
    env:
      OMP_NUM_THREADS: 16
      THREAD_NUM: 16
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: 'intel-analytics/ipex-llm'
          ref: ${{ inputs.checkout-ref }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade wheel
          python -m pip install --upgrade omegaconf
          python -m pip install --upgrade pandas
          python -m pip install --upgrade einops
          python -m pip install --upgrade tiktoken
          python -m pip install --upgrade transformers_stream_generator

      # specific for test on certain commits
      - name: Download llm binary
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/download-llm-binary

      - name: Install IPEX-LLM from source
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/setup-llm-env

      - name: Install IPEX-LLM from Pypi
        if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
        shell: bash
        run: |
          pip install --pre --upgrade ipex-llm[all] --extra-index-url https://download.pytorch.org/whl/cpu
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          if ! pip show ipex-llm | grep $test_version_date; then
            echo "Did not install ipex-llm with excepted version $test_version_date"
            exit 1
          fi

      - name: Test on cpu
        shell: bash
        run: |
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

          mv python/llm/test/benchmark/cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          source ipex-llm-init -t
          export OMP_NUM_THREADS=48
          # hide time info
          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
          python run.py
          cp ./*.csv /mnt/disk1/models/nightly_perf_cpu
          cd ../../../test/benchmark
          python -m pip install pandas==1.5.3
          python csv_to_html.py -f /mnt/disk1/models/nightly_perf_cpu
          cd /mnt/disk1/models/nightly_perf_cpu
          for f in *.html; do
            curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
          done

  llm-performance-test-on-core:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.core ) }} # please comment it for PR tests
    needs: llm-cpp-build
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows
            platform: dp
            python-version: "3.11"
          # - os: windows
          #   platform: lp
          #   python-version: "3.11"
    runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"]
    env:
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
      CSV_SAVE_PATH: ${{ (github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main'))) && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: 'intel-analytics/ipex-llm'
          ref: ${{ inputs.checkout-ref }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade wheel
          python -m pip install --upgrade omegaconf pandas
          python -m pip install --upgrade tiktoken einops transformers_stream_generator
    
      # specific for test on certain commits
      - name: Download llm binary
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/download-llm-binary

      - name: Install IPEX-LLM from source
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/setup-llm-env

      - name: Install IPEX-LLM from Pypi
        if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
        shell: bash
        run: |
          pip install --pre --upgrade ipex-llm[all]
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          if ! pip show ipex-llm | grep $test_version_date; then
            echo "Did not install ipex-llm with excepted version $test_version_date"
            exit 1
          fi


      - name: Test on core ${{ matrix.platform }}
        shell: bash
        run: |
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

          mv python/llm/test/benchmark/core-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          # hide time info
          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
          python run.py
          cp ./*.csv $CSV_SAVE_PATH
          cd ../../../test/benchmark
          python -m pip install pandas==1.5.3
          python csv_to_html.py -f $CSV_SAVE_PATH
          cd ../../dev/benchmark/all-in-one/
          if [[ ${{ github.event_name }} == "schedule" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
          elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
          fi

  llm-performance-test-on-igpu:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.igpu ) }} # please comment it for PR tests
    needs: llm-cpp-build
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows
            python-version: "3.11"
    runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-igpu]
    env:
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: 'intel-analytics/ipex-llm'
          ref: ${{ inputs.checkout-ref }}

      # TODO: Put the ipex-llm related install process for win gpu into a action function

      # specific for test on certain commits
      - name: Download llm binary
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        uses: ./.github/actions/llm/download-llm-binary

      - name: Prepare for install ipex-llm from source
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        shell: bash
        run: |
          sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
          sed -i 's/"bigdl-core-xe-batch-21==" + CORE_XE_VERSION/"bigdl-core-xe-batch-21"/g' python/llm/setup.py
          sed -i 's/"bigdl-core-xe-addons-21==" + CORE_XE_VERSION/"bigdl-core-xe-addons-21"/g' python/llm/setup.py
          sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION/"bigdl-core-xe-esimd-21"/g' python/llm/setup.py

      - name: Install ipex-llm and other related packages (install from source)
        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.checkout-ref != 'main') }}
        shell: cmd
        run: |
          call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
          call conda activate igpu-perf

          pip install --upgrade pip
          pip install --upgrade wheel
          pip install --upgrade omegaconf pandas
          pip install --upgrade tiktoken einops transformers_stream_generator matplotlib

          cd python\llm
          python setup.py clean --all bdist_wheel --win
          if not exist dist\ipex_llm*.whl (exit /b 1)
          for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i

          pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
          if %ERRORLEVEL% neq 0 (exit /b 1)
          pip list

          call conda deactivate

      - name: Determine desired ipex-llm version
        if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
        shell: bash
        run: |
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV"

      - name: Install ipex-llm and other related packages (install from pypi)
        if: ${{ github.event.schedule || (github.event_name == 'workflow_dispatch' && (inputs.checkout-ref == 'main')) }}
        shell: cmd
        run: |
          call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
          call conda activate igpu-perf

          pip install --upgrade pip
          pip install --upgrade wheel
          pip install --upgrade omegaconf pandas
          pip install --upgrade tiktoken einops transformers_stream_generator matplotlib

          pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
          pip show ipex-llm | findstr %TEST_VERSION_DATE%
          if %ERRORLEVEL% neq 0 (
            echo "Did not install ipex-llm with excepted version %TEST_VERSION_DATE%"
            exit /b 1
          )
          pip list

          call conda deactivate

      - name: Create env for html generation
        shell: cmd
        run: |
          call conda create -n html-gen python=3.11 -y
          call conda activate html-gen

          pip install pandas==1.5.3
          pip install Jinja2
          pip install "numpy<2.0.0"

          call conda deactivate

      - name: Set directory envs & and fix generated csv date name
        shell: bash
        run: |
          if [[ ${{ github.event_name }} == "schedule" ]]; then
            echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
          elif [[ ${{ github.event_name }} == "workflow_dispatch" ]] && [[ ${{ inputs.checkout-ref }} == "main" ]]; then
            echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
          else
            echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV"
          fi
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          echo "LOG_FILE=${date_for_test_version}_output.txt" >> "$GITHUB_ENV"

          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

      - name: Add extra warmup for chatglm3-6b int4+fp32 & MiniCPM int4+fp16 int4+fp32 for more stable results
        shell: bash
        run: |
          sed -i '/^\s*result = run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
                  if repo_id in ["THUDM/chatglm3-6b", "THUDM/glm-4-9b-chat", "openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
                      run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
          ' python/llm/dev/benchmark/all-in-one/run.py
          
          sed -i '/^\s*result = run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
                  if repo_id in ["openbmb/MiniCPM-1B-sft-bf16", "openbmb/MiniCPM-2B-sft-bf16"]:\
                      run_transformer_int4_fp16_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
          ' python/llm/dev/benchmark/all-in-one/run.py

      # 32-32 int4+fp16
      - name: Prepare igpu perf test (32-32 int4+fp16)
        shell: bash
        run: |
          # hide time info
          # sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml

      - name: Test on igpu (32-32 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (32-32 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_437.yaml

      - name: Test on igpu for transformers 4.37 (32-32 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate
          
      - name: Prepare igpu perf test for transformers 4.38 (32-32 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_438.yaml

      - name: Test on igpu for transformers 4.38 (32-32 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.43 (32-32 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_int4_fp16_443.yaml

      - name: Test on igpu for transformers 4.43 (32-32 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.43.1
          pip install trl

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_443.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
          if %ERRORLEVEL% neq 0 (exit /b 1)

          pip uninstall trl -y
          call conda deactivate

      - name: Concat csv and generate html (32-32 int4+fp16)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\32-32_int4_fp16\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\32-32_int4_fp16\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\32-32_int4_fp16\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # TODO: create a action function here for different input
      # 1024-128 int4+fp16
      - name: Prepare igpu perf test (1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml

      - name: Test on igpu (1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml

      - name: Test on igpu for transformers 4.37 (1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate
          
      - name: Prepare igpu perf test for transformers 4.38 (1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_438.yaml

      - name: Test on igpu for transformers 4.38 (1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.43 (1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_443.yaml

      - name: Test on igpu for transformers 4.43 (1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.43.1
          pip install trl

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_443.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
          if %ERRORLEVEL% neq 0 (exit /b 1)

          pip uninstall trl -y
          call conda deactivate

      - name: Concat csv and generate html (1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # 2048-256 int4+fp16
      - name: Prepare igpu perf test (2048-256 int4+fp16)
        shell: bash
        run: |
          sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml

      - name: Test on igpu (2048-256 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (2048-256 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_437.yaml

      - name: Test on igpu for transformers 4.37 (2048-256 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate
          
      - name: Prepare igpu perf test for transformers 4.38 (2048-256 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_438.yaml

      - name: Test on igpu for transformers 4.38 (2048-256 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.43 (2048-256 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16_443.yaml

      - name: Test on igpu for transformers 4.43 (2048-256 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.43.1
          pip install trl

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_443.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
          if %ERRORLEVEL% neq 0 (exit /b 1)

          pip uninstall trl -y
          call conda deactivate

      - name: Concat csv and generate html (2048-256 int4+fp16)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\2048-256_int4_fp16\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256_int4_fp16\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\2048-256_int4_fp16\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # 3072-384 int4+fp16
      - name: Prepare igpu perf test (3072-384 int4+fp16)
        shell: bash
        run: |
          sed -i 's/2048-256/3072-384/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16.yaml
      
      - name: Test on igpu (3072-384 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (3072-384 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_437.yaml

      - name: Test on igpu for transformers 4.37 (3072-384 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.38 (3072-384 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_438.yaml

      - name: Test on igpu for transformers 4.38 (3072-384 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.43 (3072-384 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/3072-384_int4_fp16_443.yaml

      - name: Test on igpu for transformers 4.43 (3072-384 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.43.1
          pip install trl

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_443.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
          if %ERRORLEVEL% neq 0 (exit /b 1)

          pip uninstall trl -y
          call conda deactivate

      - name: Concat csv and generate html (3072-384 int4+fp16)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\3072-384_int4_fp16\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\3072-384_int4_fp16\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\3072-384_int4_fp16\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # 4096-512 int4+fp16
      - name: Prepare igpu perf test (4096-512 int4+fp16)
        shell: bash
        run: |
          sed -i 's/3072-384/4096-512/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16.yaml
      
      - name: Test on igpu (4096-512 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (4096-512 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_437.yaml

      - name: Test on igpu for transformers 4.37 (4096-512 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.38 (4096-512 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_438.yaml

      - name: Test on igpu for transformers 4.38 (4096-512 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      # NOTE: Gemma2 not working for 4096-512. 
      #       When it works, uncomment this section and remember to change "'s/{today}_test3/{today}_test1/g'" in next section.
      
      #- name: Prepare igpu perf test for transformers 4.43 (4096-512 int4+fp16)
      #  shell: bash
      #  run: |
      #    sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
      #    sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/4096-512_int4_fp16_443.yaml

      #- name: Test on igpu for transformers 4.43 (4096-512 int4+fp16)
      #  shell: cmd
      #  run: |
      #    call conda activate igpu-perf
      #    pip install transformers==4.43.1
      #    pip install trl
      #
      #    set SYCL_CACHE_PERSISTENT=1
      #    set BIGDL_LLM_XMX_DISABLED=1
      #
      #    cd python\llm\dev\benchmark\all-in-one
      #    move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_443.yaml config.yaml
      #    set PYTHONIOENCODING=utf-8
      #    python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
      #    if %ERRORLEVEL% neq 0 (exit /b 1)
      #    python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
      #    if %ERRORLEVEL% neq 0 (exit /b 1)
      #    
      #    pip uninstall trl -y
      #    call conda deactivate

      - name: Concat csv and generate html (4096-512 int4+fp16)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\4096-512_int4_fp16\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\4096-512_int4_fp16\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\4096-512_int4_fp16\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # load_low_bit 1024-128 int4+fp16 
      - name: Prepare igpu perf test (load_low_bit 1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/4096-512/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml

      - name: Test on igpu (load_low_bit 1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_437.yaml

      - name: Test on igpu for transformers 4.37 (load_low_bit 1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.38 (load_low_bit 1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_438.yaml

      - name: Test on igpu for transformers 4.38 (load_low_bit 1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.43 (load_low_bit 1024-128 int4+fp16)
        shell: bash
        run: |
          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit_443.yaml

      - name: Test on igpu for transformers 4.43 (load_low_bit 1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.43.1
          pip install trl

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_443.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
          if %ERRORLEVEL% neq 0 (exit /b 1)

          pip uninstall trl -y
          call conda deactivate

      - name: Concat csv and generate html (load_low_bit 1024-128 int4+fp16)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # 1024-128
      - name: Prepare igpu perf test (1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test4/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml

      - name: Test on igpu (1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml

      - name: Test on igpu for transformers 4.37 (1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.38 (1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_438.yaml

      - name: Test on igpu for transformers 4.38 (1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.38.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.43 (1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test3/{today}_test4/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_443.yaml

      - name: Test on igpu for transformers 4.43 (1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.43.1
          pip install trl

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_443.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)
          python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
          if %ERRORLEVEL% neq 0 (exit /b 1)

          pip uninstall trl -y
          call conda deactivate

      - name: Concat csv and generate html (1024-128)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\1024-128\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # TODO: avoid duplicated code
      - name: Upload results to ftp
        if: ${{ always() }}
        shell: cmd
        run: |
          cd %CSV_SAVE_PATH%
          IF "${{ github.event_name }}"=="schedule" (
            for %%f in (*.html) do (
                curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
            )
          )
          IF "${{ github.event_name }}"=="workflow_dispatch" (
            IF "${{ inputs.checkout-ref }}"=="main" (
              for %%f in (*.html) do (
                  curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
              )
            )
          )
      # for test on machine when encountering error
      # - name: Remove conda env
      #   if: ${{ always() }}
      #   shell: cmd
      #   run: |
      #     call conda env remove -n igpu-perf -y