Merge branch 'intel-analytics:main' into llm-ppl-workflow

.github/actions/llm/download-llm-binary/action.yml

@@ -1,6 +1,11 @@
 name: Download LLM binary files
 description: Download built binary files from github artifact
-
+inputs:
+  platform:
+    description: 'Platforms to built on'
+    default: 'Windows,Linux'
+    required: false
+    type: string
 runs:
   using: "composite"
   steps:
@@ -11,14 +16,18 @@ runs:
       run: |
         rm -rf python/llm/llm-binary || true
         mkdir -p python/llm/llm-binary
-        mv linux-avx2/* python/llm/llm-binary/
-        mv linux-avx512/* python/llm/llm-binary/
-        mv linux-avxvnni/* python/llm/llm-binary/
-        mv linux-avx/* python/llm/llm-binary/
-        mv linux-amx/* python/llm/llm-binary/
-        mv windows-avx2/* python/llm/llm-binary/
-        mv windows-avx-vnni/* python/llm/llm-binary/
-        mv windows-avx/* python/llm/llm-binary/
+        if ${{contains(inputs.platform, 'Linux')}}; then
+          mv linux-avx2/* python/llm/llm-binary/ 
+          mv linux-avx512/* python/llm/llm-binary/
+          mv linux-avxvnni/* python/llm/llm-binary/
+          mv linux-avx/* python/llm/llm-binary/
+          mv linux-amx/* python/llm/llm-binary/
+        fi
+        if ${{contains(inputs.platform, 'Windows')}}; then
+          mv windows-avx2/* python/llm/llm-binary/
+          mv windows-avx-vnni/* python/llm/llm-binary/
+          mv windows-avx/* python/llm/llm-binary/
+        fi
         rm -rf linux-avx2 || true
         rm -rf linux-avx512 || true
         rm -rf linux-avxvnni || true

.github/workflows/llm-binary-build.yml

@@ -26,17 +26,28 @@ on:
         default: ''
         required: false
         type: string
+      platform:
+        description: 'Platforms to built on'
+        default: '["Windows", "Linux"]'
+        required: false
+        type: string
   workflow_call:
     inputs:
       llmcpp-ref:
         description: 'Ref of llm.cpp code'
         default: ''
         required: false
         type: string
-
+      platform:
+        description: 'Platforms to built on'
+        default: 'Windows,Linux'
+        required: false
+        type: string
+
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   check-linux-avxvnni-artifact:
+    if: ${{contains(inputs.platform, 'Linux')}}
     runs-on: ubuntu-latest
     outputs:
       if-exists: ${{steps.check_artifact.outputs.exists}}
@@ -149,6 +160,7 @@ jobs:
           conda remove -n python311 --all -y
 
   check-linux-avx512-artifact:
+    if: ${{contains(inputs.platform, 'Linux')}}
     runs-on: ubuntu-latest
     outputs:
       if-exists: ${{steps.check_artifact.outputs.exists}}
@@ -249,6 +261,7 @@ jobs:
           conda remove -n python39 --all -y
 
   check-linux-amx-artifact:
+    if: ${{contains(inputs.platform, 'Linux')}}
     runs-on: ubuntu-latest
     outputs:
       if-exists: ${{steps.check_artifact.outputs.exists}}
@@ -319,6 +332,7 @@ jobs:
           conda remove -n python39 --all -y
           
   check-windows-avx2-artifact:
+    if: ${{contains(inputs.platform, 'Windows')}}
     runs-on: ubuntu-latest
     outputs:
       if-exists: ${{steps.check_artifact.outputs.exists}}
@@ -363,6 +377,7 @@ jobs:
             build/Release
 
   check-windows-avx-vnni-artifact:
+    if: ${{contains(inputs.platform, 'Windows')}}
     runs-on: ubuntu-latest
     outputs:
       if-exists: ${{steps.check_artifact.outputs.exists}}
@@ -472,6 +487,7 @@ jobs:
             release
 
   check-windows-avx-artifact:
+    if: ${{contains(inputs.platform, 'Windows')}}
     runs-on: ubuntu-latest
     outputs:
       if-exists: ${{steps.check_artifact.outputs.exists}}

.github/workflows/llm-c-evaluation.yml

@@ -34,6 +34,8 @@ on:
 jobs:
   llm-cpp-build:
     uses: ./.github/workflows/llm-binary-build.yml
+    with:
+      platform: 'Linux'
   # Set the testing matrix based on the event (schedule, PR, or manual dispatch)
   set-matrix:
     runs-on: ubuntu-latest
@@ -110,6 +112,8 @@ jobs:
 
       - name: Download llm binary
         uses: ./.github/actions/llm/download-llm-binary
+        with:
+          platform: 'Linux'
 
       - name: Run LLM install (all) test
         uses: ./.github/actions/llm/setup-llm-env

.github/workflows/llm-harness-evaluation.yml

@@ -42,6 +42,8 @@ on:
 jobs:
   llm-cpp-build:
     uses: ./.github/workflows/llm-binary-build.yml
+    with:
+      platform: 'Linux'
   # Set the testing matrix based on the event (schedule, PR, or manual dispatch)
   set-matrix:
     runs-on: ubuntu-latest
@@ -68,9 +70,9 @@ jobs:
       - name: set-pr-env
         if: ${{github.event_name == 'pull_request'}}
         env:
-          PR_MATRIX_MODEL_NAME: '["Mistral-7B-v0.1"]'
-          PR_MATRIX_TASK: '["arc", "truthfulqa", "winogrande"]'
-          PR_MATRIX_PRECISION: '["fp8"]'
+          PR_MATRIX_MODEL_NAME: '["stablelm-3b-4e1t"]'
+          PR_MATRIX_TASK: '["winogrande"]'
+          PR_MATRIX_PRECISION: '["sym_int4"]'
           PR_LABELS: '["self-hosted", "llm", "temp-arc01"]'
 
         run: |
@@ -110,8 +112,6 @@ jobs:
         device: [xpu]
 
     runs-on: ${{ fromJson(needs.set-matrix.outputs.runner) }}
-    outputs:
-      output_path: ${{ steps.run_harness.outputs.output_path }}
     env:
       ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
       ORIGIN_DIR: /mnt/disk1/models
@@ -132,6 +132,8 @@ jobs:
 
       - name: Download llm binary
         uses: ./.github/actions/llm/download-llm-binary
+        with:
+          platform: 'Linux'
 
       - name: Run LLM install (all) test
         uses: ./.github/actions/llm/setup-llm-env
@@ -142,7 +144,10 @@ jobs:
         working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness/
         shell: bash
         run: |
-          pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b09
+          git clone https://github.com/EleutherAI/lm-evaluation-harness.git
+          cd lm-evaluation-harness
+          git checkout b281b09
+          pip install -e .
       
       - name: Download models and datasets
         shell: bash
@@ -160,35 +165,13 @@ jobs:
         run: |
           pip install --upgrade datasets==2.14.6 
           if [ "${{ matrix.model_name }}" = "Mistral-7B-v0.1" ]; then
-          pip install --upgrade transformers==4.36
+            pip install --upgrade transformers==4.36
           else
-          pip install --upgrade transformers==4.31
+            pip install --upgrade transformers==4.31
           fi
       
 
-      - name: Run harness nightly
-        if: ${{github.event_name == 'schedule'}}
-        shell: bash
-        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness
-        env:
-          USE_XETLA: OFF
-          # SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: 1
-        run: |
-          export HF_HOME=${HARNESS_HF_HOME}
-          export HF_DATASETS=$HARNESS_HF_HOME/datasets
-          export HF_DATASETS_CACHE=$HARNESS_HF_HOME/datasets
-          source /opt/intel/oneapi/setvars.sh
-          
-          python run_llb.py \
-            --model bigdl-llm \
-            --pretrained ${MODEL_PATH} \
-            --precision ${{ matrix.precision }} \
-            --device ${{ matrix.device }} \
-            --tasks ${{ matrix.task }} \
-            --batch_size 1 --no_cache --output_path results \
-      
-      - name: Run harness pr
-        if: ${{github.event_name == 'pull_request'}}
+      - name: Run harness
         shell: bash
         working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness
         env:
@@ -200,14 +183,19 @@ jobs:
           export HF_DATASETS_CACHE=$HARNESS_HF_HOME/datasets
           source /opt/intel/oneapi/setvars.sh
           
+          # set --limit if it's pr-triggered to accelerate pr action
+          if ${{github.event_name == 'pull_request'}}; then
+            export LIMIT="--limit 4"
+          fi
+
           python run_llb.py \
             --model bigdl-llm \
             --pretrained ${MODEL_PATH} \
             --precision ${{ matrix.precision }} \
             --device ${{ matrix.device }} \
             --tasks ${{ matrix.task }} \
             --batch_size 1 --no_cache --output_path results \
-            --limit 3 \
+            $LIMIT
 
       - uses: actions/upload-artifact@v3
         with:
@@ -246,12 +234,12 @@ jobs:
         shell: bash
         run: |
           ls results
-          python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_and_csv.py results
+          python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table.py results
 
   # TODO: change machine to store the results later        
-  llm-harness-summary-html:
+  llm-harness-html:
     if: ${{github.event_name == 'schedule' || github.event_name == 'pull_request'}}
-    needs: [set-matrix, llm-harness-evaluation]
+    needs: [llm-harness-evaluation]
     runs-on: ["self-hosted", "llm", "accuracy1", "accuracy-nightly"]
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
@@ -264,54 +252,45 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install jsonlines  pytablewriter regex
+          pip install pandas==1.5.3
       
       - name: Set output path
         shell: bash
         run: |    
-          DATE=$(date +%Y-%m-%d)
-          OUTPUT_PATH="results_$DATE"
-          echo "OUTPUT_PATH=$OUTPUT_PATH" >> $GITHUB_ENV
-          NIGHTLY_FOLDER="/home/arda/harness-action-runners/nightly-accuracy-data"
-          echo "NIGHTLY_FOLDER=$NIGHTLY_FOLDER" >> $GITHUB_ENV
-          PR_FOLDER="/home/arda/harness-action-runners/pr-accuracy-data"
-          echo "PR_FOLDER=$PR_FOLDER" >> $GITHUB_ENV
+          echo "DATE=$(date +%Y-%m-%d)" >> $GITHUB_ENV
+          if ${{github.event_name == 'pull_request'}}; then
+            echo 'ACC_FOLDER=/home/arda/harness-action-runners/pr-accuracy-data' >> $GITHUB_ENV
+          fi
+          if ${{github.event_name == 'schedule'}}; then
+            echo 'ACC_FOLDER=/home/arda/harness-action-runners/nightly-accuracy-data' >> $GITHUB_ENV
+          fi
 
-      - name: Download all results for nightly run
-        if: github.event_name == 'schedule'
-        uses: actions/download-artifact@v3
-        with:
-          name: harness_results
-          path: ${{ env.NIGHTLY_FOLDER}}/${{ env.OUTPUT_PATH }}
-
-      - name: Download all results for pr run
-        if: github.event_name == 'pull_request'
+      - name: Download harness results
         uses: actions/download-artifact@v3
         with:
           name: harness_results
-          path: ${{ env.PR_FOLDER}}/${{ env.OUTPUT_PATH }}
+          path: ${{ env.ACC_FOLDER}}/${{ env.DATE }}
+
 
       # Save fp16.csv in the parent folder of env.nightly_folder
-      - name: Download fp16.csv for summary
+      - name: Download FP16 results
         shell: bash
         run: |
-          wget https://raw.githubusercontent.com/intel-analytics/BigDL/main/python/llm/test/benchmark/harness/fp16.csv -O ${{ env.NIGHTLY_FOLDER}}/../fp16.csv
-          ls ${{ env.NIGHTLY_FOLDER}}/..
+          wget https://raw.githubusercontent.com/intel-analytics/BigDL/main/python/llm/test/benchmark/harness/fp16.csv -O $ACC_FOLDER/../fp16.csv
+          ls $ACC_FOLDER/..
 
-      - name: Summarize the results for nightly run
-        if: github.event_name == 'schedule'
+      - name: Write to CSV
+        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness
         shell: bash
         run: |
-          ls /home/arda/harness-action-runners/nightly-accuracy-data/${{ env.OUTPUT_PATH }}
-          pip install pandas==1.5.3
-          python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_and_csv.py ${{ env.NIGHTLY_FOLDER}}/${{ env.OUTPUT_PATH }} ${{ env.NIGHTLY_FOLDER}}
-          python ${{ github.workspace }}/python/llm/test/benchmark/harness/harness_csv_to_html.py -f ${{ env.NIGHTLY_FOLDER}}
-          python ${{ github.workspace }}/python/llm/test/benchmark/harness/update_html_in_parent_folder.py -f ${{ env.NIGHTLY_FOLDER }}
+          ls $ACC_FOLDER/$DATE
+          python make_csv.py $ACC_FOLDER/$DATE $ACC_FOLDER
 
-      - name: Summarize the results for pull request
-        if: github.event_name == 'pull_request'
+      - name: Update HTML
+        working-directory: ${{ github.workspace }}/python/llm/test/benchmark/harness
         shell: bash
         run: |
-          ls /home/arda/harness-action-runners/pr-accuracy-data/${{ env.OUTPUT_PATH }}
-          pip install pandas==1.5.3
-          python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table_and_csv.py ${{ env.PR_FOLDER}}/${{ env.OUTPUT_PATH }} ${{ env.PR_FOLDER}}
-          python ${{ github.workspace }}/python/llm/test/benchmark/harness/harness_csv_to_html.py -f ${{ env.PR_FOLDER}}
+          python harness_csv_to_html.py -f $ACC_FOLDER
+          if ${{github.event_name == 'schedule'}}; then
+            python update_html_in_parent_folder.py -f $ACC_FOLDER
+          fi

.github/workflows/llm-ppl-evaluation.yml

@@ -42,6 +42,8 @@ on:
 jobs:
   llm-cpp-build:
     uses: ./.github/workflows/llm-binary-build.yml
+    with:
+      platform: 'Linux'
   set-matrix:
     runs-on: ubuntu-latest
     outputs:
@@ -128,6 +130,8 @@ jobs:
 
       - name: Download llm binary
         uses: ./.github/actions/llm/download-llm-binary
+        with:
+          platform: 'Linux'
 
       - name: Run LLM install (all) test
         uses: ./.github/actions/llm/setup-llm-env

.github/workflows/llm_tests_for_stable_version_on_arc.yml

@@ -23,7 +23,8 @@ on:
 jobs:
   llm-cpp-build:
     uses: ./.github/workflows/llm-binary-build.yml
-
+    with:
+      platform: 'Linux'
   llm-perf-regression-test-on-arc:
     needs: llm-cpp-build
     strategy:
@@ -60,6 +61,8 @@ jobs:
 
       - name: Download llm binary
         uses: ./.github/actions/llm/download-llm-binary
+        with:
+          platform: 'Linux'
 
       - name: Run LLM install (all) test
         uses: ./.github/actions/llm/setup-llm-env

python/llm/dev/benchmark/harness/README.md

@@ -5,7 +5,10 @@ Before running, make sure to have [bigdl-llm](../../../README.md) installed.
 
 ## Install Harness
 ```bash
-pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b09
+git clone https://github.com/EleutherAI/lm-evaluation-harness.git
+cd lm-evaluation-harness
+git checkout b281b09
+pip install -e .
 ```
 
 ## Run
@@ -26,3 +29,7 @@ python run_multi_llb.py --model bigdl-llm --pretrained /path/to/model --precisio
 Taking example above, the script will fork 3 processes, each for one xpu, to execute the tasks.
 ## Results
 We follow [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) to record our metrics, `acc_norm` for `hellaswag` and `arc_challenge`, `mc2` for `truthful_qa` and `acc` for `mmlu`. For `mmlu`, there are 57 subtasks which means users may need to average them manually to get final result.
+## Summarize the results
+"""python
+python make_table.py <input_dir>
+"""