diff --git a/.github/actions/llm/setup-llm-env/action.yml b/.github/actions/llm/setup-llm-env/action.yml
index 868cf625bc00..ba8bb934d23b 100644
--- a/.github/actions/llm/setup-llm-env/action.yml
+++ b/.github/actions/llm/setup-llm-env/action.yml
@@ -13,10 +13,10 @@ runs:
       run: |
         # make sure we install the latest version for bigdl-core-xe
         pip uninstall bigdl-core-xe || true
-        sed -i 's/"bigdl-core-xe==" + VERSION + "/"bigdl-core-xe/g' python/llm/setup.py
+        sed -i 's/"bigdl-core-xe==" + VERSION + "/"bigdl-core-xe==2.5.0b20231111/g' python/llm/setup.py
         # make sure we install the latest version for bigdl-core-xe-esimd
         pip uninstall bigdl-core-xe-esimd || true
-        sed -i 's/"bigdl-core-xe-esimd==" + VERSION + "/"bigdl-core-xe-esimd/g' python/llm/setup.py
+        sed -i 's/"bigdl-core-xe-esimd==" + VERSION + "/"bigdl-core-xe-esimd==2.5.0b20231111/g' python/llm/setup.py
 
         pip install requests
         if [[ ${{ runner.os }} == 'Linux' ]]; then
diff --git a/.github/workflows/llm-binary-build.yml b/.github/workflows/llm-binary-build.yml
index fee8f6d8d054..33f5db1a4013 100644
--- a/.github/workflows/llm-binary-build.yml
+++ b/.github/workflows/llm-binary-build.yml
@@ -20,14 +20,14 @@ on:
     inputs:
       llmcpp-ref:
         description: 'Ref of llm.cpp code'
-        default: ''
+        default: 'cf9cba2b67e8b14841e690f85dd6c272cafb0822'
         required: false
         type: string
   workflow_call:
     inputs:
       llmcpp-ref:
         description: 'Ref of llm.cpp code'
-        default: ''
+        default: 'cf9cba2b67e8b14841e690f85dd6c272cafb0822'
         required: false
         type: string
 
diff --git a/.github/workflows/llm-nightly-test.yml b/.github/workflows/llm-nightly-test.yml
index a5a136b1c2c1..78956c70bb9f 100644
--- a/.github/workflows/llm-nightly-test.yml
+++ b/.github/workflows/llm-nightly-test.yml
@@ -18,6 +18,8 @@ on:
       - ".github/actions/llm/convert-test/action.yml"
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
+  push:
+    branches: [test-branch2.4]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml
index 0492a6376855..af716cde21ac 100644
--- a/.github/workflows/llm_performance_tests.yml
+++ b/.github/workflows/llm_performance_tests.yml
@@ -20,6 +20,8 @@ on:
       - "python/llm/test/benchmark/**"
   workflow_dispatch:
   workflow_call:
+  push:
+    branches: [test-branch2.4]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs: