From ca0e86062e751e2a46245fa33dcfc1ace32a1361 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Thu, 21 Sep 2023 16:32:09 +0800
Subject: [PATCH 01/19] modify Dockerfile

---
 docker/llm/inference/cpu/docker/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/llm/inference/cpu/docker/Dockerfile b/docker/llm/inference/cpu/docker/Dockerfile
index 97bb2a0dd01..1e5143f9be9 100644
--- a/docker/llm/inference/cpu/docker/Dockerfile
+++ b/docker/llm/inference/cpu/docker/Dockerfile
@@ -13,6 +13,7 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get update && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     apt-get install -y python3-pip python3.9-dev python3-wheel python3.9-distutils && \
     pip3 install --no-cache --upgrade requests argparse urllib3 && \
-    pip3 install --pre --upgrade bigdl-llm[all] 
+    pip3 install --pre --upgrade bigdl-llm[all] && \
+    pip3 install --pre --upgrade bigdl-nano
 
 ENTRYPOINT ["/bin/bash"]
\ No newline at end of file

From 8ca46d004f3ebd97a6a79a19cb6e1899eebbe6c5 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Thu, 21 Sep 2023 16:34:07 +0800
Subject: [PATCH 02/19] add README.md

---
 docker/llm/inference/cpu/docker/README.md | 34 +++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docker/llm/inference/cpu/docker/README.md

diff --git a/docker/llm/inference/cpu/docker/README.md b/docker/llm/inference/cpu/docker/README.md
new file mode 100644
index 00000000000..5856cc1b4b6
--- /dev/null
+++ b/docker/llm/inference/cpu/docker/README.md
@@ -0,0 +1,34 @@
+## Build/Use BigDL-LLM cpu image
+
+### Build Image
+```bash
+docker build \
+  --build-arg http_proxy=.. \
+  --build-arg https_proxy=.. \
+  --build-arg no_proxy=.. \
+  --rm --no-cache -t intelanalytics/bigdl-llm-cpu:2.4.0-SNAPSHOT .
+```
+
+
+### Use the image for doing cpu inference
+
+
+An example could be:
+```bash
+#/bin/bash
+export DOCKER_IMAGE=intelanalytics/bigdl-llm-cpu:2.4.0-SNAPSHOT
+
+sudo docker run -itd \
+        --net=host \
+        --cpuset-cpus="0-47" \
+        --cpuset-mems="0" \
+        --memory="32G" \
+        --name=CONTAINER_NAME \
+        --shm-size="16g" \
+        $DOCKER_IMAGE
+```
+
+
+After the container is booted, you could get into the container through `docker exec`.
+
+

From f985068491530a3397022b9a89b2c8440dc7c390 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Thu, 21 Sep 2023 16:58:37 +0800
Subject: [PATCH 03/19] add README.md

---
 docker/llm/inference/cpu/docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/llm/inference/cpu/docker/README.md b/docker/llm/inference/cpu/docker/README.md
index 5856cc1b4b6..805cfc07c92 100644
--- a/docker/llm/inference/cpu/docker/README.md
+++ b/docker/llm/inference/cpu/docker/README.md
@@ -31,4 +31,4 @@ sudo docker run -itd \
 
 After the container is booted, you could get into the container through `docker exec`.
 
-
+To run inference using `BigDL-LLM` using cpu, you could refer to this [documentation](https://github.com/intel-analytics/BigDL/tree/main/python/llm#cpu-int4).

From fc8bf6b0d57605fb0b59f84f3cc6ad70e6b50142 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 25 Sep 2023 14:05:08 +0800
Subject: [PATCH 04/19] Modify Dockerfile

---
 docker/llm/inference/cpu/docker/Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docker/llm/inference/cpu/docker/Dockerfile b/docker/llm/inference/cpu/docker/Dockerfile
index 1e5143f9be9..ca83f1cb12d 100644
--- a/docker/llm/inference/cpu/docker/Dockerfile
+++ b/docker/llm/inference/cpu/docker/Dockerfile
@@ -14,6 +14,10 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get update && \
     apt-get install -y python3-pip python3.9-dev python3-wheel python3.9-distutils && \
     pip3 install --no-cache --upgrade requests argparse urllib3 && \
     pip3 install --pre --upgrade bigdl-llm[all] && \
-    pip3 install --pre --upgrade bigdl-nano
+    pip3 install --pre --upgrade bigdl-nano && \
+# Download chat.py script
+    cd /root && \
+    wget https://raw.githubusercontent.com/intel-analytics/BigDL/main/python/llm/portable-executable/chat.py && \
+    export PYTHONUNBUFFERED=1
 
 ENTRYPOINT ["/bin/bash"]
\ No newline at end of file

From ceed895c3117eb4de3c33f039a09fd0275c09594 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 25 Sep 2023 14:31:43 +0800
Subject: [PATCH 05/19] Add bigdl inference cpu image build

---
 .github/workflows/manually_build.yml | 33 ++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 1e70c0a1b52..66ef1aa4fb8 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,6 +1,10 @@
 name: Manually Build
 
 on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -10,6 +14,7 @@ on:
         type: choice
         options:
         - all
+        - bigdl-llm-cpu
         - bigdl-ppml-gramine-base
         - bigdl-ppml-trusted-bigdl-llm-gramine-base
         - bigdl-ppml-trusted-bigdl-llm-gramine-ref
@@ -54,6 +59,34 @@ permissions:
   packages: write
 
 jobs:
+  bigdl-llm-cpu:
+    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    runs-on: [self-hosted, Shire]
+    steps:
+    - uses: actions/checkout@v3
+    - name: docker login
+      run: |
+        docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
+    - name: bigdl-llm-cpu
+      run: |
+        echo "##############################################################"
+        echo "####### bigdl-llm-cpu ########"
+        echo "##############################################################"
+        export image=intelanalytics/bigdl-llm-cpu
+        cd docker/llm/inference/cpu/docker
+        sudo docker build \
+          --no-cache=true \
+          --build-arg http_proxy=${HTTP_PROXY} \
+          --build-arg https_proxy=${HTTPS_PROXY} \
+          --build-arg no_proxy=${NO_PROXY} \
+          --build-arg BASE_IMAGE_NAME=${base_image} \
+          --build-arg BASE_IMAGE_TAG=${TAG} \
+          -t ${image}:${TAG} -f ./Dockerfile .
+        sudo docker push ${image}:${TAG}
+        sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+        sudo docker push 10.239.45.10/arda/${image}:${TAG}
+        sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+
   bigdl-ppml-gramine-base:
     if: ${{ github.event.inputs.artifact == 'bigdl-ppml-gramine-base' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]

From 9cae4600da3284fa2fefe04c0927a9d99cda2f84 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 25 Sep 2023 14:45:30 +0800
Subject: [PATCH 06/19] Add bigdl llm cpu image build

---
 .github/workflows/manually_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 66ef1aa4fb8..5d3cd1460e9 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -52,7 +52,7 @@ on:
         type: string
 
 env:
-  TAG: ${{ github.event.inputs.tag }}
+  TAG: zhengjin-test
 
 permissions:
   contents: read

From 7f2d2a52385de8bf41202d5c594471320ae4a6fc Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 25 Sep 2023 15:14:23 +0800
Subject: [PATCH 07/19] Add bigdl llm cpu image build

---
 .github/workflows/manually_build.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 5d3cd1460e9..3f3741bcf28 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,10 +1,6 @@
 name: Manually Build
 
 on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -52,7 +48,7 @@ on:
         type: string
 
 env:
-  TAG: zhengjin-test
+  TAG: ${{ github.event.inputs.tag }}
 
 permissions:
   contents: read
@@ -60,7 +56,7 @@ permissions:
 
 jobs:
   bigdl-llm-cpu:
-    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From 847af63e8ea62d02704f58deca0443a64670aaeb Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 25 Sep 2023 15:33:39 +0800
Subject: [PATCH 08/19] Add bigdl llm cpu image build

---
 .../workflows/manually_build_for_testing.yml  | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/.github/workflows/manually_build_for_testing.yml b/.github/workflows/manually_build_for_testing.yml
index c64f82013d3..1b7230259b2 100644
--- a/.github/workflows/manually_build_for_testing.yml
+++ b/.github/workflows/manually_build_for_testing.yml
@@ -14,6 +14,7 @@ on:
         type: choice
         options:
         - all
+        - bigdl-llm-cpu
         - bigdl-ppml-gramine-base
         - bigdl-ppml-trusted-bigdl-llm-gramine-base
         - bigdl-ppml-trusted-bigdl-llm-gramine-ref
@@ -51,6 +52,36 @@ permissions:
   packages: write
 
 jobs:
+    bigdl-llm-cpu:
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    runs-on: [self-hosted, Shire]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.inputs.sha }}
+    - name: docker login
+      run: |
+        docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
+    - name: bigdl-llm-cpu
+      run: |
+        echo "##############################################################"
+        echo "####### bigdl-llm-cpu ########"
+        echo "##############################################################"
+        export image=intelanalytics/bigdl-llm-cpu
+        cd docker/llm/inference/cpu/docker
+        sudo docker build \
+          --no-cache=true \
+          --build-arg http_proxy=${HTTP_PROXY} \
+          --build-arg https_proxy=${HTTPS_PROXY} \
+          --build-arg no_proxy=${NO_PROXY} \
+          --build-arg BASE_IMAGE_NAME=${base_image} \
+          --build-arg BASE_IMAGE_TAG=${TAG} \
+          -t ${image}:${TAG} -f ./Dockerfile .
+        sudo docker push ${image}:${TAG}
+        sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+        sudo docker push 10.239.45.10/arda/${image}:${TAG}
+        sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+
   bigdl-ppml-gramine-base:
     if: ${{ github.event.inputs.artifact == 'bigdl-ppml-gramine-base' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]

From 7a69bee8d013d0189175f6e681eb80f6c550aacc Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 13:58:42 +0800
Subject: [PATCH 09/19] Modify Dockerfile

---
 docker/llm/inference/xpu/docker/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/llm/inference/xpu/docker/Dockerfile b/docker/llm/inference/xpu/docker/Dockerfile
index 0366c933538..0b7da551b12 100644
--- a/docker/llm/inference/xpu/docker/Dockerfile
+++ b/docker/llm/inference/xpu/docker/Dockerfile
@@ -1,7 +1,7 @@
 FROM intel/oneapi-basekit:2023.2.1-devel-ubuntu22.04
 
-ENV http_proxy $HTTP_PROXY
-ENV https_proxy $HTTP_PROXY
+ARG http_proxy
+ARG https_proxy
 
 ENV TZ=Asia/Shanghai
 

From 390c90551e56e4d804dedd543b66946e7676301b Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 14:03:55 +0800
Subject: [PATCH 10/19] Add bigdl inference cpu image build

---
 .github/workflows/manually_build.yml          | 36 ++++++++++++++++++-
 .../workflows/manually_build_for_testing.yml  | 31 ++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 3f3741bcf28..28a8bdfe52e 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,6 +1,10 @@
 name: Manually Build
 
 on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -10,6 +14,7 @@ on:
         type: choice
         options:
         - all
+        - bigdl-llm-xpu
         - bigdl-llm-cpu
         - bigdl-ppml-gramine-base
         - bigdl-ppml-trusted-bigdl-llm-gramine-base
@@ -48,13 +53,42 @@ on:
         type: string
 
 env:
-  TAG: ${{ github.event.inputs.tag }}
+  TAG: zhengjin-test
+  # TAG: ${{ github.event.inputs.tag }}
 
 permissions:
   contents: read
   packages: write
 
 jobs:
+  bigdl-llm-xpu:
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    runs-on: [self-hosted, Shire]
+    steps:
+    - uses: actions/checkout@v3
+    - name: docker login
+      run: |
+        docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
+    - name: bigdl-llm-xpu
+      run: |
+        echo "##############################################################"
+        echo "####### bigdl-llm-xpu ########"
+        echo "##############################################################"
+        export image=intelanalytics/bigdl-llm-xpu
+        cd docker/llm/inference/xpu/docker
+        sudo docker build \
+          --no-cache=true \
+          --build-arg http_proxy=${HTTP_PROXY} \
+          --build-arg https_proxy=${HTTPS_PROXY} \
+          --build-arg no_proxy=${NO_PROXY} \
+          --build-arg BASE_IMAGE_NAME=${base_image} \
+          --build-arg BASE_IMAGE_TAG=${TAG} \
+          -t ${image}:${TAG} -f ./Dockerfile .
+        sudo docker push ${image}:${TAG}
+        sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+        sudo docker push 10.239.45.10/arda/${image}:${TAG}
+        sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+
   bigdl-llm-cpu:
     if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
diff --git a/.github/workflows/manually_build_for_testing.yml b/.github/workflows/manually_build_for_testing.yml
index 1b7230259b2..b0cbebc1469 100644
--- a/.github/workflows/manually_build_for_testing.yml
+++ b/.github/workflows/manually_build_for_testing.yml
@@ -14,6 +14,7 @@ on:
         type: choice
         options:
         - all
+        - bigdl-llm-xpu
         - bigdl-llm-cpu
         - bigdl-ppml-gramine-base
         - bigdl-ppml-trusted-bigdl-llm-gramine-base
@@ -52,6 +53,36 @@ permissions:
   packages: write
 
 jobs:
+  bigdl-llm-xpu:
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    runs-on: [self-hosted, Shire]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.inputs.sha }}
+    - name: docker login
+      run: |
+        docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
+    - name: bigdl-llm-xpu
+      run: |
+        echo "##############################################################"
+        echo "####### bigdl-llm-xpu ########"
+        echo "##############################################################"
+        export image=intelanalytics/bigdl-llm-xpu
+        cd docker/llm/inference/xpu/docker
+        sudo docker build \
+          --no-cache=true \
+          --build-arg http_proxy=${HTTP_PROXY} \
+          --build-arg https_proxy=${HTTPS_PROXY} \
+          --build-arg no_proxy=${NO_PROXY} \
+          --build-arg BASE_IMAGE_NAME=${base_image} \
+          --build-arg BASE_IMAGE_TAG=${TAG} \
+          -t ${image}:${TAG} -f ./Dockerfile .
+        sudo docker push ${image}:${TAG}
+        sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+        sudo docker push 10.239.45.10/arda/${image}:${TAG}
+        sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
+
     bigdl-llm-cpu:
     if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]

From 781bc5bc8dcfd83ffcda04f3d17ba509ea52932c Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 14:07:36 +0800
Subject: [PATCH 11/19] Add bigdl inference cpu image build

---
 .github/workflows/manually_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 28a8bdfe52e..343d0752452 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -62,7 +62,7 @@ permissions:
 
 jobs:
   bigdl-llm-xpu:
-    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From d0ac0941a26aeb40c51adf2145d98cc0471ddd56 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 14:25:10 +0800
Subject: [PATCH 12/19] Add bigdl llm xpu image build

---
 .github/workflows/manually_build.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 343d0752452..24603c7347d 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,10 +1,6 @@
 name: Manually Build
 
 on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -53,8 +49,7 @@ on:
         type: string
 
 env:
-  TAG: zhengjin-test
-  # TAG: ${{ github.event.inputs.tag }}
+  TAG: ${{ github.event.inputs.tag }}
 
 permissions:
   contents: read
@@ -62,7 +57,7 @@ permissions:
 
 jobs:
   bigdl-llm-xpu:
-    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From 2dc76dc358b4a5d945c60a24bf793fa98c8c46c0 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 15:15:15 +0800
Subject: [PATCH 13/19] manually build

---
 .github/workflows/manually_build.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 24603c7347d..f34b61fd698 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,6 +1,10 @@
 name: Manually Build
 
 on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -49,7 +53,8 @@ on:
         type: string
 
 env:
-  TAG: ${{ github.event.inputs.tag }}
+  # TAG: ${{ github.event.inputs.tag }}
+  TAG: 2.4.0-SNAPSHOT
 
 permissions:
   contents: read
@@ -57,7 +62,7 @@ permissions:
 
 jobs:
   bigdl-llm-xpu:
-    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3
@@ -85,7 +90,7 @@ jobs:
         sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
 
   bigdl-llm-cpu:
-    if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From b17e536a1bff5fddbb3c02c7fa526b5f8eea617d Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 15:45:03 +0800
Subject: [PATCH 14/19] recover file

---
 .github/workflows/manually_build.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index f34b61fd698..24603c7347d 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,10 +1,6 @@
 name: Manually Build
 
 on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -53,8 +49,7 @@ on:
         type: string
 
 env:
-  # TAG: ${{ github.event.inputs.tag }}
-  TAG: 2.4.0-SNAPSHOT
+  TAG: ${{ github.event.inputs.tag }}
 
 permissions:
   contents: read
@@ -62,7 +57,7 @@ permissions:
 
 jobs:
   bigdl-llm-xpu:
-    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3
@@ -90,7 +85,7 @@ jobs:
         sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
 
   bigdl-llm-cpu:
-    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From efc215821518a32887498dec4b552d318d57452f Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 15:47:47 +0800
Subject: [PATCH 15/19] manually build

---
 .github/workflows/manually_build.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 24603c7347d..f34b61fd698 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,6 +1,10 @@
 name: Manually Build
 
 on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -49,7 +53,8 @@ on:
         type: string
 
 env:
-  TAG: ${{ github.event.inputs.tag }}
+  # TAG: ${{ github.event.inputs.tag }}
+  TAG: 2.4.0-SNAPSHOT
 
 permissions:
   contents: read
@@ -57,7 +62,7 @@ permissions:
 
 jobs:
   bigdl-llm-xpu:
-    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3
@@ -85,7 +90,7 @@ jobs:
         sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
 
   bigdl-llm-cpu:
-    if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From 9935772f24f11183e791833d22c62db49cfa6c39 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Tue, 26 Sep 2023 15:50:51 +0800
Subject: [PATCH 16/19] recover file

---
 .github/workflows/manually_build.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index f34b61fd698..24603c7347d 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -1,10 +1,6 @@
 name: Manually Build
 
 on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - '.github/workflows/manually_build.yml'
   workflow_dispatch:
     inputs:
       artifact:
@@ -53,8 +49,7 @@ on:
         type: string
 
 env:
-  # TAG: ${{ github.event.inputs.tag }}
-  TAG: 2.4.0-SNAPSHOT
+  TAG: ${{ github.event.inputs.tag }}
 
 permissions:
   contents: read
@@ -62,7 +57,7 @@ permissions:
 
 jobs:
   bigdl-llm-xpu:
-    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3
@@ -90,7 +85,7 @@ jobs:
         sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
 
   bigdl-llm-cpu:
-    # if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
+    if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
     steps:
     - uses: actions/checkout@v3

From ddcd9e7d0a918561089f075eb6446c0d0e7e7618 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Wed, 27 Sep 2023 14:49:58 +0800
Subject: [PATCH 17/19] modify indentation

---
 .github/workflows/manually_build.yml             | 2 ++
 .github/workflows/manually_build_for_testing.yml | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml
index 24603c7347d..e5fe603d15b 100644
--- a/.github/workflows/manually_build.yml
+++ b/.github/workflows/manually_build.yml
@@ -59,6 +59,7 @@ jobs:
   bigdl-llm-xpu:
     if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
+    
     steps:
     - uses: actions/checkout@v3
     - name: docker login
@@ -87,6 +88,7 @@ jobs:
   bigdl-llm-cpu:
     if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
+
     steps:
     - uses: actions/checkout@v3
     - name: docker login
diff --git a/.github/workflows/manually_build_for_testing.yml b/.github/workflows/manually_build_for_testing.yml
index b0cbebc1469..17ec21e26e6 100644
--- a/.github/workflows/manually_build_for_testing.yml
+++ b/.github/workflows/manually_build_for_testing.yml
@@ -56,6 +56,7 @@ jobs:
   bigdl-llm-xpu:
     if: ${{ github.event.inputs.artifact == 'bigdl-llm-xpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
+    
     steps:
     - uses: actions/checkout@v3
       with:
@@ -83,9 +84,10 @@ jobs:
         sudo docker push 10.239.45.10/arda/${image}:${TAG}
         sudo docker rmi -f ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG}
 
-    bigdl-llm-cpu:
+  bigdl-llm-cpu:
     if: ${{ github.event.inputs.artifact == 'bigdl-llm-cpu' || github.event.inputs.artifact == 'all' }}
     runs-on: [self-hosted, Shire]
+
     steps:
     - uses: actions/checkout@v3
       with:

From 3814abf95a2317803ce65116a2d9fe418810c016 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 9 Oct 2023 12:57:28 +0800
Subject: [PATCH 18/19] add instruction for chat.py

---
 docker/llm/inference/cpu/docker/Dockerfile |  3 ++-
 docker/llm/inference/cpu/docker/README.md  | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/docker/llm/inference/cpu/docker/Dockerfile b/docker/llm/inference/cpu/docker/Dockerfile
index 80747466c03..4cc05c2dd80 100644
--- a/docker/llm/inference/cpu/docker/Dockerfile
+++ b/docker/llm/inference/cpu/docker/Dockerfile
@@ -22,7 +22,8 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get update && \
     pip install --pre --upgrade bigdl-llm[all] && \
     pip install --pre --upgrade bigdl-nano && \
 # Download chat.py script
-    wget -P /root https://raw.githubusercontent.com/intel-analytics/BigDL/main/python/llm/portable-executable/chat.py && \
+    pip install --upgrade colorama && \
+    wget -P /root https://raw.githubusercontent.com/intel-analytics/BigDL/main/python/llm/portable-zip/chat.py && \
     export PYTHONUNBUFFERED=1
 
 ENTRYPOINT ["/bin/bash"]
\ No newline at end of file
diff --git a/docker/llm/inference/cpu/docker/README.md b/docker/llm/inference/cpu/docker/README.md
index 805cfc07c92..589fe32c488 100644
--- a/docker/llm/inference/cpu/docker/README.md
+++ b/docker/llm/inference/cpu/docker/README.md
@@ -32,3 +32,13 @@ sudo docker run -itd \
 After the container is booted, you could get into the container through `docker exec`.
 
 To run inference using `BigDL-LLM` using cpu, you could refer to this [documentation](https://github.com/intel-analytics/BigDL/tree/main/python/llm#cpu-int4).
+
+### Use chat.py
+
+chat.py can be used to initiate a conversation with a specified model. The file is under directory '/root'.
+
+To run chat.py:
+```
+cd /root
+python chat.py --model-path YOUR_MODEL_PATH
+```

From a1aefdb8f4d848c57a11c0d3b8059bd85e67c930 Mon Sep 17 00:00:00 2001
From: Wang <zhengjin.wang@intel.com>
Date: Mon, 9 Oct 2023 13:36:29 +0800
Subject: [PATCH 19/19] modify README

---
 docker/llm/inference/cpu/docker/README.md | 26 ++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/docker/llm/inference/cpu/docker/README.md b/docker/llm/inference/cpu/docker/README.md
index 589fe32c488..2712a9bd77c 100644
--- a/docker/llm/inference/cpu/docker/README.md
+++ b/docker/llm/inference/cpu/docker/README.md
@@ -37,8 +37,32 @@ To run inference using `BigDL-LLM` using cpu, you could refer to this [documenta
 
 chat.py can be used to initiate a conversation with a specified model. The file is under directory '/root'.
 
-To run chat.py:
+You can download models and bind the model directory from host machine to container when start a container.
+
+Here is an example:
+```bash
+export DOCKER_IMAGE=intelanalytics/bigdl-llm-cpu:2.4.0-SNAPSHOT
+export MODEL_PATH=/home/llm/models
+
+sudo docker run -itd \
+        --net=host \
+        --cpuset-cpus="0-47" \
+        --cpuset-mems="0" \
+        --memory="32G" \
+        --name=CONTAINER_NAME \
+        --shm-size="16g" \
+        -v $MODEL_PATH:/llm/models/
+        $DOCKER_IMAGE
+
 ```
+
+After entering the container through `docker exec`, you can run chat.py by:
+```bash
 cd /root
 python chat.py --model-path YOUR_MODEL_PATH
 ```
+In the example above, it can be:
+```bash
+cd /root
+python chat.py --model-path /llm/models/MODEL_NAME
+```
\ No newline at end of file