Merge pull request #174 from kinfey/main

Update Phi-3.5 Content
microsoft · Sep 2, 2024 · f07f1c6 · f07f1c6
2 parents 1d8f658 + 0b5623d
commit f07f1c6
Show file tree

Hide file tree

Showing 24 changed files with 1,757 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -123,6 +123,17 @@ Phi, is a family of open AI models developed by Microsoft. Phi models are the mo
   - [Using Phi-3 with Promptflow and Azure AI Search](./code/07.Lab/RAG_with_PromptFlow_and_AISearch/README.md)(✅)
   - [Windows AI-PC APIs with Windows Copilot Library](https://developer.microsoft.com/windows/ai/?WT.mc_id=aiml-137032-kinfeylo)
 
+- Learning Phi-3.5
+  - [What's new Phi-3.5 Family](./md/08.Update/Phi35/010.WhatsNewInPhi35.md)(✅)
+  - [Quantifying Phi-3.5 Family](./md/08.Update/Phi35/020.QuantifyingPhi35.md)(✅)
+    - [Quantizing Phi-3.5 using llama.cpp](./md/08.Update/Phi35/021.UsingLlamacppQuantifyingPhi35.md)(✅)
+    - [Quantizing Phi-3.5 using Generative AI extensions for onnxruntime](./md/08.Update/Phi35/022.UsingORTGenAIQuantifyingPhi35.md)(✅)
+    - [Quantizing Phi-3.5 using Intel OpenVINO](./md/08.Update/Phi35/023.UsingIntelOpenVINOQuantifyingPhi35.md)(✅)
+    - [Quantizing Phi-3.5 using Apple MLX Framework](./md/08.Update/Phi35/024.UsingAppleMLXQuantifyingPhi35.md)(✅)
+  - Phi-3.5 Application Samples
+    - [Phi-3.5-Instruct WebGPU RAG Chatbot](./md/08.Update/Phi35/031.WebGPUWithPhi35Readme.md)(✅)
+
+
 ## Using Phi-3 Models
 
 ### Phi-3 on Azure AI Studio

diff --git a/code/09.UpdateSamples/Aug/imgs/demo.png b/code/09.UpdateSamples/Aug/imgs/demo.png
diff --git a/code/09.UpdateSamples/Aug/intel-phi35-instruct-zh.ipynb b/code/09.UpdateSamples/Aug/intel-phi35-instruct-zh.ipynb
@@ -0,0 +1,134 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Lab-1-Introduce Phi-3.5 Instruct"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Convert Phi-3-mini-4k-instruct model to INT4 with optimum-cli"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "llm_model_id = \"microsoft/Phi-3.5-mini-instruct\"\n",
+    "llm_model_path = \"../model/phi-3.5-mini-instruct-ov\"\n",
+    "\n",
+    "if not Path(llm_model_path).exists():\n",
+    "    !optimum-cli export openvino --model {llm_model_id} --task text-generation-with-past --weight-format int4 --group-size 128 --ratio 0.6  --sym  --trust-remote-code {llm_model_path}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoConfig, AutoTokenizer\n",
+    "from optimum.intel.openvino import OVModelForCausalLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ov_config = {\"PERFORMANCE_HINT\": \"LATENCY\", \"NUM_STREAMS\": \"1\", \"CACHE_DIR\": \"\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ov_model = OVModelForCausalLM.from_pretrained(\n",
+    "    llm_model_path,\n",
+    "    device='GPU',\n",
+    "    ov_config=ov_config,\n",
+    "    config=AutoConfig.from_pretrained(llm_model_path, trust_remote_code=True),\n",
+    "    trust_remote_code=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tok = AutoTokenizer.from_pretrained(llm_model_path, trust_remote_code=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer_kwargs =  {\"add_special_tokens\": False}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"<|user|>\\n你了解 .NET 吗?\\n<|end|><|assistant|>\\n\"\n",
+    "input_tokens = tok(prompt, return_tensors=\"pt\", **tokenizer_kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "answer = ov_model.generate(**input_tokens, max_new_tokens=1024)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tok.batch_decode(answer, skip_special_tokens=True)[0]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "openvinodev",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/code/09.UpdateSamples/Aug/intel-phi35-vision-img.ipynb b/code/09.UpdateSamples/Aug/intel-phi35-vision-img.ipynb
@@ -0,0 +1,184 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Lab-2-Introduce Phi-3.5 Vision"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if not Path(\"ov_phi3_vision.py\").exists():\n",
+    "    r = requests.get(url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/phi-3-vision/ov_phi3_vision.py\")\n",
+    "    open(\"ov_phi3_vision.py\", \"w\").write(r.text)\n",
+    "\n",
+    "\n",
+    "if not Path(\"gradio_helper.py\").exists():\n",
+    "    r = requests.get(url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/phi-3-vision/gradio_helper.py\")\n",
+    "    open(\"gradio_helper.py\", \"w\").write(r.text)\n",
+    "\n",
+    "if not Path(\"notebook_utils.py\").exists():\n",
+    "    r = requests.get(url=\"https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py\")\n",
+    "    open(\"notebook_utils.py\", \"w\").write(r.text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ov_phi3_vision import convert_phi3_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "import nncf\n",
+    "\n",
+    "\n",
+    "model_id = \"microsoft/Phi-3.5-vision-instruct\"\n",
+    "out_dir = Path(\"../model/phi-3.5-vision-128k-instruct-ov\")\n",
+    "compression_configuration = {\n",
+    "    \"mode\": nncf.CompressWeightsMode.INT4_SYM,\n",
+    "    \"group_size\": 64,\n",
+    "    \"ratio\": 0.6,\n",
+    "}\n",
+    "if not out_dir.exists():\n",
+    "    convert_phi3_model(model_id, out_dir, compression_configuration)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from notebook_utils import device_widget\n",
+    "\n",
+    "device = device_widget(default=\"GPU\", exclude=[\"NPU\"])\n",
+    "\n",
+    "device"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ov_phi3_vision import OvPhi3Vision"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out_dir"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = OvPhi3Vision(out_dir, device.value)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from PIL import Image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image = Image.open(r\"./imgs/demo.png\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoProcessor, TextStreamer\n",
+    "\n",
+    "messages = [\n",
+    "    {\"role\": \"user\", \"content\": \"<|image_1|>\\nPlease create Python code for image, and use plt to save the new picture under imgs/ and name it phi-3-vision.jpg.\"},\n",
+    "]\n",
+    "\n",
+    "processor = AutoProcessor.from_pretrained(out_dir, trust_remote_code=True)\n",
+    "\n",
+    "prompt = processor.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
+    "\n",
+    "inputs = processor(prompt, [image], return_tensors=\"pt\")\n",
+    "\n",
+    "generation_args = {\"max_new_tokens\": 3072, \"do_sample\": False, \"streamer\": TextStreamer(processor.tokenizer, skip_prompt=True, skip_special_tokens=True)}\n",
+    "\n",
+    "print(\"Coding:\")\n",
+    "generate_ids = model.generate(**inputs, eos_token_id=processor.tokenizer.eos_token_id, **generation_args)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "openvinodev",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}