[Model] Initialize deepseek-vl support

docs/source/models/supported_models.rst

@@ -210,6 +210,10 @@ Vision Language Models
     - MiniCPM-V
     - :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
     -
+  * - :code:`DeepSeekMultiModalityCausalLM`
+    - deepseek-ai
+    - :code:`deepseek-ai/deepseek-vl-1.3b-chat`, :code:`deepseek-ai/deepseek-vl-7b-chat`, etc.
+    -
 
 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
 Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>` 
@@ -271,4 +275,4 @@ We have the following levels of testing for models:
 1. **Strict Consistency**: We compare the output of the model with the output of the model in the HuggingFace Transformers library under greedy decoding. This is the most stringent test. Please refer to `test_models.py <https://github.com/vllm-project/vllm/blob/main/tests/models/test_models.py>`_ and `test_big_models.py <https://github.com/vllm-project/vllm/blob/main/tests/models/test_big_models.py>`_ for the models that have passed this test.
 2. **Output Sensibility**: We check if the output of the model is sensible and coherent, by measuring the perplexity of the output and checking for any obvious errors. This is a less stringent test.
 3. **Runtime Functionality**: We check if the model can be loaded and run without errors. This is the least stringent test. Please refer to `functionality tests <https://github.com/vllm-project/vllm/tree/main/tests>`_ and `examples <https://github.com/vllm-project/vllm/tree/main/examples>`_ for the models that have passed this test.
-4. **Community Feedback**: We rely on the community to provide feedback on the models. If a model is broken or not working as expected, we encourage users to raise issues to report it or open pull requests to fix it. The rest of the models fall under this category.
+4. **Community Feedback**: We rely on the community to provide feedback on the models. If a model is broken or not working as expected, we encourage users to raise issues to report it or open pull requests to fix it. The rest of the models fall under this category.

examples/deepseek_vl_example.py

@@ -0,0 +1,54 @@
+from vllm import LLM, SamplingParams
+from vllm.assets.image import ImageAsset
+
+sample_params = SamplingParams(temperature=0, max_tokens=1024)
+model = "deepseek-ai/deepseek-vl-7b-chat"
+model = "deepseek-ai/deepseek-vl-1.3b-chat"
+prompt_one = (
+    "You are a helpful language and vision assistant."
+    "You are able to understand the visual content that the user provides,"
+    "and assist the user with a variety of tasks using natural language.\n"
+    "User: <image_placeholder> Describe the content of this image.\nAssistant:"
+)
+
+prompt_two = (
+    "You are a helpful language and vision assistant. You are able to "
+    "understand the visual content that the user provides, and assist the "
+    "user with a variety of tasks using natural language.\n User: "
+    "<image_placeholder>What is the season?\nAssistant:")
+
+
+def run_deepseek_vl():
+    llm = LLM(model=model)
+    stop_sign_image = ImageAsset("stop_sign").pil_image
+    cherry_blossom_image = ImageAsset("cherry_blossom").pil_image
+    outputs = llm.generate(
+        [
+            {
+                "prompt": prompt_one,
+                "multi_modal_data": {
+                    "image": stop_sign_image
+                },
+            },
+            {
+                "prompt": prompt_two,
+                "multi_modal_data": {
+                    "image": cherry_blossom_image
+                },
+            },
+        ],
+        sampling_params=sample_params,
+    )
+
+    for o in outputs:
+        generated_text = o.outputs[0].text
+        print("------------------")
+        print(generated_text)
+
+
+def main():
+    run_deepseek_vl()
+
+
+if __name__ == "__main__":
+    main()

tests/entrypoints/openai/conftest.py

@@ -66,4 +66,4 @@ def sample_sql_statements():
 table: "table_1" | "table_2"
 condition: column "=" number
 number: "1" | "2"
-""")
+""")