From 95ce0e38882b7fd40b1f8bf476f7b31d8c886c6a Mon Sep 17 00:00:00 2001
From: BabyCNM <86091026+BabyCNM@users.noreply.github.com>
Date: Sun, 6 Oct 2024 11:07:08 -0700
Subject: [PATCH 1/2] LMM: use `_generate_oai_reply_from_client`

---
 .../contrib/multimodal_conversable_agent.py   | 14 +-----
 test/agentchat/contrib/test_lmm.py            | 46 ++++++++++++++++++-
 2 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/autogen/agentchat/contrib/multimodal_conversable_agent.py b/autogen/agentchat/contrib/multimodal_conversable_agent.py
index 9cb6a5cf765..946b187af86 100644
--- a/autogen/agentchat/contrib/multimodal_conversable_agent.py
+++ b/autogen/agentchat/contrib/multimodal_conversable_agent.py
@@ -15,8 +15,6 @@
 )
 from autogen.code_utils import content_str
 
-from ..._pydantic import model_dump
-
 DEFAULT_LMM_SYS_MSG = """You are a helpful AI assistant."""
 DEFAULT_MODEL = "gpt-4-vision-preview"
 
@@ -116,13 +114,5 @@ def generate_oai_reply(
 
         messages_with_b64_img = message_formatter_pil_to_b64(self._oai_system_message + messages)
 
-        # TODO: #1143 handle token limit exceeded error
-        response = client.create(
-            context=messages[-1].pop("context", None), messages=messages_with_b64_img, agent=self.name
-        )
-
-        # TODO: line 301, line 271 is converting messages to dict. Can be removed after ChatCompletionMessage_to_dict is merged.
-        extracted_response = client.extract_text_or_completion_object(response)[0]
-        if not isinstance(extracted_response, str):
-            extracted_response = model_dump(extracted_response)
-        return True, extracted_response
+        extracted_response = self._generate_oai_reply_from_client(client, messages_with_b64_img, self.client_cache)
+        return (False, None) if extracted_response is None else (True, extracted_response)
diff --git a/test/agentchat/contrib/test_lmm.py b/test/agentchat/contrib/test_lmm.py
index 5d2d45aaff4..fe58470ec5b 100755
--- a/test/agentchat/contrib/test_lmm.py
+++ b/test/agentchat/contrib/test_lmm.py
@@ -10,10 +10,11 @@
 from unittest.mock import MagicMock
 
 import pytest
+from annotated_types import Annotated
 from conftest import MOCK_OPEN_AI_API_KEY
 
 import autogen
-from autogen.agentchat.conversable_agent import ConversableAgent
+from autogen import AssistantAgent, ConversableAgent, UserProxyAgent
 
 try:
     from autogen.agentchat.contrib.img_utils import get_pil_image
@@ -23,6 +24,7 @@
 else:
     skip = False
 
+VISION_MODEL_NAME = "gpt-4-turbo"
 
 base64_encoded_image = (
     "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4"
@@ -44,7 +46,7 @@ def setUp(self):
             llm_config={
                 "timeout": 600,
                 "seed": 42,
-                "config_list": [{"model": "gpt-4-vision-preview", "api_key": MOCK_OPEN_AI_API_KEY}],
+                "config_list": [{"model": VISION_MODEL_NAME, "api_key": MOCK_OPEN_AI_API_KEY}],
             },
         )
 
@@ -144,5 +146,45 @@ def test_group_chat_with_lmm():
     assert all(len(arr) <= max_round for arr in user_proxy._oai_messages.values()), "User proxy exceeded max rounds"
 
 
+@pytest.mark.skipif(skip, reason="Dependency not installed")
+def test_func_call_with_lmm():
+    assistant = MultimodalConversableAgent(
+        name="Assistant",
+        system_message="Describe all the colors in the image.",
+        human_input_mode="NEVER",
+        max_consecutive_auto_reply=2,
+        llm_config={
+            "timeout": 600,
+            "seed": 42,
+            "config_list": [{"model": VISION_MODEL_NAME, "api_key": MOCK_OPEN_AI_API_KEY}],
+        },
+    )
+
+    coder = AssistantAgent(
+        name="Coder",
+        system_message="YOU MUST USE THE FUNCTION PROVIDED.",
+        llm_config={
+            "timeout": 600,
+            "seed": 42,
+            "config_list": [{"model": VISION_MODEL_NAME, "api_key": MOCK_OPEN_AI_API_KEY}],
+        },
+        human_input_mode="NEVER",
+        code_execution_config=False,
+        max_consecutive_auto_reply=2,
+    )
+
+    def count_colors(colors: list) -> int:
+        return len(colors)
+
+    coder.register_for_llm(name="count_colors", description="Count colors.")(count_colors)
+    assistant.register_for_execution(name="count_colors")(count_colors)
+
+    coder.initiate_chat(
+        assistant, clear_history=True, message=f"""How many colors here: <img {base64_encoded_image}>"""
+    )
+
+    assert len(coder._oai_messages[assistant]) > 1, "Function call did not happen"
+
+
 if __name__ == "__main__":
     unittest.main()

From c5c8eadaa4d5d4495e576f80b7ce1cdf33a28954 Mon Sep 17 00:00:00 2001
From: BabyCNM <86091026+BabyCNM@users.noreply.github.com>
Date: Sun, 6 Oct 2024 11:16:42 -0700
Subject: [PATCH 2/2] Update the default vision model to gpt-4-turbo

---
 autogen/agentchat/contrib/multimodal_conversable_agent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/autogen/agentchat/contrib/multimodal_conversable_agent.py b/autogen/agentchat/contrib/multimodal_conversable_agent.py
index 946b187af86..598e9503fc6 100644
--- a/autogen/agentchat/contrib/multimodal_conversable_agent.py
+++ b/autogen/agentchat/contrib/multimodal_conversable_agent.py
@@ -16,7 +16,7 @@
 from autogen.code_utils import content_str
 
 DEFAULT_LMM_SYS_MSG = """You are a helpful AI assistant."""
-DEFAULT_MODEL = "gpt-4-vision-preview"
+DEFAULT_MODEL = "gpt-4-turbo"
 
 
 class MultimodalConversableAgent(ConversableAgent):