Updated ParameterizedModelParser run method for handing run_with_dependencies parameter

python/src/aiconfig/default_parsers/parameterized_model_parser.py

@@ -45,13 +45,10 @@ async def run(
         aiconfig: AIConfig,
         options: Optional[InferenceOptions] = None,
         parameters: Dict = {},
-        **kwargs,  # TODO: We should remove and make arguments explicit
+        run_with_dependencies: Optional[bool] = False,
     ) -> List[Output]:
-        # maybe use prompt metadata instead of kwargs?
-        if kwargs.get("run_with_dependencies", False):
-            return await self.run_with_dependencies(
-                prompt, aiconfig, options, parameters
-            )
+        if run_with_dependencies:
+            return await self.run_with_dependencies(prompt, aiconfig, options, parameters)
         else:
             return await self.run_inference(
                 prompt, aiconfig, options, parameters

python/src/aiconfig/model_parser.py

@@ -66,7 +66,6 @@ async def run(
         aiconfig: AIConfig,
         options: Optional["InferenceOptions"] = None,
         parameters: Dict = {},
-        **kwargs,  # TODO: Remove this, just a hack for now to ensure that it doesn't break
     ) -> ExecuteResult:
         """
         Execute model inference based on completion data to be constructed in deserialize(), which includes the input prompt and

python/tests/aiconfigs/tarvel_gpt_prompts_with_dependency.json

@@ -0,0 +1,35 @@
+{
+  "name": "NYC Trip Planner",
+  "schema_version": "latest",
+  "description": "Intrepid explorer with ChatGPT and AIConfig",
+
+  "metadata": {
+    "parameters": {},
+    "models": {
+      "gpt-3.5-turbo": {
+        "model": "gpt-3.5-turbo",
+        "top_p": 1,
+        "temperature": 1,
+        "stream": false
+      }
+    }
+  },
+
+  "prompts": [
+    {
+      "name": "get_activities",
+      "input": "Tell me 10 fun attractions to do in {{city}}.",
+      "metadata": {
+        "model": "gpt-3.5-turbo"
+      }
+    },
+    {
+      "name": "gen_itinerary",
+      "input": "Generate an itinerary ordered by {{order_by}} for these activities: {{get_activities.output}}.",
+      "metadata": {
+        "model": "gpt-3.5-turbo"
+      }
+    }
+  ],
+  "$schema": "https://json.schemastore.org/aiconfig-1.0"
+}

python/tests/conftest.py

@@ -99,6 +99,82 @@ def mock_openai_chat_completion(**kwargs):
     raise Exception("Unexpected arguments:\n {}".format(kwargs))
 
 
+def mock_openai_chat_completion_with_dependencies(**kwargs):
+    response_map_list = [
+        [
+            {
+                "model": "gpt-3.5-turbo",
+                "top_p": 1,
+                "temperature": 1,
+                "stream": False,
+                "messages": [{"content": "Tell me 10 fun attractions to do in NYC.", "role": "user"}],
+            },
+            ChatCompletion(
+                **{
+                    "id": "chatcmpl-8khHUHbetNyprE1ftmRu5WyVNP3O3",
+                    "choices": [
+                        {
+                            "finish_reason": "stop",
+                            "index": 0,
+                            "message": {
+                                "content": '1. Visit Times Square: Experience the iconic "Crossroads of the World" with its dazzling billboards, Broadway shows, and lively atmosphere.\n2. Explore Central Park: Take a leisurely stroll or rent a bike to discover this sprawling oasis in the heart of Manhattan, complete with picturesque landscapes, walking trails, and various attractions.\n3. Take a ferry to the Statue of Liberty: Visit the renowned Statue of Liberty on Liberty Island and enjoy stunning views of the city skyline from the ferry ride itself.\n4. Visit the Metropolitan Museum of Art: Immerse yourself in world-class art collections spanning thousands of years and diverse cultures, all housed in the magnificent Met museum.\n5. Explore the High Line: Walk along the elevated park built on a historic freight rail line, offering beautiful green spaces, public art installations, and fantastic views of the city.\n6. Enjoy a Broadway show: Watch a live performance of a Broadway musical or play in one of the iconic theaters in the Theater District, offering a memorable experience for theater enthusiasts.\n7. Visit the 9/11 Memorial and Museum: Pay your respects at the site of the former World Trade Center twin towers and learn about the events of September 11, 2001, at the informative museum.\n8. Take a food tour in Greenwich Village: Indulge in a delicious gastronomic adventure, exploring the vibrant food scene in Greenwich Village, renowned for its diverse restaurants and eateries.\n9. Visit the Brooklyn Bridge: Take a walk across this iconic suspension bridge, providing breathtaking views of the Manhattan skyline and the East River.\n10. Explore the Museum of Modern Art (MoMA): Discover a vast collection of modern and contemporary art, including works by artists like Van Gogh, Picasso, Warhol, and many more at this renowned art institution.',
+                                "role": "assistant",
+                            },
+                        }
+                    ],
+                    "created": 1706140152,
+                    "model": "gpt-3.5-turbo-0613",
+                    "object": "chat.completion",
+                    "usage": {"completion_tokens": 363, "prompt_tokens": 18, "total_tokens": 381},
+                }
+            ),
+        ],
+        # Example 2
+        [
+            {
+                "model": "gpt-3.5-turbo",
+                "temperature": 1,
+                "top_p": 1,
+                "stream": False,
+                "messages": [
+                    {"content": "Tell me 10 fun attractions to do in NYC.", "role": "user"},
+                    {
+                        "content": '1. Visit Times Square: Experience the iconic "Crossroads of the World" with its dazzling billboards, Broadway shows, and lively atmosphere.\n2. Explore Central Park: Take a leisurely stroll or rent a bike to discover this sprawling oasis in the heart of Manhattan, complete with picturesque landscapes, walking trails, and various attractions.\n3. Take a ferry to the Statue of Liberty: Visit the renowned Statue of Liberty on Liberty Island and enjoy stunning views of the city skyline from the ferry ride itself.\n4. Visit the Metropolitan Museum of Art: Immerse yourself in world-class art collections spanning thousands of years and diverse cultures, all housed in the magnificent Met museum.\n5. Explore the High Line: Walk along the elevated park built on a historic freight rail line, offering beautiful green spaces, public art installations, and fantastic views of the city.\n6. Enjoy a Broadway show: Watch a live performance of a Broadway musical or play in one of the iconic theaters in the Theater District, offering a memorable experience for theater enthusiasts.\n7. Visit the 9/11 Memorial and Museum: Pay your respects at the site of the former World Trade Center twin towers and learn about the events of September 11, 2001, at the informative museum.\n8. Take a food tour in Greenwich Village: Indulge in a delicious gastronomic adventure, exploring the vibrant food scene in Greenwich Village, renowned for its diverse restaurants and eateries.\n9. Visit the Brooklyn Bridge: Take a walk across this iconic suspension bridge, providing breathtaking views of the Manhattan skyline and the East River.\n10. Explore the Museum of Modern Art (MoMA): Discover a vast collection of modern and contemporary art, including works by artists like Van Gogh, Picasso, Warhol, and many more at this renowned art institution.',
+                        "role": "assistant",
+                    },
+                    {
+                        "content": "Generate an itinerary ordered by geographic location for these activities: 1. Visit Times Square: Experience the iconic "Crossroads of the World" with its dazzling billboards, Broadway shows, and lively atmosphere.\n2. Explore Central Park: Take a leisurely stroll or rent a bike to discover this sprawling oasis in the heart of Manhattan, complete with picturesque landscapes, walking trails, and various attractions.\n3. Take a ferry to the Statue of Liberty: Visit the renowned Statue of Liberty on Liberty Island and enjoy stunning views of the city skyline from the ferry ride itself.\n4. Visit the Metropolitan Museum of Art: Immerse yourself in world-class art collections spanning thousands of years and diverse cultures, all housed in the magnificent Met museum.\n5. Explore the High Line: Walk along the elevated park built on a historic freight rail line, offering beautiful green spaces, public art installations, and fantastic views of the city.\n6. Enjoy a Broadway show: Watch a live performance of a Broadway musical or play in one of the iconic theaters in the Theater District, offering a memorable experience for theater enthusiasts.\n7. Visit the 9/11 Memorial and Museum: Pay your respects at the site of the former World Trade Center twin towers and learn about the events of September 11, 2001, at the informative museum.\n8. Take a food tour in Greenwich Village: Indulge in a delicious gastronomic adventure, exploring the vibrant food scene in Greenwich Village, renowned for its diverse restaurants and eateries.\n9. Visit the Brooklyn Bridge: Take a walk across this iconic suspension bridge, providing breathtaking views of the Manhattan skyline and the East River.\n10. Explore the Museum of Modern Art (MoMA): Discover a vast collection of modern and contemporary art, including works by artists like Van Gogh, Picasso, Warhol, and many more at this renowned art institution..",
+                        "role": "user",
+                    },
+                ],
+            },
+            ChatCompletion(
+                **{
+                    "id": "chatcmpl-8khHahyYOoH456N6fjXgs2PQjfqDD",
+                    "choices": [
+                        {
+                            "finish_reason": "stop",
+                            "index": 0,
+                            "message": {
+                                "content": "Here's an itinerary ordered by geographic location:\n\n1. Start your day at Times Square.\n2. From there, head south to the 9/11 Memorial and Museum.\n3. After paying your respects, take a walk to the nearby Staten Island Ferry Terminal to catch a ferry to the Statue of Liberty on Liberty Island.\n4. Return from the ferry and head north to the High Line, located in the Chelsea neighborhood.\n5. Continue north to the Museum of Modern Art (MoMA) in Midtown Manhattan.\n6. From MoMA, head east to visit Central Park.\n7. Explore Central Park at your leisure, taking a stroll or renting a bike to fully enjoy the park's attractions.\n8. Afterward, head downtown to the Metropolitan Museum of Art (Met) on the Upper East Side to immerse yourself in its world-class art collections.\n9. From the Met, travel southwest to Greenwich Village for a delightful food tour, exploring the diverse culinary offerings.\n10. Wrap up your day by walking across the Brooklyn Bridge to enjoy stunning views of the Manhattan skyline and the East River.\n\nNote: This itinerary can be adjusted based on personal preferences and time availability.",
+                                "role": "assistant",
+                            },
+                        }
+                    ],
+                    "created": 1706140158,
+                    "model": "gpt-3.5-turbo-0613",
+                    "object": "chat.completion",
+                    "usage": {"completion_tokens": 236, "prompt_tokens": 767, "total_tokens": 1003},
+                }
+            ),
+        ],
+    ]
+
+    for input_params, response in response_map_list:
+        if kwargs == input_params:
+            return response
+    raise Exception("Unexpected arguments:\n {}".format(kwargs))
+
 @pytest.fixture
 def set_temporary_env_vars():
     """

python/tests/test_run_config.py

@@ -1,9 +1,11 @@
+from unittest.mock import Mock
+
 import openai
 import pytest
 from aiconfig.Config import AIConfigRuntime
 from mock import patch
 
-from .conftest import mock_openai_chat_completion
+from .conftest import mock_openai_chat_completion, mock_openai_chat_completion_with_dependencies
 from .util.file_path_utils import get_absolute_file_path_from_relative
 
 @pytest.mark.xfail
@@ -48,3 +50,23 @@ async def test_load_parametrized_data_config(set_temporary_env_vars):
                 },
             ],
         }
+
+@pytest.mark.xfail
+@pytest.mark.asyncio
+async def test_running_prompt_with_dependencies(set_temporary_env_vars):
+    """Test running a prompt with dependencies with the run_with_dependencies flag set to True"""
+
+    mock_openai = Mock(side_effect=mock_openai_chat_completion_with_dependencies)
+
+    with patch.object(openai.chat.completions, "create", new=mock_openai):
+        config_relative_path = "aiconfigs/tarvel_gpt_prompts_with_dependency.json"
+        config_absolute_path = get_absolute_file_path_from_relative(__file__, config_relative_path)
+        config = AIConfigRuntime.load(config_absolute_path)
+
+        combined_prompt_parameters = {
+            "city": "NYC",
+            "order_by": "geographic location",
+        }
+        await config.run("gen_itinerary", combined_prompt_parameters, run_with_dependencies=True)
+
+    assert mock_openai.call_count == 2