feat: PHS-392 added metadata to Example

CHANGELOG.md

@@ -12,6 +12,7 @@
  - `Lineages` now contain `Tracer` for individual `Output`s.
  - `convert_to_pandas_data_frame` now also creates a column containing the `Tracer`s.
  - `run_dataset` now has a flag `trace_examples_individually` to create `Tracer`s for each example. Defaults to True.
+ - Added optional `metadata` field to `Example`.
 
 ### Fixes
   - ControlModels throw a warning instead of an error in case a not-recommended model is selected.

src/documentation/how_tos/how_to_create_a_dataset.ipynb

@@ -53,6 +53,9 @@
     "    Example(\n",
     "        input=StoryTaskInput(topic=\"rain\", targeted_word_count=42),\n",
     "        expected_output=StoryTaskExpectedOutput(keywords=[\"wet\"]),\n",
+    "        metadata={\n",
+    "            \"author\": \"Shakespeare\"\n",
+    "        },  # the metadata is optional and can contain custom information\n",
     "    ),\n",
     "    # ...\n",
     "]\n",

src/intelligence_layer/connectors/base/json_serializable.py

@@ -23,3 +23,5 @@
         | Sequence["JsonSerializable"]
         | Mapping[str, "JsonSerializable"],
     )
+
+SerializableDict = dict[str, JsonSerializable]

src/intelligence_layer/evaluation/dataset/domain.py

@@ -1,9 +1,10 @@
-from typing import Generic, TypeVar
+from typing import Generic, Optional, TypeVar
 from uuid import uuid4
 
 from pydantic import BaseModel, Field
 from rich.tree import Tree
 
+from intelligence_layer.connectors.base.json_serializable import SerializableDict
 from intelligence_layer.core.task import Input
 from intelligence_layer.core.tracer.tracer import PydanticSerializable
 
@@ -30,6 +31,7 @@ class Example(BaseModel, Generic[Input, ExpectedOutput]):
     input: Input
     expected_output: ExpectedOutput
     id: str = Field(default_factory=lambda: str(uuid4()))
+    metadata: Optional[SerializableDict] = None
 
     def __repr__(self) -> str:
         return self.__str__()
@@ -39,12 +41,15 @@ def __str__(self) -> str:
             f"Example ID = {self.id}\n"
             f"Input = {self.input}\n"
             f'Expected output = "{self.expected_output}"\n'
+            f"Metadata = {self.metadata}\n"
         )
 
     def _rich_render(self) -> Tree:
         example_tree = Tree(f"Example: {self.id}")
         example_tree.add("Input").add(str(self.input))
         example_tree.add("Expected Output").add(str(self.expected_output))
+        if self.metadata:
+            example_tree.add("Metadata").add(str(self.metadata))
         return example_tree
 
 

tests/evaluation/conftest.py

@@ -113,7 +113,11 @@ def file_run_repository(tmp_path: Path) -> FileRunRepository:
 
 @fixture
 def dummy_string_example() -> Example[DummyStringInput, DummyStringOutput]:
-    return Example(input=DummyStringInput(), expected_output=DummyStringOutput())
+    return Example(
+        input=DummyStringInput(),
+        expected_output=DummyStringOutput(),
+        metadata={"some_key": "some_value"},
+    )
 
 
 @fixture