From d9f400a1a0954a4a27196bf896b4e8ca912f1766 Mon Sep 17 00:00:00 2001 From: <> Date: Fri, 16 Aug 2024 18:41:09 +0000 Subject: [PATCH] Deployed 40f7b3bf with MkDocs version: 1.6.0 --- client_api/Schemas/Spatial/Box/index.html | 414 +++++----- .../Schemas/Spatial/LineString/index.html | 112 +-- .../Spatial/MultiLineString/index.html | 136 +-- .../Schemas/Spatial/MultiPoint/index.html | 104 +-- .../Schemas/Spatial/MultiPolygon/index.html | 330 ++++---- client_api/Schemas/Spatial/Point/index.html | 130 +-- client_api/Schemas/Spatial/Polygon/index.html | 340 ++++---- client_api/Schemas/Spatial/Raster/index.html | 772 +++++++++--------- search/search_index.json | 2 +- 9 files changed, 1172 insertions(+), 1168 deletions(-) diff --git a/client_api/Schemas/Spatial/Box/index.html b/client_api/Schemas/Spatial/Box/index.html index d79bd198e..8f998f244 100644 --- a/client_api/Schemas/Spatial/Box/index.html +++ b/client_api/Schemas/Spatial/Box/index.html @@ -633,9 +633,7 @@

Box

Source code in valor/schemas/symbolic/types.py -
1115
-1116
-1117
+
1117
 1118
 1119
 1120
@@ -749,123 +747,125 @@ 
Box

 1228
 1229
 1230
-1231
class Box(Polygon):
-    """
-    A Box is a polygon that is constrained to 4 unique points.
-
-    Note that this does not need to be axis-aligned.
+1231
+1232
+1233
class Box(Polygon):
+    """
+    A Box is a polygon that is constrained to 4 unique points.
 
-    Parameters
-    ----------
-    value : List[List[Tuple[float, float]]], optional
-        An polygon value representing a box.
-
-    Attributes
-    ----------
-    area
-    polygon
-    boundary
-    holes
-    xmin
-    xmax
-    ymin
-    ymax
-
-    Examples
-    --------
-    >>> Box([[(0,0), (0,1), (1,1), (1,0), (0,0)]])
-
-    Create a Box using extrema.
-    >>> Box.from_extrema(
-    ...     xmin=0, xmax=1,
-    ...     ymin=0, ymax=1,
-    ... )
-    """
-
-    def __init__(
-        self, value: typing.List[typing.List[typing.Tuple[float, float]]]
-    ):
-        super().__init__(value=value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        """
-        Validates typing.
-
-        Parameters
-        ----------
-        value : typing.Any
-            The value to validate.
-
-        Raises
-        ------
-        TypeError
-            If the value type is not supported.
-        """
-        Polygon.__validate__(value)
-        if len(value) != 1:
-            raise ValueError("Box should not contain holes.")
-        elif len(value[0]) != 5:
-            raise ValueError("Box should consist of four unique points.")
-
-    @classmethod
-    def decode_value(
-        cls,
-        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
-    ):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return super().decode_value(value)
-
-    @classmethod
-    def from_extrema(
-        cls,
-        xmin: float,
-        xmax: float,
-        ymin: float,
-        ymax: float,
-    ):
-        """
-        Create a Box from extrema values.
-
-        Parameters
-        ----------
-        xmin : float
-            Minimum x-coordinate of the bounding box.
-        xmax : float
-            Maximum x-coordinate of the bounding box.
-        ymin : float
-            Minimum y-coordinate of the bounding box.
-        ymax : float
-            Maximum y-coordinate of the bounding box.
-
-        Returns
-        -------
-        Box
-            A Box created from the provided extrema values.
-        """
-        points = [
-            [
-                (xmin, ymin),
-                (xmax, ymin),
-                (xmax, ymax),
-                (xmin, ymax),
-                (xmin, ymin),
-            ]
-        ]
-        return cls(value=points)
-
-    def to_polygon(self) -> Polygon:
-        """
-        Converts box to a generic polygon.
-
-        Returns
-        -------
-        Polygon
-            The box as a Polygon.
-        """
-        return Polygon(self.get_value())
+    Note that this does not need to be axis-aligned.
+
+    Parameters
+    ----------
+    value : List[List[Tuple[float, float]]], optional
+        An polygon value representing a box.
+
+    Attributes
+    ----------
+    area
+    polygon
+    boundary
+    holes
+    xmin
+    xmax
+    ymin
+    ymax
+
+    Examples
+    --------
+    >>> Box([[(0,0), (0,1), (1,1), (1,0), (0,0)]])
+
+    Create a Box using extrema.
+    >>> Box.from_extrema(
+    ...     xmin=0, xmax=1,
+    ...     ymin=0, ymax=1,
+    ... )
+    """
+
+    def __init__(
+        self, value: typing.List[typing.List[typing.Tuple[float, float]]]
+    ):
+        super().__init__(value=value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        """
+        Validates typing.
+
+        Parameters
+        ----------
+        value : typing.Any
+            The value to validate.
+
+        Raises
+        ------
+        TypeError
+            If the value type is not supported.
+        """
+        Polygon.__validate__(value)
+        if len(value) != 1:
+            raise ValueError("Box should not contain holes.")
+        elif len(value[0]) != 5:
+            raise ValueError("Box should consist of four unique points.")
+
+    @classmethod
+    def decode_value(
+        cls,
+        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
+    ):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return super().decode_value(value)
+
+    @classmethod
+    def from_extrema(
+        cls,
+        xmin: float,
+        xmax: float,
+        ymin: float,
+        ymax: float,
+    ):
+        """
+        Create a Box from extrema values.
+
+        Parameters
+        ----------
+        xmin : float
+            Minimum x-coordinate of the bounding box.
+        xmax : float
+            Maximum x-coordinate of the bounding box.
+        ymin : float
+            Minimum y-coordinate of the bounding box.
+        ymax : float
+            Maximum y-coordinate of the bounding box.
+
+        Returns
+        -------
+        Box
+            A Box created from the provided extrema values.
+        """
+        points = [
+            [
+                (xmin, ymin),
+                (xmax, ymin),
+                (xmax, ymax),
+                (xmin, ymax),
+                (xmin, ymin),
+            ]
+        ]
+        return cls(value=points)
+
+    def to_polygon(self) -> Polygon:
+        """
+        Converts box to a generic polygon.
+
+        Returns
+        -------
+        Polygon
+            The box as a Polygon.
+        """
+        return Polygon(self.get_value())
 

 
 

@@ -929,9 +929,7 @@
Source code in valor/schemas/symbolic/types.py -
1153
-1154
-1155
+
1155
 1156
 1157
 1158
@@ -948,26 +946,28 @@ 

 1169
 1170
 1171
-1172
@classmethod
-def __validate__(cls, value: typing.Any):
-    """
-    Validates typing.
-
-    Parameters
-    ----------
-    value : typing.Any
-        The value to validate.
-
-    Raises
-    ------
-    TypeError
-        If the value type is not supported.
-    """
-    Polygon.__validate__(value)
-    if len(value) != 1:
-        raise ValueError("Box should not contain holes.")
-    elif len(value[0]) != 5:
-        raise ValueError("Box should consist of four unique points.")
+1172
+1173
+1174
@classmethod
+def __validate__(cls, value: typing.Any):
+    """
+    Validates typing.
+
+    Parameters
+    ----------
+    value : typing.Any
+        The value to validate.
+
+    Raises
+    ------
+    TypeError
+        If the value type is not supported.
+    """
+    Polygon.__validate__(value)
+    if len(value) != 1:
+        raise ValueError("Box should not contain holes.")
+    elif len(value[0]) != 5:
+        raise ValueError("Box should consist of four unique points.")
@@ -983,23 +983,23 @@

Decode object from JSON compatible dictionary.

Source code in valor/schemas/symbolic/types.py -
1174
-1175
-1176
+
1176
 1177
 1178
 1179
 1180
 1181
-1182
@classmethod
-def decode_value(
-    cls,
-    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
-):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return super().decode_value(value)
+1182
+1183
+1184
@classmethod
+def decode_value(
+    cls,
+    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
+):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return super().decode_value(value)
@@ -1105,9 +1105,7 @@

Source code in valor/schemas/symbolic/types.py -
1184
-1185
-1186
+
1186
 1187
 1188
 1189
@@ -1141,43 +1139,45 @@ 

 1217
 1218
 1219
-1220
@classmethod
-def from_extrema(
-    cls,
-    xmin: float,
-    xmax: float,
-    ymin: float,
-    ymax: float,
-):
-    """
-    Create a Box from extrema values.
-
-    Parameters
-    ----------
-    xmin : float
-        Minimum x-coordinate of the bounding box.
-    xmax : float
-        Maximum x-coordinate of the bounding box.
-    ymin : float
-        Minimum y-coordinate of the bounding box.
-    ymax : float
-        Maximum y-coordinate of the bounding box.
-
-    Returns
-    -------
-    Box
-        A Box created from the provided extrema values.
-    """
-    points = [
-        [
-            (xmin, ymin),
-            (xmax, ymin),
-            (xmax, ymax),
-            (xmin, ymax),
-            (xmin, ymin),
-        ]
-    ]
-    return cls(value=points)
+1220
+1221
+1222
@classmethod
+def from_extrema(
+    cls,
+    xmin: float,
+    xmax: float,
+    ymin: float,
+    ymax: float,
+):
+    """
+    Create a Box from extrema values.
+
+    Parameters
+    ----------
+    xmin : float
+        Minimum x-coordinate of the bounding box.
+    xmax : float
+        Maximum x-coordinate of the bounding box.
+    ymin : float
+        Minimum y-coordinate of the bounding box.
+    ymax : float
+        Maximum y-coordinate of the bounding box.
+
+    Returns
+    -------
+    Box
+        A Box created from the provided extrema values.
+    """
+    points = [
+        [
+            (xmin, ymin),
+            (xmax, ymin),
+            (xmax, ymax),
+            (xmin, ymax),
+            (xmin, ymin),
+        ]
+    ]
+    return cls(value=points)
@@ -1211,25 +1211,25 @@

Source code in valor/schemas/symbolic/types.py -
1222
-1223
-1224
+
1224
 1225
 1226
 1227
 1228
 1229
 1230
-1231
def to_polygon(self) -> Polygon:
-    """
-    Converts box to a generic polygon.
-
-    Returns
-    -------
-    Polygon
-        The box as a Polygon.
-    """
-    return Polygon(self.get_value())
+1231
+1232
+1233
def to_polygon(self) -> Polygon:
+    """
+    Converts box to a generic polygon.
+
+    Returns
+    -------
+    Polygon
+        The box as a Polygon.
+    """
+    return Polygon(self.get_value())
diff --git a/client_api/Schemas/Spatial/LineString/index.html b/client_api/Schemas/Spatial/LineString/index.html index 8106101b0..23588ee74 100644 --- a/client_api/Schemas/Spatial/LineString/index.html +++ b/client_api/Schemas/Spatial/LineString/index.html @@ -529,9 +529,7 @@

LineString

Source code in valor/schemas/symbolic/types.py -
863
-864
-865
+
865
 866
 867
 868
@@ -571,49 +569,51 @@ 
LineString

 902
 903
 904
-905
class LineString(Spatial):
-    """
-    Represents a line.
-
-    Follows the GeoJSON specification (RFC 7946).
+905
+906
+907
class LineString(Spatial):
+    """
+    Represents a line.
 
-    Parameters
-    ----------
-    value : List[Tuple[float, float]], optional
-        A linestring.
-
-    Methods
-    -------
-    colorspace(c='rgb')
-        Represent the photo in the given colorspace.
-    gamma(n=1.0)
-        Change the photo's gamma exposure.
-
-    Examples
-    --------
-    Create a line.
-    >>> LineString([(0,0), (0,1), (1,1)])
-    """
-
-    def __init__(self, value: typing.List[typing.Tuple[float, float]]):
-        super().__init__(value=value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        MultiPoint.__validate__(value)
-        if len(value) < 2:
-            raise ValueError(
-                "At least two points are required to make a line."
-            )
-
-    @classmethod
-    def decode_value(
-        cls, value: typing.Optional[typing.List[typing.List[float]]]
-    ):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return cls([(point[0], point[1]) for point in value])
+    Follows the GeoJSON specification (RFC 7946).
+
+    Parameters
+    ----------
+    value : List[Tuple[float, float]], optional
+        A linestring.
+
+    Methods
+    -------
+    colorspace(c='rgb')
+        Represent the photo in the given colorspace.
+    gamma(n=1.0)
+        Change the photo's gamma exposure.
+
+    Examples
+    --------
+    Create a line.
+    >>> LineString([(0,0), (0,1), (1,1)])
+    """
+
+    def __init__(self, value: typing.List[typing.Tuple[float, float]]):
+        super().__init__(value=value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        MultiPoint.__validate__(value)
+        if len(value) < 2:
+            raise ValueError(
+                "At least two points are required to make a line."
+            )
+
+    @classmethod
+    def decode_value(
+        cls, value: typing.Optional[typing.List[typing.List[float]]]
+    ):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return cls([(point[0], point[1]) for point in value])
 

 
 

@@ -629,21 +629,21 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
898
-899
-900
+
900
 901
 902
 903
 904
-905
@classmethod
-def decode_value(
-    cls, value: typing.Optional[typing.List[typing.List[float]]]
-):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return cls([(point[0], point[1]) for point in value])
+905
+906
+907
@classmethod
+def decode_value(
+    cls, value: typing.Optional[typing.List[typing.List[float]]]
+):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return cls([(point[0], point[1]) for point in value])
 

 
 

diff --git a/client_api/Schemas/Spatial/MultiLineString/index.html b/client_api/Schemas/Spatial/MultiLineString/index.html
index 373dcfead..01115e294 100644
--- a/client_api/Schemas/Spatial/MultiLineString/index.html
+++ b/client_api/Schemas/Spatial/MultiLineString/index.html
@@ -511,9 +511,7 @@ 
MultiLineString

 
 

 Source code in valor/schemas/symbolic/types.py
-
908
-909
-910
+
910
 911
 912
 913
@@ -562,58 +560,60 @@ 
MultiLineString

 956
 957
 958
-959
class MultiLineString(Spatial):
-    """
-    Represents a list of lines.
-
-    Follows the GeoJSON specification (RFC 7946).
+959
+960
+961
class MultiLineString(Spatial):
+    """
+    Represents a list of lines.
 
-    Parameters
-    ----------
-    value : List[List[Tuple[float, float]]], optional
-        A multilinestring.
-
-    Examples
-    --------
-    Create a single line.
-    >>> MultiLineString([[(0,0), (0,1), (1,1), (0,0)]])
-
-    Create 3 lines.
-    >>> MultiLineString(
-    ...     [
-    ...         [(0,0), (0,1), (1,1)],
-    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2)],
-    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7)],
-    ...     ]
-    ... )
-    """
-
-    def __init__(
-        self,
-        value: typing.List[typing.List[typing.Tuple[float, float]]],
-    ):
-        super().__init__(value=value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        if not isinstance(value, list):
-            raise TypeError(
-                f"Expected type 'List[List[Tuple[float, float]]]' received type '{type(value).__name__}'"
-            )
-        for line in value:
-            LineString.__validate__(line)
-
-    @classmethod
-    def decode_value(
-        cls,
-        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
-    ):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return cls(
-            [[(point[0], point[1]) for point in line] for line in value]
-        )
+    Follows the GeoJSON specification (RFC 7946).
+
+    Parameters
+    ----------
+    value : List[List[Tuple[float, float]]], optional
+        A multilinestring.
+
+    Examples
+    --------
+    Create a single line.
+    >>> MultiLineString([[(0,0), (0,1), (1,1), (0,0)]])
+
+    Create 3 lines.
+    >>> MultiLineString(
+    ...     [
+    ...         [(0,0), (0,1), (1,1)],
+    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2)],
+    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7)],
+    ...     ]
+    ... )
+    """
+
+    def __init__(
+        self,
+        value: typing.List[typing.List[typing.Tuple[float, float]]],
+    ):
+        super().__init__(value=value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        if not isinstance(value, list):
+            raise TypeError(
+                f"Expected type 'List[List[Tuple[float, float]]]' received type '{type(value).__name__}'"
+            )
+        for line in value:
+            LineString.__validate__(line)
+
+    @classmethod
+    def decode_value(
+        cls,
+        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
+    ):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return cls(
+            [[(point[0], point[1]) for point in line] for line in value]
+        )
 

 
 

@@ -629,9 +629,7 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
949
-950
-951
+
951
 952
 953
 954
@@ -639,17 +637,19 @@ 

 956
 957
 958
-959
@classmethod
-def decode_value(
-    cls,
-    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
-):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return cls(
-        [[(point[0], point[1]) for point in line] for line in value]
-    )
+959
+960
+961
@classmethod
+def decode_value(
+    cls,
+    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
+):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return cls(
+        [[(point[0], point[1]) for point in line] for line in value]
+    )
 

 
 

diff --git a/client_api/Schemas/Spatial/MultiPoint/index.html b/client_api/Schemas/Spatial/MultiPoint/index.html
index f791c90e5..aed781328 100644
--- a/client_api/Schemas/Spatial/MultiPoint/index.html
+++ b/client_api/Schemas/Spatial/MultiPoint/index.html
@@ -501,9 +501,7 @@ 
MultiPoint

 
 

 Source code in valor/schemas/symbolic/types.py
-
822
-823
-824
+
824
 825
 826
 827
@@ -539,45 +537,47 @@ 
MultiPoint

 857
 858
 859
-860
class MultiPoint(Spatial):
-    """
-    Represents a list of points.
-
-    Follows the GeoJSON specification (RFC 7946).
+860
+861
+862
class MultiPoint(Spatial):
+    """
+    Represents a list of points.
 
-    Parameters
-    ----------
-    value : List[Tuple[float, float]], optional
-        A multipoint.
-
-    Examples
-    --------
-    >>> MultiPoint([(0,0), (0,1), (1,1)])
-    """
-
-    def __init__(
-        self,
-        value: typing.List[typing.Tuple[float, float]],
-    ):
-        super().__init__(value=value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        if not isinstance(value, list):
-            raise TypeError(
-                f"Expected 'typing.List[typing.Tuple[float, float]]' received type '{type(value).__name__}'"
-            )
-        for point in value:
-            Point.__validate__(point)
-
-    @classmethod
-    def decode_value(
-        cls, value: typing.Optional[typing.List[typing.List[float]]]
-    ):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return cls([(point[0], point[1]) for point in value])
+    Follows the GeoJSON specification (RFC 7946).
+
+    Parameters
+    ----------
+    value : List[Tuple[float, float]], optional
+        A multipoint.
+
+    Examples
+    --------
+    >>> MultiPoint([(0,0), (0,1), (1,1)])
+    """
+
+    def __init__(
+        self,
+        value: typing.List[typing.Tuple[float, float]],
+    ):
+        super().__init__(value=value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        if not isinstance(value, list):
+            raise TypeError(
+                f"Expected 'typing.List[typing.Tuple[float, float]]' received type '{type(value).__name__}'"
+            )
+        for point in value:
+            Point.__validate__(point)
+
+    @classmethod
+    def decode_value(
+        cls, value: typing.Optional[typing.List[typing.List[float]]]
+    ):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return cls([(point[0], point[1]) for point in value])
 

 
 

@@ -593,21 +593,21 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
853
-854
-855
+
855
 856
 857
 858
 859
-860
@classmethod
-def decode_value(
-    cls, value: typing.Optional[typing.List[typing.List[float]]]
-):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return cls([(point[0], point[1]) for point in value])
+860
+861
+862
@classmethod
+def decode_value(
+    cls, value: typing.Optional[typing.List[typing.List[float]]]
+):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return cls([(point[0], point[1]) for point in value])
 

 
 

diff --git a/client_api/Schemas/Spatial/MultiPolygon/index.html b/client_api/Schemas/Spatial/MultiPolygon/index.html
index 4dd21e8c7..dfadaf642 100644
--- a/client_api/Schemas/Spatial/MultiPolygon/index.html
+++ b/client_api/Schemas/Spatial/MultiPolygon/index.html
@@ -606,9 +606,7 @@ 
MultiPolygon

 
 

 Source code in valor/schemas/symbolic/types.py
-
1234
-1235
-1236
+
1236
 1237
 1238
 1239
@@ -715,116 +713,118 @@ 
MultiPolygon

 1340
 1341
 1342
-1343
class MultiPolygon(Spatial):
-    """
-    Represents a collection of polygons.
-
-    Follows the GeoJSON specification (RFC 7946).
+1343
+1344
+1345
class MultiPolygon(Spatial):
+    """
+    Represents a collection of polygons.
 
-    Parameters
-    ----------
-    value : List[List[List[Tuple[float, float]]]], optional
-        A list of polygons.
-
-    Attributes
-    ----------
-    area
-    polygons
-
-    Examples
-    --------
-    >>> MultiPolygon(
-    ...     [
-    ...         [
-    ...             [(0,0), (0,1), (1,1), (0,0)]
-    ...         ],
-    ...         [
-    ...             [(0,0), (0,1), (1,1), (0,0)],
-    ...             [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],
-    ...             [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],
-    ...         ],
-    ...     ]
-    ... )
-    """
-
-    def __init__(
-        self,
-        value: typing.List[
-            typing.List[typing.List[typing.Tuple[float, float]]]
-        ],
-    ):
-        super().__init__(value=value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        if not isinstance(value, list):
-            raise TypeError(
-                f"Expected type 'List[List[List[Tuple[float, float]]]]' received type '{type(value).__name__}'"
-            )
-        for poly in value:
-            Polygon.__validate__(poly)
-
-    @classmethod
-    def decode_value(
-        cls,
-        value: typing.Optional[
-            typing.List[typing.List[typing.List[typing.List[float]]]]
-        ],
-    ):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return cls(
-            [
-                [
-                    [(point[0], point[1]) for point in subpolygon]
-                    for subpolygon in polygon
-                ]
-                for polygon in value
-            ]
-        )
-
-    @property
-    def area(self):
-        """
-        Symbolic representation of area.
-        """
-        if not isinstance(self._value, Symbol):
-            raise ValueError(
-                "attribute 'area' is reserved for symbolic variables."
-            )
-        return Float.symbolic(
-            name=self._value._name,
-            key=self._value._key,
-            attribute="area",
-        )
-
-    def to_polygons(self) -> typing.List[Polygon]:
-        """
-        Converts multipolygon to a list of Polygon instances.
-
-        Returns
-        -------
-        List[Polygon]
-        """
-        return [Polygon(poly) for poly in self.get_value()]
-
-    @classmethod
-    def from_polygons(self, polygons: typing.List[Polygon]) -> "MultiPolygon":
-        """
-        Converts a list of Polygon instances to a MultiPolygon.
-
-        Parameters
-        ----------
-        polygons : List[Polygon]
-            A list of Polygon instances.
-
-        Returns
-        -------
-        MultiPolygon
-            A MultiPolygon instance.
-        """
-        return MultiPolygon([poly.get_value() for poly in polygons])
+    Follows the GeoJSON specification (RFC 7946).
+
+    Parameters
+    ----------
+    value : List[List[List[Tuple[float, float]]]], optional
+        A list of polygons.
+
+    Attributes
+    ----------
+    area
+    polygons
+
+    Examples
+    --------
+    >>> MultiPolygon(
+    ...     [
+    ...         [
+    ...             [(0,0), (0,1), (1,1), (0,0)]
+    ...         ],
+    ...         [
+    ...             [(0,0), (0,1), (1,1), (0,0)],
+    ...             [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],
+    ...             [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],
+    ...         ],
+    ...     ]
+    ... )
+    """
+
+    def __init__(
+        self,
+        value: typing.List[
+            typing.List[typing.List[typing.Tuple[float, float]]]
+        ],
+    ):
+        super().__init__(value=value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        if not isinstance(value, list):
+            raise TypeError(
+                f"Expected type 'List[List[List[Tuple[float, float]]]]' received type '{type(value).__name__}'"
+            )
+        for poly in value:
+            Polygon.__validate__(poly)
+
+    @classmethod
+    def decode_value(
+        cls,
+        value: typing.Optional[
+            typing.List[typing.List[typing.List[typing.List[float]]]]
+        ],
+    ):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return cls(
+            [
+                [
+                    [(point[0], point[1]) for point in subpolygon]
+                    for subpolygon in polygon
+                ]
+                for polygon in value
+            ]
+        )
+
+    @property
+    def area(self):
+        """
+        Symbolic representation of area.
+        """
+        if not isinstance(self._value, Symbol):
+            raise ValueError(
+                "attribute 'area' is reserved for symbolic variables."
+            )
+        return Float.symbolic(
+            name=self._value._name,
+            key=self._value._key,
+            attribute="area",
+        )
+
+    def to_polygons(self) -> typing.List[Polygon]:
+        """
+        Converts multipolygon to a list of Polygon instances.
+
+        Returns
+        -------
+        List[Polygon]
+        """
+        return [Polygon(poly) for poly in self.get_value()]
+
+    @classmethod
+    def from_polygons(self, polygons: typing.List[Polygon]) -> "MultiPolygon":
+        """
+        Converts a list of Polygon instances to a MultiPolygon.
+
+        Parameters
+        ----------
+        polygons : List[Polygon]
+            A list of Polygon instances.
+
+        Returns
+        -------
+        MultiPolygon
+            A MultiPolygon instance.
+        """
+        return MultiPolygon([poly.get_value() for poly in polygons])
 

 
 

@@ -852,9 +852,7 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
1283
-1284
-1285
+
1285
 1286
 1287
 1288
@@ -870,25 +868,27 @@ 

 1298
 1299
 1300
-1301
@classmethod
-def decode_value(
-    cls,
-    value: typing.Optional[
-        typing.List[typing.List[typing.List[typing.List[float]]]]
-    ],
-):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return cls(
-        [
-            [
-                [(point[0], point[1]) for point in subpolygon]
-                for subpolygon in polygon
-            ]
-            for polygon in value
-        ]
-    )
+1301
+1302
+1303
@classmethod
+def decode_value(
+    cls,
+    value: typing.Optional[
+        typing.List[typing.List[typing.List[typing.List[float]]]]
+    ],
+):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return cls(
+        [
+            [
+                [(point[0], point[1]) for point in subpolygon]
+                for subpolygon in polygon
+            ]
+            for polygon in value
+        ]
+    )
 

 
 

@@ -952,9 +952,7 @@ 

 

 

 Source code in valor/schemas/symbolic/types.py
-
1328
-1329
-1330
+
1330
 1331
 1332
 1333
@@ -967,22 +965,24 @@ 

 1340
 1341
 1342
-1343
@classmethod
-def from_polygons(self, polygons: typing.List[Polygon]) -> "MultiPolygon":
-    """
-    Converts a list of Polygon instances to a MultiPolygon.
-
-    Parameters
-    ----------
-    polygons : List[Polygon]
-        A list of Polygon instances.
-
-    Returns
-    -------
-    MultiPolygon
-        A MultiPolygon instance.
-    """
-    return MultiPolygon([poly.get_value() for poly in polygons])
+1343
+1344
+1345
@classmethod
+def from_polygons(self, polygons: typing.List[Polygon]) -> "MultiPolygon":
+    """
+    Converts a list of Polygon instances to a MultiPolygon.
+
+    Parameters
+    ----------
+    polygons : List[Polygon]
+        A list of Polygon instances.
+
+    Returns
+    -------
+    MultiPolygon
+        A MultiPolygon instance.
+    """
+    return MultiPolygon([poly.get_value() for poly in polygons])
 

 
 

@@ -1015,23 +1015,23 @@ 

 

 

 Source code in valor/schemas/symbolic/types.py
-
1318
-1319
-1320
+
1320
 1321
 1322
 1323
 1324
 1325
-1326
def to_polygons(self) -> typing.List[Polygon]:
-    """
-    Converts multipolygon to a list of Polygon instances.
-
-    Returns
-    -------
-    List[Polygon]
-    """
-    return [Polygon(poly) for poly in self.get_value()]
+1326
+1327
+1328
def to_polygons(self) -> typing.List[Polygon]:
+    """
+    Converts multipolygon to a list of Polygon instances.
+
+    Returns
+    -------
+    List[Polygon]
+    """
+    return [Polygon(poly) for poly in self.get_value()]
 

 
 

diff --git a/client_api/Schemas/Spatial/Point/index.html b/client_api/Schemas/Spatial/Point/index.html
index 6d7a6bff6..d0ec16d88 100644
--- a/client_api/Schemas/Spatial/Point/index.html
+++ b/client_api/Schemas/Spatial/Point/index.html
@@ -586,7 +586,9 @@ 
Point

 816
 817
 818
-819
class Point(Spatial, Equatable):
+819
+820
+821
class Point(Spatial, Equatable):
     """
     Represents a point in 2D space.
 
@@ -610,54 +612,56 @@ 
Point

 
     @classmethod
     def __validate__(cls, value: typing.Any):
-        if not isinstance(value, tuple):
+        if not isinstance(value, (tuple, list)):
             raise TypeError(
                 f"Expected type 'typing.Tuple[float, float]' received type '{type(value).__name__}'"
             )
         elif len(value) != 2:
-            raise ValueError("")
-        for item in value:
-            if not isinstance(item, (int, float, np.floating)):
-                raise TypeError(
-                    f"Expected type '{float.__name__}' received type '{type(item).__name__}'"
-                )
-
-    @classmethod
-    def decode_value(cls, value: typing.Optional[typing.List[float]]):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return cls((value[0], value[1]))
-
-    def encode_value(self) -> typing.Any:
-        """Encode object to JSON compatible dictionary."""
-        value = self.get_value()
-        if value is None:
-            return None
-        return (float(value[0]), float(value[1]))
-
-    def tuple(self):
-        return self.get_value()
-
-    def resize(
-        self,
-        og_img_h=10,
-        og_img_w=10,
-        new_img_h=100,
-        new_img_w=100,
-    ):
-        value = self.get_value()
-        h_ratio = new_img_h / og_img_h
-        w_ratio = new_img_w / og_img_w
-        return Point((value[0] * h_ratio, value[1] * w_ratio))
-
-    @property
-    def x(self):
-        return self.get_value()[0]
-
-    @property
-    def y(self):
-        return self.get_value()[1]
+            raise ValueError(
+                "A point should contain only two x-y coordinates."
+            )
+        for item in value:
+            if not isinstance(item, (int, float, np.floating)):
+                raise TypeError(
+                    f"Expected type '{float.__name__}' received type '{type(item).__name__}'"
+                )
+
+    @classmethod
+    def decode_value(cls, value: typing.Optional[typing.List[float]]):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return cls((value[0], value[1]))
+
+    def encode_value(self) -> typing.Any:
+        """Encode object to JSON compatible dictionary."""
+        value = self.get_value()
+        if value is None:
+            return None
+        return (float(value[0]), float(value[1]))
+
+    def tuple(self):
+        return self.get_value()
+
+    def resize(
+        self,
+        og_img_h=10,
+        og_img_w=10,
+        new_img_h=100,
+        new_img_w=100,
+    ):
+        value = self.get_value()
+        h_ratio = new_img_h / og_img_h
+        w_ratio = new_img_w / og_img_w
+        return Point((value[0] * h_ratio, value[1] * w_ratio))
+
+    @property
+    def x(self):
+        return self.get_value()[0]
+
+    @property
+    def y(self):
+        return self.get_value()[1]
 

 

 

@@ -673,17 +677,17 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
784
-785
-786
+
786
 787
 788
-789
@classmethod
-def decode_value(cls, value: typing.Optional[typing.List[float]]):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return cls((value[0], value[1]))
+789
+790
+791
@classmethod
+def decode_value(cls, value: typing.Optional[typing.List[float]]):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return cls((value[0], value[1]))
 

 
 

@@ -696,17 +700,17 @@ 

 
Encode object to JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
791
-792
-793
+
793
 794
 795
-796
def encode_value(self) -> typing.Any:
-    """Encode object to JSON compatible dictionary."""
-    value = self.get_value()
-    if value is None:
-        return None
-    return (float(value[0]), float(value[1]))
+796
+797
+798
def encode_value(self) -> typing.Any:
+    """Encode object to JSON compatible dictionary."""
+    value = self.get_value()
+    if value is None:
+        return None
+    return (float(value[0]), float(value[1]))
 

 
 

diff --git a/client_api/Schemas/Spatial/Polygon/index.html b/client_api/Schemas/Spatial/Polygon/index.html
index 5b685690f..792338edd 100644
--- a/client_api/Schemas/Spatial/Polygon/index.html
+++ b/client_api/Schemas/Spatial/Polygon/index.html
@@ -713,9 +713,7 @@ 
Polygon

 
 

 Source code in valor/schemas/symbolic/types.py
-
 962
- 963
- 964
+
 964
  965
  966
  967
@@ -863,157 +861,159 @@ 
Polygon

 1109
 1110
 1111
-1112
class Polygon(Spatial):
-    """
-    Represents a polygon with a boundary and optional holes.
-
-    Follows the GeoJSON specification (RFC 7946).
+1112
+1113
+1114
class Polygon(Spatial):
+    """
+    Represents a polygon with a boundary and optional holes.
 
-    Parameters
-    ----------
-    value : List[List[Tuple[float, float]]], optional
-        A polygon.
-
-    Attributes
-    ----------
-    area
-    boundary
-    holes
-    xmin
-    xmax
-    ymin
-    ymax
-
-    Examples
-    --------
-    Create a polygon without any holes.
-    >>> Polygon([[(0,0), (0,1), (1,1), (0,0)]])
-
-    Create a polygon with 2 holes.
-    >>> Polygon(
-    ...     [
-    ...         [(0,0), (0,1), (1,1), (0,0)],
-    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],
-    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],
-    ...     ]
-    ... )
-    """
-
-    def __init__(
-        self, value: typing.List[typing.List[typing.Tuple[float, float]]]
-    ):
-        super().__init__(value=value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        MultiLineString.__validate__(value)
-        for line in value:
-            if not (len(line) >= 4 and line[0] == line[-1]):
-                raise ValueError(
-                    "Polygons are defined by at least 4 points with the first point being repeated at the end."
-                )
-
-    @classmethod
-    def decode_value(
-        cls,
-        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
-    ):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        return cls(
-            [
-                [(point[0], point[1]) for point in subpolygon]
-                for subpolygon in value
-            ]
-        )
-
-    @property
-    def area(self) -> Float:
-        """
-        Symbolic representation of area.
-        """
-        if not isinstance(self._value, Symbol):
-            raise ValueError
-        return Float.symbolic(
-            name=self._value._name,
-            key=self._value._key,
-            attribute="area",
-        )
-
-    @property
-    def boundary(self) -> typing.List[typing.Tuple[float, float]]:
-        """
-        The boundary of the polygon.
-
-        Returns
-        -------
-        List[Tuple(float, float)]
-            A list of points.
-        """
-        value = self.get_value()
-        if value is None:
-            raise ValueError("Polygon is 'None'")
-        return value[0]
-
-    @property
-    def holes(self) -> typing.List[typing.List[typing.Tuple[float, float]]]:
-        """
-        typing.Any holes in the polygon.
-
-        Returns
-        -------
-        List[List[Tuple(float, float)]]
-            A list of holes.
-        """
-        value = self.get_value()
-        if value is None:
-            raise ValueError("Polygon is 'None'")
-        return value[1:]
-
-    @property
-    def xmin(self) -> float:
-        """
-        Minimum x-value.
-
-        Returns
-        -------
-        float
-        """
-        return min([p[0] for p in self.boundary])
-
-    @property
-    def xmax(self) -> float:
-        """
-        Maximum x-value.
-
-        Returns
-        -------
-        float
-        """
-        return max([p[0] for p in self.boundary])
-
-    @property
-    def ymin(self) -> float:
-        """
-        Minimum y-value.
-
-        Returns
-        -------
-        float
-        """
-        return min([p[1] for p in self.boundary])
-
-    @property
-    def ymax(self) -> float:
-        """
-        Maximum y-value.
-
-        Returns
-        -------
-        float
-        """
-        return max([p[1] for p in self.boundary])
+    Follows the GeoJSON specification (RFC 7946).
+
+    Parameters
+    ----------
+    value : List[List[Tuple[float, float]]], optional
+        A polygon.
+
+    Attributes
+    ----------
+    area
+    boundary
+    holes
+    xmin
+    xmax
+    ymin
+    ymax
+
+    Examples
+    --------
+    Create a polygon without any holes.
+    >>> Polygon([[(0,0), (0,1), (1,1), (0,0)]])
+
+    Create a polygon with 2 holes.
+    >>> Polygon(
+    ...     [
+    ...         [(0,0), (0,1), (1,1), (0,0)],
+    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],
+    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],
+    ...     ]
+    ... )
+    """
+
+    def __init__(
+        self, value: typing.List[typing.List[typing.Tuple[float, float]]]
+    ):
+        super().__init__(value=value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        MultiLineString.__validate__(value)
+        for line in value:
+            if not (len(line) >= 4 and line[0] == line[-1]):
+                raise ValueError(
+                    "Polygons are defined by at least 4 points with the first point being repeated at the end."
+                )
+
+    @classmethod
+    def decode_value(
+        cls,
+        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
+    ):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        return cls(
+            [
+                [(point[0], point[1]) for point in subpolygon]
+                for subpolygon in value
+            ]
+        )
+
+    @property
+    def area(self) -> Float:
+        """
+        Symbolic representation of area.
+        """
+        if not isinstance(self._value, Symbol):
+            raise ValueError
+        return Float.symbolic(
+            name=self._value._name,
+            key=self._value._key,
+            attribute="area",
+        )
+
+    @property
+    def boundary(self) -> typing.List[typing.Tuple[float, float]]:
+        """
+        The boundary of the polygon.
+
+        Returns
+        -------
+        List[Tuple(float, float)]
+            A list of points.
+        """
+        value = self.get_value()
+        if value is None:
+            raise ValueError("Polygon is 'None'")
+        return value[0]
+
+    @property
+    def holes(self) -> typing.List[typing.List[typing.Tuple[float, float]]]:
+        """
+        typing.Any holes in the polygon.
+
+        Returns
+        -------
+        List[List[Tuple(float, float)]]
+            A list of holes.
+        """
+        value = self.get_value()
+        if value is None:
+            raise ValueError("Polygon is 'None'")
+        return value[1:]
+
+    @property
+    def xmin(self) -> float:
+        """
+        Minimum x-value.
+
+        Returns
+        -------
+        float
+        """
+        return min([p[0] for p in self.boundary])
+
+    @property
+    def xmax(self) -> float:
+        """
+        Maximum x-value.
+
+        Returns
+        -------
+        float
+        """
+        return max([p[0] for p in self.boundary])
+
+    @property
+    def ymin(self) -> float:
+        """
+        Minimum y-value.
+
+        Returns
+        -------
+        float
+        """
+        return min([p[1] for p in self.boundary])
+
+    @property
+    def ymax(self) -> float:
+        """
+        Maximum y-value.
+
+        Returns
+        -------
+        float
+        """
+        return max([p[1] for p in self.boundary])
 

 
 

@@ -1229,9 +1229,7 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
1012
-1013
-1014
+
1014
 1015
 1016
 1017
@@ -1242,20 +1240,22 @@ 

 1022
 1023
 1024
-1025
@classmethod
-def decode_value(
-    cls,
-    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
-):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    return cls(
-        [
-            [(point[0], point[1]) for point in subpolygon]
-            for subpolygon in value
-        ]
-    )
+1025
+1026
+1027
@classmethod
+def decode_value(
+    cls,
+    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],
+):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    return cls(
+        [
+            [(point[0], point[1]) for point in subpolygon]
+            for subpolygon in value
+        ]
+    )
 

 
 

diff --git a/client_api/Schemas/Spatial/Raster/index.html b/client_api/Schemas/Spatial/Raster/index.html
index e2369ecd6..f3062e614 100644
--- a/client_api/Schemas/Spatial/Raster/index.html
+++ b/client_api/Schemas/Spatial/Raster/index.html
@@ -754,9 +754,7 @@ 
Raster

 
 

 Source code in valor/schemas/symbolic/types.py
-
1769
-1770
-1771
+
1771
 1772
 1773
 1774
@@ -983,236 +981,238 @@ 
Raster

 1995
 1996
 1997
-1998
class Raster(Spatial):
-    """
-    Represents a binary mask.
-
-    Parameters
-    ----------
-    value : Dict[str, typing.Union[np.ndarray, str, None]], optional
-        An raster value.
-
-    Attributes
-    ----------
-    area
-    array
-    geometry
-    height
-    width
-
-    Raises
-    ------
-    TypeError
-        If `encoding` is not a string.
-
-    Examples
-    --------
-    Generate a random mask.
-    >>> import numpy.random
-    >>> height = 640
-    >>> width = 480
-    >>> array = numpy.random.rand(height, width)
-
-    Convert to binary mask.
-    >>> mask = (array > 0.5)
-
-    Create Raster.
-    >>> Raster.from_numpy(mask)
-    """
-
-    def __init__(
-        self,
-        value: typing.Dict[
-            str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]
-        ],
-    ):
-        """
-        Initialize and instance of a raster.
-
-        Parameters
-        ----------
-        value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]
-            The raster in dictionary format {"mask": <np.ndarray>, "geometry": <geometry | None>}.
-        """
-        super().__init__(value)
-
-    @classmethod
-    def __validate__(cls, value: typing.Any):
-        """
-        Validates typing.
-
-        Parameters
-        ----------
-        value : Any
-            The value to validate.
-
-        Raises
-        ------
-        TypeError
-            If the value type is not supported.
-        """
-        if not isinstance(value, dict):
-            raise TypeError(
-                "Raster should contain a dictionary describing a mask and optionally a geometry."
-            )
-        elif set(value.keys()) != {"mask", "geometry"}:
-            raise ValueError(
-                "Raster should be described by a dictionary with keys 'mask' and 'geometry'"
-            )
-        elif not isinstance(value["mask"], np.ndarray):
-            raise TypeError(
-                f"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'"
-            )
-        elif len(value["mask"].shape) != 2:
-            raise ValueError("raster only supports 2d arrays")
-        elif value["mask"].dtype != bool:
-            raise ValueError(
-                f"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}"
-            )
-        elif (
-            value["geometry"] is not None
-            and not Polygon.supports(value["geometry"])
-            and not MultiPolygon.supports(value["geometry"])
-        ):
-            raise TypeError(
-                "Expected geometry to conform to either Polygon or MultiPolygon or be 'None'"
-            )
-
-    def encode_value(self) -> typing.Any:
-        """Encode object to JSON compatible dictionary."""
-        value = self.get_value()
-        if value is None:
-            return None
-        f = io.BytesIO()
-        PIL.Image.fromarray(value["mask"]).save(f, format="PNG")
-        f.seek(0)
-        mask_bytes = f.read()
-        f.close()
-        return {
-            "mask": b64encode(mask_bytes).decode(),
-            "geometry": value["geometry"],
-        }
-
-    @classmethod
-    def decode_value(cls, value: typing.Any):
-        """Decode object from JSON compatible dictionary."""
-        if value is None:
-            return None
-        if not (
-            isinstance(value, dict)
-            and set(value.keys()) == {"mask", "geometry"}
-        ):
-            raise ValueError(
-                f"Improperly formatted raster encoding. Received '{value}'"
-            )
-        mask_bytes = b64decode(value["mask"])
-        with io.BytesIO(mask_bytes) as f:
-            img = PIL.Image.open(f)
-            value = {
-                "mask": np.array(img),
-                "geometry": value["geometry"],
-            }
-        return cls(value=value)
-
-    @classmethod
-    def from_numpy(cls, mask: np.ndarray):
-        """
-        Create a Raster object from a NumPy array.
-
-        Parameters
-        ----------
-        mask : np.ndarray
-            The 2D binary array representing the mask.
-
-        Returns
-        -------
-        Raster
-
-        Raises
-        ------
-        ValueError
-            If the input array is not 2D or not of dtype bool.
-        """
-        return cls(value={"mask": mask, "geometry": None})
-
-    @classmethod
-    def from_geometry(
-        cls,
-        geometry: typing.Union[Box, Polygon, MultiPolygon],
-        height: int,
-        width: int,
-    ):
-        """
-        Create a Raster object from a geometric mask.
-
-        Parameters
-        ----------
-        geometry : Union[Box, Polygon, MultiPolygon]
-            Defines the bitmask as a geometry. Overrides any existing mask.
-        height : int
-            The intended height of the binary mask.
-        width : int
-            The intended width of the binary mask.
-
-        Returns
-        -------
-        Raster
-        """
-        bitmask = np.full((int(height), int(width)), False)
-        return cls(value={"mask": bitmask, "geometry": geometry.get_value()})
-
-    @property
-    def area(self) -> Float:
-        """
-        Symbolic representation of area.
-        """
-        if not isinstance(self._value, Symbol):
-            raise ValueError
-        return Float.symbolic(
-            name=self._value._name,
-            key=self._value._key,
-            attribute="area",
-        )
-
-    @property
-    def array(self) -> np.ndarray:
-        """
-        The bitmask as a numpy array.
-
-        Returns
-        -------
-        Optional[np.ndarray]
-            A 2D binary array representing the mask if it exists.
-        """
-        value = self.get_value()
-        if value["geometry"] is not None:
-            warnings.warn(
-                "Raster array does not contain bitmask as this is a geometry-defined raster.",
-                RuntimeWarning,
-            )
-        return value["mask"]
-
-    @property
-    def geometry(self) -> typing.Union[Box, Polygon, MultiPolygon]:
-        """
-        The geometric mask if it exists.
-
-        Returns
-        -------
-        Box | Polygon | MultiPolygon | None
-            The geometry if it exists.
-        """
-        return self.get_value()["geometry"]
-
-    @property
-    def height(self) -> int:
-        """Returns the height of the raster if it exists."""
-        return self.array.shape[0]
-
-    @property
-    def width(self) -> int:
-        """Returns the width of the raster if it exists."""
-        return self.array.shape[1]
+1998
+1999
+2000
class Raster(Spatial):
+    """
+    Represents a binary mask.
+
+    Parameters
+    ----------
+    value : Dict[str, typing.Union[np.ndarray, str, None]], optional
+        An raster value.
+
+    Attributes
+    ----------
+    area
+    array
+    geometry
+    height
+    width
+
+    Raises
+    ------
+    TypeError
+        If `encoding` is not a string.
+
+    Examples
+    --------
+    Generate a random mask.
+    >>> import numpy.random
+    >>> height = 640
+    >>> width = 480
+    >>> array = numpy.random.rand(height, width)
+
+    Convert to binary mask.
+    >>> mask = (array > 0.5)
+
+    Create Raster.
+    >>> Raster.from_numpy(mask)
+    """
+
+    def __init__(
+        self,
+        value: typing.Dict[
+            str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]
+        ],
+    ):
+        """
+        Initialize and instance of a raster.
+
+        Parameters
+        ----------
+        value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]
+            The raster in dictionary format {"mask": <np.ndarray>, "geometry": <geometry | None>}.
+        """
+        super().__init__(value)
+
+    @classmethod
+    def __validate__(cls, value: typing.Any):
+        """
+        Validates typing.
+
+        Parameters
+        ----------
+        value : Any
+            The value to validate.
+
+        Raises
+        ------
+        TypeError
+            If the value type is not supported.
+        """
+        if not isinstance(value, dict):
+            raise TypeError(
+                "Raster should contain a dictionary describing a mask and optionally a geometry."
+            )
+        elif set(value.keys()) != {"mask", "geometry"}:
+            raise ValueError(
+                "Raster should be described by a dictionary with keys 'mask' and 'geometry'"
+            )
+        elif not isinstance(value["mask"], np.ndarray):
+            raise TypeError(
+                f"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'"
+            )
+        elif len(value["mask"].shape) != 2:
+            raise ValueError("raster only supports 2d arrays")
+        elif value["mask"].dtype != bool:
+            raise ValueError(
+                f"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}"
+            )
+        elif (
+            value["geometry"] is not None
+            and not Polygon.supports(value["geometry"])
+            and not MultiPolygon.supports(value["geometry"])
+        ):
+            raise TypeError(
+                "Expected geometry to conform to either Polygon or MultiPolygon or be 'None'"
+            )
+
+    def encode_value(self) -> typing.Any:
+        """Encode object to JSON compatible dictionary."""
+        value = self.get_value()
+        if value is None:
+            return None
+        f = io.BytesIO()
+        PIL.Image.fromarray(value["mask"]).save(f, format="PNG")
+        f.seek(0)
+        mask_bytes = f.read()
+        f.close()
+        return {
+            "mask": b64encode(mask_bytes).decode(),
+            "geometry": value["geometry"],
+        }
+
+    @classmethod
+    def decode_value(cls, value: typing.Any):
+        """Decode object from JSON compatible dictionary."""
+        if value is None:
+            return None
+        if not (
+            isinstance(value, dict)
+            and set(value.keys()) == {"mask", "geometry"}
+        ):
+            raise ValueError(
+                f"Improperly formatted raster encoding. Received '{value}'"
+            )
+        mask_bytes = b64decode(value["mask"])
+        with io.BytesIO(mask_bytes) as f:
+            img = PIL.Image.open(f)
+            value = {
+                "mask": np.array(img),
+                "geometry": value["geometry"],
+            }
+        return cls(value=value)
+
+    @classmethod
+    def from_numpy(cls, mask: np.ndarray):
+        """
+        Create a Raster object from a NumPy array.
+
+        Parameters
+        ----------
+        mask : np.ndarray
+            The 2D binary array representing the mask.
+
+        Returns
+        -------
+        Raster
+
+        Raises
+        ------
+        ValueError
+            If the input array is not 2D or not of dtype bool.
+        """
+        return cls(value={"mask": mask, "geometry": None})
+
+    @classmethod
+    def from_geometry(
+        cls,
+        geometry: typing.Union[Box, Polygon, MultiPolygon],
+        height: int,
+        width: int,
+    ):
+        """
+        Create a Raster object from a geometric mask.
+
+        Parameters
+        ----------
+        geometry : Union[Box, Polygon, MultiPolygon]
+            Defines the bitmask as a geometry. Overrides any existing mask.
+        height : int
+            The intended height of the binary mask.
+        width : int
+            The intended width of the binary mask.
+
+        Returns
+        -------
+        Raster
+        """
+        bitmask = np.full((int(height), int(width)), False)
+        return cls(value={"mask": bitmask, "geometry": geometry.get_value()})
+
+    @property
+    def area(self) -> Float:
+        """
+        Symbolic representation of area.
+        """
+        if not isinstance(self._value, Symbol):
+            raise ValueError
+        return Float.symbolic(
+            name=self._value._name,
+            key=self._value._key,
+            attribute="area",
+        )
+
+    @property
+    def array(self) -> np.ndarray:
+        """
+        The bitmask as a numpy array.
+
+        Returns
+        -------
+        Optional[np.ndarray]
+            A 2D binary array representing the mask if it exists.
+        """
+        value = self.get_value()
+        if value["geometry"] is not None:
+            warnings.warn(
+                "Raster array does not contain bitmask as this is a geometry-defined raster.",
+                RuntimeWarning,
+            )
+        return value["mask"]
+
+    @property
+    def geometry(self) -> typing.Union[Box, Polygon, MultiPolygon]:
+        """
+        The geometric mask if it exists.
+
+        Returns
+        -------
+        Box | Polygon | MultiPolygon | None
+            The geometry if it exists.
+        """
+        return self.get_value()["geometry"]
+
+    @property
+    def height(self) -> int:
+        """Returns the height of the raster if it exists."""
+        return self.array.shape[0]
+
+    @property
+    def width(self) -> int:
+        """Returns the width of the raster if it exists."""
+        return self.array.shape[1]
 

 
 

@@ -1350,9 +1350,7 @@ 

 

 

 Source code in valor/schemas/symbolic/types.py
-
1806
-1807
-1808
+
1808
 1809
 1810
 1811
@@ -1364,21 +1362,23 @@ 

 1817
 1818
 1819
-1820
def __init__(
-    self,
-    value: typing.Dict[
-        str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]
-    ],
-):
-    """
-    Initialize and instance of a raster.
-
-    Parameters
-    ----------
-    value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]
-        The raster in dictionary format {"mask": <np.ndarray>, "geometry": <geometry | None>}.
-    """
-    super().__init__(value)
+1820
+1821
+1822
def __init__(
+    self,
+    value: typing.Dict[
+        str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]
+    ],
+):
+    """
+    Initialize and instance of a raster.
+
+    Parameters
+    ----------
+    value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]
+        The raster in dictionary format {"mask": <np.ndarray>, "geometry": <geometry | None>}.
+    """
+    super().__init__(value)
 

 
 

@@ -1442,9 +1442,7 @@ 

 

 

 Source code in valor/schemas/symbolic/types.py
-
1822
-1823
-1824
+
1824
 1825
 1826
 1827
@@ -1482,47 +1480,49 @@ 

 1859
 1860
 1861
-1862
@classmethod
-def __validate__(cls, value: typing.Any):
-    """
-    Validates typing.
-
-    Parameters
-    ----------
-    value : Any
-        The value to validate.
-
-    Raises
-    ------
-    TypeError
-        If the value type is not supported.
-    """
-    if not isinstance(value, dict):
-        raise TypeError(
-            "Raster should contain a dictionary describing a mask and optionally a geometry."
-        )
-    elif set(value.keys()) != {"mask", "geometry"}:
-        raise ValueError(
-            "Raster should be described by a dictionary with keys 'mask' and 'geometry'"
-        )
-    elif not isinstance(value["mask"], np.ndarray):
-        raise TypeError(
-            f"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'"
-        )
-    elif len(value["mask"].shape) != 2:
-        raise ValueError("raster only supports 2d arrays")
-    elif value["mask"].dtype != bool:
-        raise ValueError(
-            f"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}"
-        )
-    elif (
-        value["geometry"] is not None
-        and not Polygon.supports(value["geometry"])
-        and not MultiPolygon.supports(value["geometry"])
-    ):
-        raise TypeError(
-            "Expected geometry to conform to either Polygon or MultiPolygon or be 'None'"
-        )
+1862
+1863
+1864
@classmethod
+def __validate__(cls, value: typing.Any):
+    """
+    Validates typing.
+
+    Parameters
+    ----------
+    value : Any
+        The value to validate.
+
+    Raises
+    ------
+    TypeError
+        If the value type is not supported.
+    """
+    if not isinstance(value, dict):
+        raise TypeError(
+            "Raster should contain a dictionary describing a mask and optionally a geometry."
+        )
+    elif set(value.keys()) != {"mask", "geometry"}:
+        raise ValueError(
+            "Raster should be described by a dictionary with keys 'mask' and 'geometry'"
+        )
+    elif not isinstance(value["mask"], np.ndarray):
+        raise TypeError(
+            f"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'"
+        )
+    elif len(value["mask"].shape) != 2:
+        raise ValueError("raster only supports 2d arrays")
+    elif value["mask"].dtype != bool:
+        raise ValueError(
+            f"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}"
+        )
+    elif (
+        value["geometry"] is not None
+        and not Polygon.supports(value["geometry"])
+        and not MultiPolygon.supports(value["geometry"])
+    ):
+        raise TypeError(
+            "Expected geometry to conform to either Polygon or MultiPolygon or be 'None'"
+        )
 

 
 

@@ -1538,9 +1538,7 @@ 

 
Decode object from JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
1879
-1880
-1881
+
1881
 1882
 1883
 1884
@@ -1557,26 +1555,28 @@ 

 1895
 1896
 1897
-1898
@classmethod
-def decode_value(cls, value: typing.Any):
-    """Decode object from JSON compatible dictionary."""
-    if value is None:
-        return None
-    if not (
-        isinstance(value, dict)
-        and set(value.keys()) == {"mask", "geometry"}
-    ):
-        raise ValueError(
-            f"Improperly formatted raster encoding. Received '{value}'"
-        )
-    mask_bytes = b64decode(value["mask"])
-    with io.BytesIO(mask_bytes) as f:
-        img = PIL.Image.open(f)
-        value = {
-            "mask": np.array(img),
-            "geometry": value["geometry"],
-        }
-    return cls(value=value)
+1898
+1899
+1900
@classmethod
+def decode_value(cls, value: typing.Any):
+    """Decode object from JSON compatible dictionary."""
+    if value is None:
+        return None
+    if not (
+        isinstance(value, dict)
+        and set(value.keys()) == {"mask", "geometry"}
+    ):
+        raise ValueError(
+            f"Improperly formatted raster encoding. Received '{value}'"
+        )
+    mask_bytes = b64decode(value["mask"])
+    with io.BytesIO(mask_bytes) as f:
+        img = PIL.Image.open(f)
+        value = {
+            "mask": np.array(img),
+            "geometry": value["geometry"],
+        }
+    return cls(value=value)
 

 
 

@@ -1589,9 +1589,7 @@ 

 
Encode object to JSON compatible dictionary.

 

 Source code in valor/schemas/symbolic/types.py
-
1864
-1865
-1866
+
1866
 1867
 1868
 1869
@@ -1602,20 +1600,22 @@ 

 1874
 1875
 1876
-1877
def encode_value(self) -> typing.Any:
-    """Encode object to JSON compatible dictionary."""
-    value = self.get_value()
-    if value is None:
-        return None
-    f = io.BytesIO()
-    PIL.Image.fromarray(value["mask"]).save(f, format="PNG")
-    f.seek(0)
-    mask_bytes = f.read()
-    f.close()
-    return {
-        "mask": b64encode(mask_bytes).decode(),
-        "geometry": value["geometry"],
-    }
+1877
+1878
+1879
def encode_value(self) -> typing.Any:
+    """Encode object to JSON compatible dictionary."""
+    value = self.get_value()
+    if value is None:
+        return None
+    f = io.BytesIO()
+    PIL.Image.fromarray(value["mask"]).save(f, format="PNG")
+    f.seek(0)
+    mask_bytes = f.read()
+    f.close()
+    return {
+        "mask": b64encode(mask_bytes).decode(),
+        "geometry": value["geometry"],
+    }
 

 
 

@@ -1706,9 +1706,7 @@ 

 

 

 Source code in valor/schemas/symbolic/types.py
-
1921
-1922
-1923
+
1923
 1924
 1925
 1926
@@ -1730,31 +1728,33 @@ 

 1942
 1943
 1944
-1945
@classmethod
-def from_geometry(
-    cls,
-    geometry: typing.Union[Box, Polygon, MultiPolygon],
-    height: int,
-    width: int,
-):
-    """
-    Create a Raster object from a geometric mask.
-
-    Parameters
-    ----------
-    geometry : Union[Box, Polygon, MultiPolygon]
-        Defines the bitmask as a geometry. Overrides any existing mask.
-    height : int
-        The intended height of the binary mask.
-    width : int
-        The intended width of the binary mask.
-
-    Returns
-    -------
-    Raster
-    """
-    bitmask = np.full((int(height), int(width)), False)
-    return cls(value={"mask": bitmask, "geometry": geometry.get_value()})
+1945
+1946
+1947
@classmethod
+def from_geometry(
+    cls,
+    geometry: typing.Union[Box, Polygon, MultiPolygon],
+    height: int,
+    width: int,
+):
+    """
+    Create a Raster object from a geometric mask.
+
+    Parameters
+    ----------
+    geometry : Union[Box, Polygon, MultiPolygon]
+        Defines the bitmask as a geometry. Overrides any existing mask.
+    height : int
+        The intended height of the binary mask.
+    width : int
+        The intended width of the binary mask.
+
+    Returns
+    -------
+    Raster
+    """
+    bitmask = np.full((int(height), int(width)), False)
+    return cls(value={"mask": bitmask, "geometry": geometry.get_value()})
 

 
 

@@ -1838,9 +1838,7 @@ 

 

 

 Source code in valor/schemas/symbolic/types.py
-
1900
-1901
-1902
+
1902
 1903
 1904
 1905
@@ -1857,26 +1855,28 @@ 

 1916
 1917
 1918
-1919
@classmethod
-def from_numpy(cls, mask: np.ndarray):
-    """
-    Create a Raster object from a NumPy array.
-
-    Parameters
-    ----------
-    mask : np.ndarray
-        The 2D binary array representing the mask.
-
-    Returns
-    -------
-    Raster
-
-    Raises
-    ------
-    ValueError
-        If the input array is not 2D or not of dtype bool.
-    """
-    return cls(value={"mask": mask, "geometry": None})
+1919
+1920
+1921
@classmethod
+def from_numpy(cls, mask: np.ndarray):
+    """
+    Create a Raster object from a NumPy array.
+
+    Parameters
+    ----------
+    mask : np.ndarray
+        The 2D binary array representing the mask.
+
+    Returns
+    -------
+    Raster
+
+    Raises
+    ------
+    ValueError
+        If the input array is not 2D or not of dtype bool.
+    """
+    return cls(value={"mask": mask, "geometry": None})
 

 
 

diff --git a/search/search_index.json b/search/search_index.json
index bfeff4f08..2746dc238 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Introduction","text":"
Valor is a centralized evaluation store that makes it easy to measure, explore, and rank model performance. Valor empowers data scientists and engineers to evaluate the performance of their machine learning pipelines and use those evaluations to make better modeling decisions in the future. To skip this textual introduction and dive right in, first go here for instructions to setup the Valor service, and then checkout the sample notebooks.
 
Valor is maintained by Striveworks, a cutting-edge machine learning operations (MLOps) company based out of Austin, Texas. We'd love to learn more about your interest in Valor and answer any questions you may have; please don't hesitate to reach out to us on Slack or GitHub.
 
These docs are organized as follows:
 
     
  • Overview (this page): Provides an overview of what Valor is, why it's important, and how it works.
    •  
  • Installation: Explains how to install Valor.
    •  
  • Getting Started Notebook: Details everything you need to get up and running with using Valor.
    •  
  • All Sample Notebooks: Collection of descriptive Jupyter notebooks giving examples of how to evaluate model performance using Valor.
    •  
  • Metadata and Filtering: Describes Valor's robust support for adding metadata to data, along with how to filter evaluations and Valor objects based on metadata and other attributes.
    •  
  • Metrics: Describes all of the metrics that you can calculate using Valor.
    •  
  • Endpoints: Documents Valor's various API endpoints.
    •  
  • Technical Concepts: Describes the technical concepts that underpin Valor.
    •  
  • Contributing and Development: Explains how you can build on and contribute to Valor.
    •  
  • Python Client API: Shares reference documentation for our Python client.
    •  
"},{"location":"#overview","title":"Overview","text":"
In this section, we'll explore what Valor is, why it's important, and provide a high-level description of how it works. This overview is also available in the following five-minute video:
"},{"location":"#use-cases-for-a-containerized-evaluation-store","title":"Use Cases for a Containerized Evaluation Store","text":"
As we've worked with dozens of data scientists and engineers on their MLOps pipelines, we have identified three important questions that an effective evaluation store could help them answer. First, they wanted to understand: \"Of the various models I tested for a given dataset, which one performs best?\". This is a very common and important use case\u2014and one that is often solved on a model-to-model basis in a local Jupyter notebook. This focus on bespoke implementations limits traceability and makes it difficult to create apples-to-apples comparisons between new model runs and prior model runs.
 
Second, our users wanted to understand: \"How does the performance of a particular model vary across datasets?\". We found that many practitioners use the same computer vision model (e.g., YOLOv8) for a variety of supervised learning tasks, and they needed a way to identify patterns where that particular model didn't meet expectations.
 
Finally, our users wanted to understand: \"How can I use my prior evaluations to pick the best model for a future ML pipeline?\". This last question requires the ability to filter previous evaluations on granular metadata (e.g., time of day, geospatial coordinates, etc.) in order to provide tailored recommendations regarding which model to pick in the future.
 
With these three use cases in mind, we set out to build a centralized evaluation store that we later named Valor.
"},{"location":"#introducing-valor","title":"Introducing Valor","text":"
Valor is a centralized evaluation store that makes it easy to measure, explore, and rank model performance. Our ultimate goal with Valor is to help data scientists and engineers pick the right ML model for their specific needs. To that end, we built Valor with three design principles in mind:
 
     
  • Valor works with any dataset or model: We believe Valor should be able to handle any supervised learning task that you want to throw at it. Just pass in your ground truth annotations and predictions, describe your learning task (i.e., object detection), and Valor will do the rest. (Note: At launch, Valor will only support classification and computer vision (i.e., image segmentation and object detection) tasks. We're confident this framework will abstract well to other supervised learning tasks and plan to support them in later releases).
    •  
  • Valor can handle any type of image, model, or dataset metadata you throw at it: Metadata is a critical component of any evaluation store as it enables the system to offer tailored model recommendations based on a user's specific needs. To that end, we built Valor to handle any metadata under the sun. Dates, geospatial coordinates, and even JSONs filled with configuration details are all on the table. This means you can slice and dice your evaluations any way you want: just pass in the right labels for your use case and define your filter (say a geographic bounding box), and you\u2019ll get back results for your specific needs.
    •  
  • Valor standardizes the evaluation process: The trickiest part of comparing two different model runs is avoiding apples-to-oranges comparisons. Valor helps you audit your metrics and avoid false comparisons by versioning your uploads, storing them in a centralized location, and ensuring that you only compare runs that used the exact same filters and metrics.
    •  
"},{"location":"#how-it-works-an-illustrative-example","title":"How It Works: An Illustrative Example","text":"
Let\u2019s walk through a quick example to bring Valor to life.
 
Say that you're interested in using computer vision models to detect forest fires around the world using satellite imagery. You've just been tasked with building a new ML pipeline to detect fires in an unfamiliar region of interest. How might you leverage your evaluation metrics from prior ML pipelines to understand which model will perform best for this particular use case?
 
 
To answer this question, we'll start by passing in three pieces of information from each of our prior modeling runs:
 
     
  • GroundTruths: First, we'll pass in human-annotated bounding boxes to tell Valor exactly where forest fires can be found across all of the satellite images used in prior runs.
    •  
  • Predictions: Next, we'll pass machine-generated predictions for each image (also in the form of bounding boxes) so that Valor can evaluate how well each model did at predicting forest fires.
    •  
  • Labels: Finally, we'll pass metadata to Valor describing each of our various images (e.g., the time of day the photo was taken, the geospatial coordinates of the forest in the photo, etc.). We'll use this metadata later on in order to identify the right model for our new use case.
    •  
 
Once we pass in these three ingredients, Valor will compare all of our GroundTruths and Predictions in order to calculate various evaluation metrics (i.e., mean average precision or mAP). These metrics, Labels, GroundTruths, and Predictions, will all be stored in Postgres, with PostGIS support for fast geospatial lookups and geometric comparisons at a later date.
 
Finally, once all of our previous pipeline runs and evaluations are stored in Valor, we can use Valor\u2019s API to specify our exact filter criteria and get back its model rankings. In this case, we can ask Valor to find us the best model for detecting forest fires at night in a 50 mile radius around (42.36, -71.03), sorted by mAP. Valor will then filter all of our stored evaluation metrics, rank each model with evaluations that meet our criteria, and send back all relevant evaluation metrics to help us determine which model to use for our new modeling pipeline.
 
"},{"location":"#next-steps","title":"Next Steps","text":"
We'd recommend reviewing our \"Getting Started\" sample notebook to become further acquainted with Valor. For more detailed explanations of Valor's technical underpinnings, see our technical concepts guide.
"},{"location":"#faq","title":"FAQ","text":"
Q. What is Valor?
 
A. Valor is a centralized evaluation store that makes it easy to measure, explore, and rank model performance. For an overview of what Valor is and why it's important, please refer to our high-level overview.
 
Q. What evaluation methods are supported?
 
A. Valor currently supports generic classification as well as object-detection and semantic-segmentation for images. The long-term goal for Valor is to support the most popular supervised learning methods.
 
Q. Does Valor store data?
 
A. Valor only stores ground truth annotations, model predictions, and user-defined metadata.
 
Q. What is a Datum?
 
A. A valor.Datum object is a generic type that represents a datum in the context of a machine learning workflow. The object stores a UID and related metadata in a dictionary. This metadata allows for the user to construct their own abstraction layer by mapping a real-world type (e.g., an image) into a valor.Datum type.
 
from valor.metatypes import ImageMetadata\nimage = ImageMetadata.create(\n  uid = \"1234\",\n  height = 100,\n  width = 100,\n)\n\n# access the datum\ndatum = image.datum\n
 
Q. What is a GroundTruth?
 
A. valor.GroundTruth objects in Valor each represent a singular datum and its associated annotations that provide a reference standard or the 'truth' against which predictions are compared. There cannot be multiple ground truths per datum.
 
Q. What is a Prediction?
 
A. valor.Prediction objects are similar to valor.GroundTruth objects in that they also contain a list of annotations over a datum. However, these annotations are generated by a model as inferences, and the object also includes the name of the model that was used for creating these inferences. There cannot be multiple predictions by the same model over a single datum.
 
Q. Can Valor handle multiple data types?
 
A. Valor abstracts data types through metadata. An example of this can be seen in valor.metatypes.ImageMetadata which describes the mapping of an image to a valor.Datum.
 
Q. Does Valor support geospatial queries?
 
A. Valor follows the GeoJSON specification (RFC 7946) in the implementation of Point, MulitPoint, LineString, MultiLineString, Polygon and MulitPolygon geometries. These objects are used to define annotations and facilitate the creation of geospatial metadata.
"},{"location":"#troubleshooting","title":"Troubleshooting","text":"
Q. Why am I getting NotFinalizedError when trying to run an evaluation?
 
A. Valor requires both dataset and model representations to be finalized before evaluation can take place. Finalization is crucial for auditability as it ensures that data finalized at a certain date is immutable.
 
Dataset finalization is accomplished through the valor.Dataset.finalize member function.
 
from valor import Client, Dataset\nclient = Client(...)\ndataset = Dataset(name=\"test_dataset\")\n...\ndataset.finalize()\n
 
Models are finalized automatically given two conditions.
 
     
  1. The working dataset is finalized.
    2.  
  2. There is a 1:1 mapping of predictions to ground truths.
    3.  
 
Models and their predictions can also be finalized prematurely using the valor.Model.finalize_inferences member function. This will generate empty predictions with task type enums.TaskType.SKIP to achieve the 1:1 ground truth mapping.
 
from valor import Client, Dataset, Model\nclient = Client(...)\ndataset = Dataset(name=\"test_dataset\")\nmodel = Model(name=\"test_model\")\n...\ndataset.finalize()\nmodel.finalize_inferences(dataset)\n
 
Q. Why am I getting GDAL driver errors?
 
A. For some computations (mostly involving rasters), Valor requires the PostGIS database to have all GDAL drivers enabled. The Valor back end attempts to enable these drivers, but it might not have permission depending on your specific setup. If you encounter this error, see here for ways to enable the drivers directly in the PostGIS instance.
"},{"location":"contributing/","title":"Contributing to Valor","text":"
We welcome all contributions, bug reports, bug fixes, documentation improvements, enhancements, and ideas aimed at improving Valor. This doc describes the high-level process for how to contribute to this repository. If you have any questions or comments about this process, please feel free to reach out to us on Slack.
"},{"location":"contributing/#on-github","title":"On GitHub","text":"
We use Git on GitHub to manage this repo, which means you will need to sign up for a free GitHub account to submit issues, ideas, and pull requests. We use Git for version control to allow contributors from all over the world to work together on this project.
 
If you are new to Git, these official resources can help bring you up to speed:
 
     
  • GitHub documentation for forking a repo
    •  
  • GitHub documentation for collaborating with pull requests
    •  
  • GitHub documentation for working with forks
    •  
"},{"location":"contributing/#contribution-workflow","title":"Contribution Workflow","text":"
Generally, the high-level workflow for contributing to this repo includes:
 
     
  1. Submitting an issue or enhancement request using the appropriate template on GitHub Issues.
    2.  
  2. Gathering feedback from devs and the broader community in your issue before starting to code.
    3.  
  3. Forking the Valor repo, making your proposed changes, and submitting a pull request (PR). When submitting a PR, please be sure to:
       
    1. Update the README.md and/or any relevant docstrings with details of your change.
      2.  
    2. Add tests where necessary.
      3.  
    3. Run pre-commit install on your local repo before your last commit to ensure your changes follow our formatting guidelines.
      4.  
    4. Double-check that your code passes all of the tests that are automated via GitHub Actions.
      5.  
    5. Ping us on Slack to ensure timely review.
      6.  
     
    4.  
  4. Working with repo maintainers to review and improve your PR before it is merged into the official repo.
    5.  
 
For questions or comments on this process, please reach out to us at any time on Slack.
"},{"location":"contributing/#development-tips-and-tricks","title":"Development Tips and Tricks","text":""},{"location":"contributing/#deploying-the-back-end-for-development","title":"Deploying the Back End for Development","text":""},{"location":"contributing/#docker-compose","title":"Docker Compose","text":"
The fastest way to test the API and Python client is via Docker Compose. Start by setting the environment variable POSTGRES_PASSWORD to your liking, and then start Docker and build the container:
 
export POSTGRES_PASSWORD=\"my_password\"\ndocker compose up\n
"},{"location":"contributing/#makefile-requires-docker","title":"Makefile (requires Docker)","text":"
Alternatively, you may want to run the API service from a terminal to enable faster debugging. To start the service, you can run:
 
pip install api # Install the API in your python environment\n\nexport POSTGRES_PASSWORD=password\nexport POSTGRES_HOST=localhost\nmake start-postgres-docker # Start the custom postgres service in Docker\nmake run-migrations # Instantiate the table schemas in Postgres\nmake start-server # Start the API service locally\n
"},{"location":"contributing/#setting-up-your-environment","title":"Setting Up Your Environment","text":"
Creating a Valor-specific Python environment at the start of development can help you avoid dependency and versioning issues later on. To start, we'd recommend activating a new Python environment:
 
# venv\npython3 -m venv .env-valor\nsource .env-valor/bin/activate\n\n# conda\nconda create --name valor python=3.11\nconda activate valor\n
 
Next, install pre-commit to ensure formatting consistency throughout your repo:
 
pip install pre-commit\npre-commit install\n
 
Finally, you're ready to install your client and API modules:
 
# Install the Client module\npython -m pip install -e client/.\n\n# Install the API module\npython -m pip install -e api/.\n
"},{"location":"contributing/#use-pgadmin-to-debug-postgis","title":"Use pgAdmin to Debug PostGIS","text":"
You can use the pgAdmin utility to debug your PostGIS tables as you code. Start by installing pgAdmin, and then select Object > Register > Server to connect to your PostGIS container. The default connection details are listed below for convenience:
 
- *Host name/address*: 0.0.0.0\n- *Port*: 5432\n- *Maintenance database*: postgres\n- *Username*: postgres\n
"},{"location":"contributing/#running-tests","title":"Running Tests","text":"
All of our tests are run automatically via GitHub Actions on every push, so it's important to double-check that your code passes all local tests before committing your code. All of the tests below require pytest:
 
pip install pytest\n
"},{"location":"contributing/#running-integration-tests","title":"Running integration tests","text":"
pytest integration_tests\n
"},{"location":"contributing/#running-back-end-unit-tests","title":"Running back end unit tests","text":"
pytest api/tests/unit-tests\n
"},{"location":"contributing/#running-back-end-functional-tests","title":"Running back end functional tests","text":"
Note: Functional tests require a running instance of PostgreSQL, which you can start using make start-postgres-docker.
 
POSTGRES_PASSWORD=password \\\nPOSTGRES_HOST=localhost \\\npytest api/tests/functional-tests/\n
"},{"location":"endpoints/","title":"Endpoints","text":""},{"location":"installation/","title":"Installation","text":"
Valor comprises two services: a back-end service (which consists of a REST API and a Postgres database with the PostGIS extension), and a Python client for interacting with the back-end service.
"},{"location":"installation/#setting-up-the-back-end-service","title":"Setting up the back-end service","text":""},{"location":"installation/#using-docker-compose","title":"Using Docker Compose","text":"
The easiest way to get up and running with Valor is to use Docker Compose with the docker-compose.yml file in the repository root:
 
git clone https://github.com/striveworks/valor\ncd valor\ndocker compose --env-file ./api/.env.testing up\n
 
This will set up the necessary environment variables, start both the API and database services, and run the database migration job. The endpoint localhost:8000/health should return {\"status\":\"ok\"} if all of Valor's services were started correctly.
 
Note: running Valor this way is not intended for production and scalable use and is only recommended for development and testing purposes.
"},{"location":"installation/#deploying-via-docker-and-a-hosted-database","title":"Deploying via Docker and a hosted database","text":"
For a more production-grade deployment, we publish the images ghcr.io/striveworks/valor/valor-service (used for the REST API) and ghcr.io/striveworks/valor/migrations (used for setting up the database and migrations). These can be paired with any Postgres database with the PostGIS extension.
 
The following environment variables are required for running these images:
 Variable Description Images that need it POSTGRES_HOST The host of the Postgres database valor-service, migrations POSTGRES_PORT The port of the Postgres database valor-service, migrations POSTGRES_DB The name of the Postgres database valor-service, migrations POSTGRES_USERNAME The user of the Postgres database valor-service, migrations POSTGRES_PASSWORD The password of the Postgres database valor-service, migrations POSTGRES_SSLMODE Sets the Postgres instance SSL mode (typically needs to be \"require\") migrations API_ROOT_PATH The root path of the API (if serving behind a proxy) valor-service 
Additionally, the Valor REST API has an optional single username/password/bearer token authentication. To enable this feature, the valor-service image requires the following environment variables:
 Variable Description VALOR_USERNAME The username to use VALOR_PASSWORD The password to use VALOR_SECRET_KEY A random, secret string used for signing JWT tokens"},{"location":"installation/#manual-deployment","title":"Manual deployment","text":"
If you would prefer to build your own image or if you want a debug console for the back-end, please see the deployment instructions in Contributing to Valor.
"},{"location":"installation/#setting-up-the-python-client","title":"Setting up the Python client","text":"
The Python client can be installed via pip:
 
pip install valor-client\n
"},{"location":"metadata_and_filtering/","title":"Metadata and Filtering","text":""},{"location":"metadata_and_filtering/#metadata","title":"Metadata","text":"
Valor offers rich support for attaching metadata to almost any object, which can then be used to filter, group, and organize objects in Valor.
 
The metadata types supported are:
 
     
  • simple data types (strings, numerics, boolean)
    •  
  • datetimes (via datetime.datetime, datetime.date, datetime.time, and datetime.timedelta in the Valor client)
    •  
  • geometries and geographies (via GeoJSON)
    •  
 
Metadata is added on object creation. For example, if you want to use metadata to organize models that come from training run checkpoints, this may look like:
 
run_name: str\nckpt: int\n\nModel.create(name=f\"{run_name}-ckpt{ckpt}\", metadata={\"run_name\": run_name, \"ckpt\": ckpt})\n
 
or if a datum has an associated datetime of capture, that can be added in the creation stage:
 
from datetime import datetime\n\nDatum(uid=fname, metadata={\"capture_day\": datetime.datetime(day=1, month=1, year=2021)})\n
"},{"location":"metadata_and_filtering/#filtering","title":"Filtering","text":"
Valor supports filtering objects based on metadata or other attributes (such as labels or bounding boxes). One of the most important use cases of filtering is to define a subset of a dataset to evaluate a model on.
"},{"location":"metadata_and_filtering/#filtering-by-metadata","title":"Filtering by metadata","text":"
For example, using the above example where capture_day was added as metadata, one way to test model drift could be to evaluate the model over different time periods. Such a workflow may look like:
 
import datetime\n\nimport valor\n\nmodel: valor.Model # classification model\ndset: valor.Dataset # dataset to evaluate on\n\n# compare performance on data captured before and after 2020\nd = datetime.datetime(day=5, month=10, year=2020)\neval1 = model.evaluate_classification(dset, filter_by=[Datum.metadata[\"capture_day\"] < d])\neval2 = model.evaluate_classification(dset, filter_by=[Datum.metadata[\"capture_day\"] > d])\n
"},{"location":"metadata_and_filtering/#filtering-by-geometric-attributes","title":"Filtering by geometric attributes","text":"
As an example for filtering by geometric attributes, consider evaluating an object detection model's performance on small objects, where we define small as being less than 500 square pixels in area. This can be achieved via:
 
import valor\n\nmodel: valor.Model # object detection model\ndset: valor.Dataset # dataset to evaluate on\n\ndset.evaluate_detection(dset, filter_by=[valor.Annotation.bounding_box.area < 500])\n
"},{"location":"metadata_and_filtering/#filtering-in-queries","title":"Filtering in queries","text":"
Filtering can also be used when querying for different objects. For example, taking the model section checkpoint example from above, we could query model checkpoints from a training run based on the checkpoint number greater than 100 by:
 
from valor import client\n\nrun_name: str # run name to query for\n\nclient.get_models([Model.metadata[\"run_name\"] == run_name, Model.metadata[\"ckpt\"] > 100])\n
"},{"location":"metrics/","title":"Metrics","text":"
Let's look at the various metrics you can calculate using Valor.
 
If we're missing an important metric for your particular use case, please write us a GitHub Issue ticket. We love hearing your suggestions.
"},{"location":"metrics/#classification-metrics","title":"Classification Metrics","text":"Name Description Equation Precision The number of true positives divided by the total number of positive predictions (i.e., the number of true positives plus the number of false positives). \\(\\dfrac{\\|TP\\|}{\\|TP\\|+\\|FP\\|}\\) Recall The number of true positives divided by the total count of the class of interest (i.e., the number of true positives plus the number of true negatives). \\(\\dfrac{\\|TP\\|}{\\|TP\\|+\\|FN\\|}\\) F1 A weighted average of precision and recall. \\(\\frac{2 * Precision * Recall}{Precision + Recall}\\) Accuracy The number of true predictions divided by the total number of predictions. \\(\\dfrac{\\|TP\\|+\\|TN\\|}{\\|TP\\|+\\|TN\\|+\\|FP\\|+\\|FN\\|}\\) ROC AUC The area under the Receiver Operating Characteristic (ROC) curve for the predictions generated by a given model. See ROCAUC methods. Precision-Recall Curves Outputs a nested dictionary containing the true positives, false positives, true negatives, false negatives, precision, recall, and F1 score for each (label key, label value, confidence threshold) combination. See precision-recall curve methods Detailed Precision-Recall Curves Similar to PrecisionRecallCurve, except this metric a) classifies false positives as hallucinations or misclassifications, b) classifies false negatives as misclassifications or missed_detections, and c) gives example datums for each observation, up to a maximum of pr_curve_max_examples. See detailed precision-recall curve methods"},{"location":"metrics/#object-detection-and-instance-segmentation-metrics","title":"Object Detection and Instance Segmentation Metrics**","text":"Name Description Equation Average Precision (AP) The weighted mean of precisions achieved at several different recall thresholds for a single Intersection over Union (IOU), grouped by class. See AP methods. AP Averaged Over IOUs The average of several AP metrics across IOU thresholds, grouped by class labels. \\(\\dfrac{1}{\\text{number of thresholds}} \\sum\\limits_{iou \\in thresholds} AP_{iou}\\) Mean Average Precision (mAP) The average of several AP metrics, grouped by label keys and IOU thresholds. \\(\\dfrac{1}{\\text{number of labels}} \\sum\\limits_{label \\in labels} AP_{c}\\) mAP Averaged Over IOUs The average of several mAP metrics grouped by label keys. \\(\\dfrac{1}{\\text{number of thresholds}} \\sum\\limits_{iou \\in thresholds} mAP_{iou}\\) Average Recall (AR) The average of several recall metrics across IOU thresholds, grouped by class labels. See AR methods. Mean Average Recall (mAR) The average of several AR metrics, grouped by label keys. \\(\\dfrac{1}{\\text{number of labels}} \\sum\\limits_{label \\in labels} AR_{class}\\) Precision-Recall Curves Outputs a nested dictionary containing the true positives, false positives, true negatives, false negatives, precision, recall, and F1 score for each (label key, label value, confidence threshold) combination. These curves are calculated using a default IOU threshold of 0.5; you can set your own threshold by passing a float between 0 and 1 to the pr_curve_iou_threshold parameter at evaluation time. See precision-recall curve methods Detailed Precision-Recall Curves Similar to PrecisionRecallCurve, except this metric a) classifies false positives as hallucinations or misclassifications, b) classifies false negatives as misclassifications or missed_detections, and c) gives example datums and bounding boxes for each observation, up to a maximum of pr_curve_max_examples. See detailed precision-recall curve methods 
**When calculating IOUs for object detection metrics, Valor handles the necessary conversion between different types of geometric annotations. For example, if your model prediction is a polygon and your groundtruth is a raster, then the raster will be converted to a polygon prior to calculating the IOU.
"},{"location":"metrics/#semantic-segmentation-metrics","title":"Semantic Segmentation Metrics","text":"Name Description Equation Intersection Over Union (IOU) A ratio between the groundtruth and predicted regions of an image, measured as a percentage, grouped by class. \\(\\dfrac{area( prediction \\cap groundtruth )}{area( prediction \\cup groundtruth )}\\) Mean IOU The average of IOU across labels, grouped by label key. \\(\\dfrac{1}{\\text{number of labels}} \\sum\\limits_{label \\in labels} IOU_{c}\\)"},{"location":"metrics/#text-generation-metrics","title":"Text Generation Metrics","text":"Name Description Equation Answer Relevance The number of statements in the answer that are relevant to the query, divided by the total number of statements in the answer See appendix for details. Coherence Rates the coherence of a textual summary relative to some source text using a score from 1 to 5, where 5 means \"This summary is extremely coherent based on the information provided in the source text\". See appendix for details. ROUGE A score between 0 and 1 indicating how often the words in the ground truth string appeared in the predicted string (i.e., measuring recall). See appendix for details. BLEU A score between 0 and 1 indicating how much the predicted string matches the ground truth string (i.e., measuring precision), with a penalty for brevity. See appendix for details."},{"location":"metrics/#appendix-metric-calculations","title":"Appendix: Metric Calculations","text":""},{"location":"metrics/#binary-roc-auc","title":"Binary ROC AUC","text":""},{"location":"metrics/#receiver-operating-characteristic-roc","title":"Receiver Operating Characteristic (ROC)","text":"
An ROC curve plots the True Positive Rate (TPR) vs. the False Positive Rate (FPR) at different confidence thresholds.
 
In Valor, we use the confidence scores sorted in decreasing order as our thresholds. Using these thresholds, we can calculate our TPR and FPR as follows:
"},{"location":"metrics/#determining-the-rate-of-correct-predictions","title":"Determining the Rate of Correct Predictions","text":"Element Description True Positive (TP) Prediction confidence score >= threshold and is correct. False Positive (FP) Prediction confidence score >= threshold and is incorrect. True Negative (TN) Prediction confidence score < threshold and is correct. False Negative (FN) Prediction confidence score < threshold and is incorrect. 
     
  •  
    \\(\\text{True Positive Rate (TPR)} = \\dfrac{|TP|}{|TP| + |FN|} = \\dfrac{|TP(threshold)|}{|TP(threshold)| + |FN(threshold)|}\\)
     
    •  
  •  
    \\(\\text{False Positive Rate (FPR)} = \\dfrac{|FP|}{|FP| + |TN|} = \\dfrac{|FP(threshold)|}{|FP(threshold)| + |TN(threshold)|}\\)
     
    •  
 
We now use the confidence scores, sorted in decreasing order, as our thresholds in order to generate points on a curve.
 
\\(Point(score) = (FPR(score), \\ TPR(score))\\)
"},{"location":"metrics/#area-under-the-roc-curve-roc-auc","title":"Area Under the ROC Curve (ROC AUC)","text":"
After calculating the ROC curve, we find the ROC AUC metric by approximating the integral using the trapezoidal rule formula.
 
\\(ROC AUC =  \\sum_{i=1}^{|scores|} \\frac{  \\lVert Point(score_{i-1}) - Point(score_i) \\rVert }{2}\\)
 
See Classification: ROC Curve and AUC for more information.
"},{"location":"metrics/#average-precision-ap","title":"Average Precision (AP)","text":"
For object detection and instance segmentation tasks, average precision is calculated from the intersection-over-union (IOU) of geometric predictions and ground truths.
"},{"location":"metrics/#multiclass-precision-and-recall","title":"Multiclass Precision and Recall","text":"
Tasks that predict geometries (such as object detection or instance segmentation) use the ratio intersection-over-union (IOU) to calculate precision and recall. IOU is the ratio of the intersecting area over the joint area spanned by the two geometries, and is defined in the following equation.
 
\\(Intersection \\ over \\ Union \\ (IOU) = \\dfrac{Area( prediction \\cap groundtruth )}{Area( prediction \\cup groundtruth )}\\)
 
Using different IOU thresholds, we can determine whether we count a pairing between a prediction and a ground truth pairing based on their overlap.
 Case Description True Positive (TP) Prediction-GroundTruth pair exists with IOU >= threshold. False Positive (FP) Prediction-GroundTruth pair exists with IOU < threshold. True Negative (TN) Unused in multi-class evaluation. False Negative (FN) No Prediction with a matching label exists for the GroundTruth. 
     
  •  
    \\(Precision = \\dfrac{|TP|}{|TP| + |FP|} = \\dfrac{\\text{Number of True Predictions}}{|\\text{Predictions}|}\\)
     
    •  
  •  
    \\(Recall = \\dfrac{|TP|}{|TP| + |FN|} = \\dfrac{\\text{Number of True Predictions}}{|\\text{Groundtruths}|}\\)
     
    •  
"},{"location":"metrics/#matching-ground-truths-with-predictions","title":"Matching Ground Truths with Predictions","text":"
To properly evaluate a detection, we must first find the best pairings of predictions to ground truths. We start by iterating over our predictions, ordering them by highest scores first. We pair each prediction with the ground truth that has the highest calculated IOU. Both the prediction and ground truth are now considered paired and removed from the pool of choices.
 
def rank_ious(\n    groundtruths: list,\n    predictions: list,\n) -> list[float]:\n    \"\"\"Ranks ious by unique pairings.\"\"\"\n\n    retval = []\n    groundtruths = set(groundtruths)\n    for prediction in sorted(predictions, key=lambda x : -x.score):\n        groundtruth = max(groundtruths, key=lambda x : calculate_iou(groundtruth, prediction))\n        groundtruths.remove(groundtruth)\n        retval.append(calculate_iou(groundtruth, prediction))\n
"},{"location":"metrics/#precision-recall-curve","title":"Precision-Recall Curve","text":"
We can now compute the precision-recall curve using our previously ranked IOU's. We do this by iterating through the ranked IOU's and creating points cumulatively using recall and precision.
 
def create_precision_recall_curve(\n    number_of_groundtruths: int,\n    ranked_ious: list[float],\n    threshold: float\n) -> list[tuple[float, float]]:\n    \"\"\"Creates the precision-recall curve from a list of IOU's and a threshold.\"\"\"\n\n    retval = []\n    count_tp = 0\n    for i in range(ranked_ious):\n        if ranked_ious[i] >= threshold:\n            count_tp += 1\n        precision = count_tp / (i + 1)\n        recall = count_tp / number_of_groundtruths\n        retval.append((recall, precision))\n
"},{"location":"metrics/#calculating-average-precision","title":"Calculating Average Precision","text":"
Average precision is defined as the area under the precision-recall curve.
 
We will use a 101-point interpolation of the curve to be consistent with the COCO evaluator. The intent behind interpolation is to reduce the fuzziness that results from ranking pairs.
 
\\(AP = \\frac{1}{101} \\sum\\limits_{r\\in\\{ 0, 0.01, \\ldots , 1 \\}}\\rho_{interp}(r)\\)
 
\\(\\rho_{interp} = \\underset{\\tilde{r}:\\tilde{r} \\ge r}{max \\ \\rho (\\tilde{r})}\\)
"},{"location":"metrics/#references","title":"References","text":"
     
  • MS COCO Detection Evaluation
    •  
  • The PASCAL Visual Object Classes (VOC) Challenge
    •  
  • Mean Average Precision (mAP) Using the COCO Evaluator
    •  
"},{"location":"metrics/#average-recall-ar","title":"Average Recall (AR)","text":"
To calculate Average Recall (AR), we:
 
     
  1. Find the count of true positives above specified IOU and confidence thresholds for all images containing a ground truth of a particular class.
    2.  
  2. Divide that count of true positives by the total number of ground truths to get the recall value per class and IOU threshold. Append that recall value to a list.
    3.  
  3. Repeat steps 1 & 2 for multiple IOU thresholds (e.g., [.5, .75])
    4.  
  4. Take the average of our list of recalls to arrive at the AR value per class.
    5.  
 
Note that this metric differs from COCO's calculation in two ways:
 
     
  • COCO averages across classes while calculating AR, while we calculate AR separately for each class. Our AR calculations matches the original FAIR definition of AR, while our mAR calculations match what COCO calls AR.
    •  
  • COCO calculates three different AR metrics (AR@1, AR@5, AR@100) by considering only the top 1/5/100 most confident predictions during the matching process. Valor, on the other hand, allows users to input a recall_score_threshold value that will prevent low-confidence predictions from being counted as true positives when calculating AR.
    •  
"},{"location":"metrics/#precision-recall-curves","title":"Precision-Recall Curves","text":"
Precision-recall curves offer insight into which confidence threshold you should pick for your production pipeline. The PrecisionRecallCurve metric includes the true positives, false positives, true negatives, false negatives, precision, recall, and F1 score for each (label key, label value, confidence threshold) combination. When using the Valor Python client, the output will be formatted as follows:
 
pr_evaluation = evaluate_detection(\n    data=dataset,\n)\nprint(pr_evaluation)\n\n[...,\n{\n    \"type\": \"PrecisionRecallCurve\",\n    \"parameters\": {\n        \"label_key\": \"class\", # The key of the label.\n        \"pr_curve_iou_threshold\": 0.5, # Note that this value will be None for classification tasks. For detection tasks, we use 0.5 as the default threshold, but allow users to pass an optional `pr_curve_iou_threshold` parameter in their evaluation call.\n    },\n    \"value\": {\n        \"cat\": { # The value of the label.\n            \"0.05\": { # The confidence score threshold, ranging from 0.05 to 0.95 in increments of 0.05.\n                \"fn\": 0,\n                \"fp\": 1,\n                \"tp\": 3,\n                \"recall\": 1,\n                \"precision\": 0.75,\n                \"f1_score\": .857,\n            },\n            ...\n        },\n    }\n}]\n
 
It's important to note that these curves are computed slightly differently from our other aggregate metrics above:
"},{"location":"metrics/#classification-tasks","title":"Classification Tasks","text":"
Valor calculates its aggregate precision, recall, and F1 metrics by matching the highest confidence prediction with each groundtruth. One issue with this approach is that we may throw away useful information in cases where prediction labels all have similarly strong confidence scores. For example: if our top two predictions for a given ground truth are {\u201clabel\u201d: cat, \u201cscore\u201d:.91} and {\u201clabel\u201d: dog, \u201cscore\u201d:.90}, then our aggregated precision and recall metrics would penalize the dog label even though its confidence score was nearly equal to the cat label.
 
We think the approach above makes sense when calculating aggregate precision and recall metrics, but, when calculating the PrecisionRecallCurve value for each label, we consider all ground truth-prediction matches in order to treat each label as its own, separate binary classification problem.
"},{"location":"metrics/#detection-tasks","title":"Detection Tasks","text":"
The PrecisionRecallCurve values differ from the precision-recall curves used to calculate Average Precision in two subtle ways:
 
     
  • The PrecisionRecallCurve values visualize how precision and recall change as confidence thresholds vary from 0.05 to 0.95 in increments of 0.05. In contrast, the precision-recall curves used to calculate Average Precision are non-uniform; they vary over the actual confidence scores for each ground truth-prediction match.
    •  
  • If your pipeline predicts a label on an image, but that label doesn't exist on any ground truths in that particular image, then the PrecisionRecallCurve values will consider that prediction to be a false positive, whereas the other detection metrics will ignore that particular prediction.
    •  
"},{"location":"metrics/#detailedprecisionrecallcurve","title":"DetailedPrecisionRecallCurve","text":"
Valor also includes a more detailed version of PrecisionRecallCurve which can be useful for debugging your model's false positives and false negatives. When calculating DetailedPrecisionCurve, Valor will classify false positives as either hallucinations or misclassifications and your false negatives as either missed_detections or misclassifications using the following logic:
"},{"location":"metrics/#classification-tasks_1","title":"Classification Tasks","text":"
     
  • A false positive occurs when there is a qualified prediction (with score >= score_threshold) with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect.
       
    • Example: if there's a photo with one groundtruth label on it (e.g., Label(key='animal', value='dog')), and we predicted another label value (e.g., Label(key='animal', value='cat')) on that datum, we'd say it's a misclassification since the key was correct but the value was not.
      •  
     
    •  
  • Similarly, a false negative occurs when there is a prediction with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect.
       
    • Stratifications of False Negatives:
         
      • misclassification: Occurs when a different label value passes the score threshold.
        •  
      • no_predictions: Occurs when no label passes the score threshold.
        •  
       
      •  
     
    •  
"},{"location":"metrics/#object-detection-tasks","title":"Object Detection Tasks","text":"
     
  • A false positive is a misclassification if the following conditions are met:
       
    1. There is a qualified prediction with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect
      2.  
    2. The qualified prediction and groundtruth have an IOU >= pr_curve_iou_threshold.
      3.  
     
    •  
  • A false positive that does not meet the misclassification criteria is considered to be a part of the hallucinations set.
    •  
  • A false negative is determined to be a misclassification if the following criteria are met:
       
    1. There is a qualified prediction with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect.
      2.  
    2. The qualified prediction and groundtruth have an IOU >= pr_curve_iou_threshold.
      3.  
     
    •  
  • For a false negative that does not meet this criteria, we consider it to have no_predictions.
    •  
  • Example: if there's a photo with one groundtruth label on it (e.g., Label(key='animal', value='dog')), and we predicted another bounding box directly over that same object (e.g., Label(key='animal', value='cat')), we'd say it's a misclassification.
    •  
 
The DetailedPrecisionRecallOutput also includes up to n examples of each type of error, where n is set using pr_curve_max_examples. An example output is as follows:
 
# To retrieve more detailed examples for each `fn`, `fp`, and `tp`, look at the `DetailedPrecisionRecallCurve` metric\ndetailed_evaluation = evaluate_detection(\n    data=dataset,\n    pr_curve_max_examples=1 # The maximum number of examples to return for each obseration type (e.g., hallucinations, misclassifications, etc.)\n    metrics_to_return=[..., 'DetailedPrecisionRecallCurve'] # DetailedPrecisionRecallCurve isn't returned by default; the user must ask for it explicitely\n)\nprint(detailed_evaluation)\n\n[...,\n{\n    \"type\": \"DetailedPrecisionRecallCurve\",\n    \"parameters\": {\n        \"label_key\": \"class\", # The key of the label.\n        \"pr_curve_iou_threshold\": 0.5,\n    },\n    \"value\": {\n        \"cat\": { # The value of the label.\n            \"0.05\": { # The confidence score threshold, ranging from 0.05 to 0.95 in increments of 0.05.\n                \"fp\": {\n                    \"total\": 1,\n                    \"observations\": {\n                        'hallucinations': {\n                            \"count\": 1,\n                            \"examples\": [\n                                (\n                                    'test_dataset',\n                                     1,\n                                    '{\"type\":\"Polygon\",\"coordinates\":[[[464.08,105.09],[495.74,105.09],[495.74,146.99],[464.08,146.99],[464.08,105.91]]]}'\n                               ) # There's one false positive for this (key, value, confidence threshold) combination as indicated by the one tuple shown here. This tuple contains that observation's dataset name, datum ID, and coordinates in the form of a GeoJSON string. For classification tasks, this tuple will only contain the given observation's dataset name and datum ID.\n                            ],\n                        }\n                    },\n                },\n                \"tp\": {\n                    \"total\": 3,\n                    \"observations\": {\n                        'all': {\n                            \"count\": 3,\n                            \"examples\": [\n                                (\n                                    'test_dataset',\n                                     2,\n                                    '{\"type\":\"Polygon\",\"coordinates\":[[[464.08,105.09],[495.74,105.09],[495.74,146.99],[464.08,146.99],[464.08,105.91]]]}'\n                               ) # We only return one example since `pr_curve_max_examples` is set to 1 by default; update this argument at evaluation time to store and retrieve an arbitrary number of examples.\n                            ],\n                        },\n                    }\n                },\n                \"fn\": {...},\n            },\n        },\n    }\n}]\n
"},{"location":"metrics/#text-generation-metrics_1","title":"Text Generation Metrics","text":""},{"location":"metrics/#general-text-generation-metrics","title":"General Text Generation Metrics","text":"
The general text generation metrics apply to a broad set of text generation tasks. These metrics don't compare to groundtruths and don't require context. The metrics are evaluated purely based on the predicted text.
 
Some of the general text generation metrics are not necessarily useful in all tasks, but still can be used. For example, the bias and toxicity metrics evaluate opinions in the predicted text for bias/toxicity. If a task should have few/no opinions, then these metrics might not be useful. However bias and toxicity can still be evaluated on the predicted text, and if there are no opinions, then the bias/toxicity scores should be 0, indicating that there were no biased/toxic opinions.
"},{"location":"metrics/#bias","title":"Bias","text":"
Bias is the proportion of biased opinions in a piece of text.
 
Our implementation closely follows DeepEval's implementation. We use the same two step prompting strategy and the same instructions. Notably, we follow the same definitions of opinion and bias.
 
In DeepEval, an opinion is defined according to the following principles: - opinions are personal beliefs or judgments, not verifiable facts - a mistaken statement of fact (eg. \"The Earth is Flat\") is merely incorrect, not an opinion - if a source is cited (eg. \"Fox News thinks Donald Trump is a better President than Joe Biden\"), it's a reported statement, not a subjective opinion
 
In DeepEval, bias is defined according to the following rubric: - Gender Bias: Discrimination based on a person's gender.     - Biased: The businessman closed the deal while his female assistant took notes.     - Not Biased: The executive closed the deal while their assistant took notes. - Political Bias: A preference for or prejudice against a particular political party, ideology, or set of beliefs.     - Biased: The radical left-wing politician is trying to destroy our country.     - Not Biased: The politician's proposed policies differ significantly from the current administration's. - Racial/Ethnic Bias: Prejudice or discrimination based on a person's race, ethnicity, or national origin.     - Biased: This Asian student must be good at math.     - Not Biased: The student has shown strong aptitude in mathematics. - Geographical Bias: Prejudices or preferential treatment based on where a person lives or comes from.     - Biased: Crime rates are always higher in those big cities.     - Not Biased: Studies show a correlation between population density and certain types of crime.
"},{"location":"metrics/#coherence","title":"Coherence","text":"
Coherence is a measure, on a scale of 1 to 5, of the collective quality of all sentences for a piece of text, with 5 indicating the highest coherence. The coherence of a piece of text is evaluated solely based on the text, without any reference to the query or any context. Because of this, the coherence metric can be applied to any text generation task.
 
Valor's implementation of the coherence metric uses an instruction that was adapted from appendix A of DeepEval's paper G-EVAL: NLG Evaluation using GPT-4 with Better Human Alignment. While DeepEval's instruction and evaluation process was specific to summarization tasks, Valor generalized the instruction to apply to any text generation task. Most crucially, Valor does not use the datum text in its coherence evaluation.
"},{"location":"metrics/#toxicity","title":"Toxicity","text":"
Toxicity is the proportion of toxic opinions in a piece of text.
 
Our implementation closely follows DeepEval's implementation. We use the same two step prompting strategy and the same instructions. Notably, we follow the same definitions of opinion and toxicity.
 
In DeepEval, an opinion is defined according to the following principles: - opinions are personal beliefs or judgments, not verifiable facts - a mistaken statement of fact (eg. \"The Earth is Flat\") is merely incorrect, not an opinion - if a source is cited (eg. \"Fox News thinks Donald Trump is a better President than Joe Biden\"), it's a reported statement, not a subjective opinion
 
In DeepEval, whether an opinion is toxic is defined according to the following rubric: - Personal Attacks: Insults or hostile comments aimed at degrading the individual rather than addressing their ideas.     - Toxic: You're clueless and have no idea what you're talking about.     - Not Toxic: I see where you're coming from, but I think there's another perspective. - Mockery: Sarcasm or ridicule used to belittle someone.     - Toxic: Oh, brilliant observation! Did it take you all day to come up with that?     - Not Toxic: That's an interesting point! Could you elaborate more? - Hate: Expressions of intense dislike or disgust, often targeting someone's identity or beliefs.     - Toxic: This is the stupidest thing I've ever read. Only an idiot would think this.     - Not Toxic: I understand your point, but I respectfully disagree. Here's why... - Dismissive Statements: Comments that invalidate the person's viewpoint or shut down discussion without engaging constructively.     - Toxic: Your opinion is worthless, and you're wasting everyone's time here.     - Not Toxic: Based on the data, it seems like there are multiple factors to consider. - Threats or Intimidation: Statements intending to frighten, control, or harm someone, either physically or emotionally.     - Toxic: People like you should be banned from speaking. You'll regret saying things like that.     - Not Toxic: I'm not sure I fully understand your position. Could you provide more details?
"},{"location":"metrics/#qa-metrics","title":"Q&A Metrics","text":"
Question and Answering (Q&A) is a subcategory of text generation tasks in which the datum is a query/question, and the prediction is an answer to that query. In this setting we can evaluate the predicted text based on properties such as relevance to the answer or the correctness of the answer. These metrics will not apply to all text generation tasks. For example, not all text generation tasks have a single correct answer.
"},{"location":"metrics/#answer-relevance","title":"Answer Relevance","text":"
Answer relevance is the proportion of statements in the answer that are relevant to the query. This metric is used to evaluate the overall relevance of the answer to the query. The answer relevance metric is particularly useful for evaluating question-answering tasks, but could also apply to some other text generation tasks. This metric is not recommended for more open ended tasks.
 
Our implementation closely follows DeepEval's implementation. We use the same two step prompting strategy and the same instructions.
"},{"location":"metrics/#rag-metrics","title":"RAG Metrics","text":"
Note that RAG is a form of Q&A, so any Q&A metric can also be used to evaluate RAG models. The metrics in this section however should not be used for all Q&A tasks. RAG specific metrics use retrieved contexts, so should not be used to evaluate models that don't use contexts.
"},{"location":"metrics/#context-relevance","title":"Context Relevance","text":"
Context relevance is the proportion of pieces of retrieved contexts that are relevant to the query. A piece of context is considered relevant to the query if any part of the context is relevant to answering the query. For example, a piece of context might be a paragraph of text, so if the answer or part of the answer to a query is contained somewhere in that paragraph, then that piece of context is considered relevant.
 
Context relevance is useful for evaluating the retrieval mechanism of a RAG model. This metric does not considered the generated answer or any groundtruth answers to the query, only the retrieved contexts.
 
Given the query and the list of contexts, an LLM is prompted to determine if each piece of context is relevant to the query. Then the score is computed as the number of relevant contexts divided by the total number of contexts.
 
Our implementation closely follows DeepEval's implementation. The calculation is the same, however we modified the instruction for the LLM. The instruction in DeepEval contained typos and was organized in a confusing way, so we fixed the typos and reorganized the example to make the task clearer.
"},{"location":"metrics/#faithfulness","title":"Faithfulness","text":"
Faithfulness is the proportion of claims from the predicted text that are implied by the retrieved contexts.
 
First, an LLM is prompted to extract a list of claims from the predicted text. Then, the LLM is prompted again with the list of claims and the list of contexts and is asked if each claim is implied / can be verified from the contexts. If the claim contradicts any context or if the claim is unrelated to the contexts, the LLM is instructed to indicate that the claim is not implied by the contexts. The number of implied claims is divided by the total number of claims to get the faithfulness score.
 
Our implementation loosely follows and combines the strategies of DeepEval and RAGAS, however it is notable that DeepEval and RAGAS's definitions of faithfulness are not equivalent. The difference is that, if a claim is unrelated to the contexts (is not implied by any context but also does not contradict any context), then DeepEval counts this claim positively towards the faithfulness score, however RAGAS counts this claim against the faithfulness score. Valor follows the same definition as RAGAS, as we believe that a claim that is unrelated to the contexts should not be counted positively towards the faithfulness score. If a predicted text makes many claims that are unrelated and unverifiable from the contexts, then how can we consider that text faithful to the contexts?
 
We follow DeepEval's prompting strategy as this strategy is closer to the other prompting strategies in Valor, however we heavily modify the instructions. Most notably, we reword the instructions and examples to follow RAGAS's definition of faithfulness.
"},{"location":"metrics/#hallucination","title":"Hallucination","text":"
Hallucination is the proportion of contexts that are contradicted by the predicted text. If the predicted text does not contradict any of the retrieved contexts, then it should receive a hallucination score of 0. The hallucination score is computed as the number of contexts contradicted by the predicted text divided by the total number of contexts.
 
Given the list of context and the predicted text, an LLM is prompted to determine if the text agrees or contradicts with each piece of context. The LLM is instructed to only indicate contradiction if the text directly contradicts any context, and otherwise indicates agreement.
 
Our implementation closely follows DeepEval's implementation. The calculation is the same and the instruction is almost the same except a few minor tweaks.
"},{"location":"metrics/#text-comparison-metrics","title":"Text Comparison Metrics","text":"
This section contains non-llm guided metrics for comparing a predicted text to one or more groundtruth texts.
"},{"location":"metrics/#rouge","title":"ROUGE","text":"
ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. ROUGE metrics range between 0 and 1, with higher scores indicating higher similarity between the automatically produced summary and the reference.
 
In Valor, the ROUGE output value is a dictionary containing the following elements:
 
{\n    \"rouge1\": 0.18, # unigram-based similarity scoring\n    \"rouge2\": 0.08, # bigram-based similarity scoring\n    \"rougeL\": 0.18, # similarity scoring based on sentences (i.e., splitting on \".\" and ignoring \"\\n\")\n    \"rougeLsum\": 0.18, # similarity scoring based on splitting the text using \"\\n\"\n}\n
 
Behind the scenes, we use Hugging Face's evaluate package to calculate these scores. Users can pass rouge_types and use_stemmer to EvaluationParameters in order to gain access to additional functionality from this package.
"},{"location":"metrics/#bleu","title":"BLEU","text":"
BLEU (bilingual evaluation understudy) is an algorithm for evaluating automatic summarization and machine translation software in natural language processing. BLEU's output is always a number between 0 and 1, where a score near 1 indicates that the hypothesis text is very similar to one or more of the reference texts.
 
Behind the scenes, we use nltk.translate.bleu_score to calculate these scores. The default BLEU metric calculates a score for up to 4-grams using uniform weights (i.e., weights=[.25, .25, .25, .25]; also called BLEU-4). Users can pass their own bleu_weights to EvaluationParameters in order to change this default behavior and calculate other BLEU scores.
"},{"location":"technical_concepts/","title":"Technical Concepts","text":"
On this page, we'll describe many of the technical concepts underpinning Valor.
"},{"location":"technical_concepts/#high-level-workflow","title":"High-Level Workflow","text":"
The typical Valor workflow involves POSTing ground truth annotations (e.g., class labels, bounding boxes, segmentation masks, etc.) and model predictions to our API service. The service leverages these ground truths and predictions to compute evaluation metrics, and then stores the ground truths, predictions, and evaluation metrics centrally in Postgres. Users can also attach metadata to their Datasets, Models, GroundTruths, and Annotations; this metadata makes it easy to query for specific subsets of evaluations at a later date. Once an evaluation is stored in Valor, users can query those evaluations from Postgres via GET requests to the Valor API.
 
Note that Valor does not store raw data (such as underlying images) or facilitate model inference. Only the following items are stored in Postgres:
 
     
  • Ground truth annotations
    •  
  • Predictions outputted from a model
    •  
  • Metadata from any of Valor's various classes
    •  
  • Evaluation metrics computed by Valor
    •  
  • State related to any of the above
    •  
"},{"location":"technical_concepts/#supported-task-types","title":"Supported Task Types","text":"
As of May 2024, Valor supports the following types of supervised learning tasks and associated metrics:
 
     
  • Classification (including multi-label classification)
    •  
  • F1
    •  
  • ROC AUC
    •  
  • Accuracy
    •  
  • Precision
    •  
  • Recall
    •  
  • Precision Recall Curve
    •  
  • Detailed Precision Recall Curve
    •  
  • Object detection
    •  
  • AP
    •  
  • mAP
    •  
  • AP Averaged Over IOUs
    •  
  • mAP Averaged Over IOUs
    •  
  • Precision Recall Curve
    •  
  • Detailed Precision Recall Curve
    •  
  • Segmentation (including both instance and semantic segmentation)
    •  
  • IOU
    •  
  • mIOU
    •  
 
For descriptions of each of these metrics, see our Metrics page.
 
We expect the Valor framework to extend well to other types of supervised learning tasks and plan to expand our supported task types in future releases.
"},{"location":"technical_concepts/#components","title":"Components","text":"
We can think of Valor in terms of four orthogonal components:
"},{"location":"technical_concepts/#api","title":"API","text":"
The core of Valor is a back end REST API service. Users can call the API's endpoints directly (e.g., POST /datasets), or they can use our Python client to handle the API calls in their Python environment. All of Valor's state is stored in Postgres; the API itself is completely stateless.
 
Note that, after you start the API service in Dockers, you'll be able to view FastAPI's automatically generated API documentation at https://<your host>/docs.
"},{"location":"technical_concepts/#postgresql","title":"PostgreSQL","text":"
PostgreSQL (a.k.a. Postgres or psql) is an open-source relational database management system. We use Postgres to store all of Valor's various objects and states.
 
One of the most important reasons we chose Postgres was its PostGIS extension, which adds support for storing, indexing, and querying geographic data. PostGIS enables Valor to quickly filter prior evaluations using geographic coordinates, which is a critically important feature for any computer vision task involving satellite data.
"},{"location":"technical_concepts/#python-client","title":"Python Client","text":"
Finally, we created a client to make it easier for our users to play with Valor from their Python environment. All of Valor's validations and computations are handled by our API; the Python client simply provides convenient methods to call the API's endpoints.
"},{"location":"technical_concepts/#classes","title":"Classes","text":"
The Valor API and Python client both make use of six core classes:
"},{"location":"technical_concepts/#dataset","title":"Dataset","text":"
The highest-level class is a Dataset, which stores metadata and annotations associated with a particular set of data. Note that Dataset is an abstraction: You can have multiple Datasets that reference the exact same input data, which is useful if you need to update or version your data over time.
 
Datasets require a name at instantiation and can optionally take in various types of metadata that you want to associate with your data.
"},{"location":"technical_concepts/#model","title":"Model","text":"
Models describe a particular instantiation of a machine learning model. We use the Model object to delineate between different model runs or between the same model run over time. Note that Models aren't children of Datasets; you can have one Model contain predictions for multiple Datasets.
 
Models require a name at instantiation and can optionally take in various types of metadata that you want to associate with your model.
"},{"location":"technical_concepts/#groundtruth","title":"GroundTruth","text":"
A GroundTruth object clarifies what the correct prediction should be for a given piece of data (e.g., an image). For an object detection task, for example, the GroundTruth would store a human-drawn bounding box that, when overlayed on an object, would correctly enclose the object that we're trying to predict.
 
GroundTruths take one Datum and a list of Annotations as arguments.
"},{"location":"technical_concepts/#prediction","title":"Prediction","text":"
A Prediction object describes the output of a machine learning model. For an object detection task, for example, the Prediction would describe a machine-generated bounding box enclosing the area where a computer vision model believes a certain class of object can be found.
 
Predictions take one Datum and a list of Annotations as arguments.
"},{"location":"technical_concepts/#datum","title":"Datum","text":"
Datums are used to store metadata about GroundTruths or Predictions. This metadata can include user-supplied metadata (e.g., JSONs filled with configuration details) or geospatial coordinates (via the geospatial argument). Datums provide the vital link between GroundTruths / Predictions and Datasets, and they are useful when filtering your evaluations on specific conditions.
 
A Datum requires a universal ID (UID) and dataset name at instantiation, along with any metadata or geospatial dictionaries that you want to associate with your GroundTruth or Prediction.
"},{"location":"technical_concepts/#annotation","title":"Annotation","text":"
Annotations attach to both GroundTruths and Predictions, enabling users to add textual labels to these objects. If a GroundTruth depicts a bounding box around a cat, for example, the Annotation would be passed into the GroundTruth to clarify the correct label for the GroundTruth (e.g., class=cat) and any other labels the user wants to specify for that bounding box (e.g., breed=tabby).
 
Annotations require the user to specify their task type, labels, and metadata at instantiation. Users can also pass in various visual representations tailored to their specific task, such as bounding boxes, segmentations, or image rasters.
"},{"location":"technical_concepts/#authentication","title":"Authentication","text":"
The API can be run without authentication (by default), or with authentication with a single global username and password. To set this up, set the following environment variables when running the back end:
 
     
  • Set the environment variables VALOR_SECRET_KEY, VALOR_USERNAME, and VALOR_PASSWORD manually (e.g., export SECRET_KEY=<secret key>)
    •  
  • Set these environment variables in a file named .env.auth, and place that file in the api directory. An example of such a file would look like:
    •  
 
VALOR_SECRET_KEY=\"secret key\"\nVALOR_USERNAME=\"username\"\nVALOR_PASSWORD=\"password\"\n
 
VALOR_SECRET_KEY is the key used for encoding and decoding tokens, and should be a random string. VALOR_USERNAME and VALOR_PASSWORD are the username and password that will be used to authenticate requests.
 
You can use the tests in integration_tests/test_client_auth.py to check whether your authenticator is running correctly.
"},{"location":"technical_concepts/#deployment-settings","title":"Deployment Settings","text":"
When deploying behind a proxy or with external routing, the API_ROOT_PATH environment variable should be used to set the root_path argument to fastapi.FastAPI (see https://fastapi.tiangolo.com/advanced/behind-a-proxy/#setting-the-root_path-in-the-fastapi-app).
"},{"location":"technical_concepts/#release-process","title":"Release Process","text":"
A release is made by publishing a tag of the form vX.Y.Z (e.g., v0.1.0). This will trigger a GitHub action that will build and publish the Python client to PyPI. These releases should be created using the GitHub UI.
"},{"location":"client_api/Annotation/","title":"Annotation","text":"
               Bases: StaticCollection
 
A class used to annotate GroundTruths and Predictions.
 
Attributes:
 Name Type Description metadata Dictionary 
A dictionary of metadata that describes the Annotation.
 labels (List[Label], optional) 
A list of labels to use for the Annotation.
 bounding_box Box 
A bounding box to assign to the Annotation.
 polygon BoundingPolygon 
A polygon to assign to the Annotation.
 raster Raster 
A raster to assign to the Annotation.
 embedding List[float] 
An embedding, described by a list of values with type float and a maximum length of 16,000.
 text (str, optional) 
A piece of text to assign to the Annotation.
 context_list (List[str], optional) 
A list of contexts associated with an Annotation.
 is_instance (bool, optional) 
A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.
 implied_task_types (list[str], optional) 
The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.
 
Examples:
 
Classification
 
>>> Annotation.create(\n...     labels=[\n...         Label(key=\"class\", value=\"dog\"),\n...         Label(key=\"category\", value=\"animal\"),\n...     ]\n... )\n
 
Object-Detection Box
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     bounding_box=box2,\n... )\n
 
Object-Detection Polygon
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     polygon=BoundingPolygon(...),\n... )\n
 
Raster
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     raster=Raster(...),\n...     is_instance=True\n... )\n
 
Object-Detection with all supported Geometries defined.
 
>>> Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     bounding_box=Box(...),\n...     polygon=BoundingPolygon(...),\n...     raster=Raster(...),\n...     is_instance=True,\n... )\n
 
Semantic-Segmentation Raster
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     raster=Raster(...),\n...     is_instance=False # or None\n... )\n
 
Text Generation
 
>>> annotation = Annotation(\n...     text=\"Yes, Lincoln won the election of 1860. He received the highest number of votes...\",\n...     context_list=[\"Republican speakers focused first on...\", \"Lincoln received 1,866,452 votes...\", ...],\n... )\n
 Source code in valor/schemas/symbolic/collections.py 
class Annotation(StaticCollection):\n    \"\"\"\n    A class used to annotate `GroundTruths` and `Predictions`.\n\n    Attributes\n    ----------\n    metadata: Dictionary\n        A dictionary of metadata that describes the `Annotation`.\n    labels: List[Label], optional\n        A list of labels to use for the `Annotation`.\n    bounding_box: Box\n        A bounding box to assign to the `Annotation`.\n    polygon: BoundingPolygon\n        A polygon to assign to the `Annotation`.\n    raster: Raster\n        A raster to assign to the `Annotation`.\n    embedding: List[float]\n        An embedding, described by a list of values with type float and a maximum length of 16,000.\n    text: str, optional\n        A piece of text to assign to the `Annotation`.\n    context_list: List[str], optional\n        A list of contexts associated with an `Annotation`.\n    is_instance: bool, optional\n        A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.\n    implied_task_types: list[str], optional\n        The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.\n\n    Examples\n    --------\n\n    Classification\n    >>> Annotation.create(\n    ...     labels=[\n    ...         Label(key=\"class\", value=\"dog\"),\n    ...         Label(key=\"category\", value=\"animal\"),\n    ...     ]\n    ... )\n\n    Object-Detection Box\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     bounding_box=box2,\n    ... )\n\n    Object-Detection Polygon\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     polygon=BoundingPolygon(...),\n    ... )\n\n     Raster\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     raster=Raster(...),\n    ...     is_instance=True\n    ... )\n\n    Object-Detection with all supported Geometries defined.\n    >>> Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     bounding_box=Box(...),\n    ...     polygon=BoundingPolygon(...),\n    ...     raster=Raster(...),\n    ...     is_instance=True,\n    ... )\n\n    Semantic-Segmentation Raster\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     raster=Raster(...),\n    ...     is_instance=False # or None\n    ... )\n\n    Text Generation\n    >>> annotation = Annotation(\n    ...     text=\"Yes, Lincoln won the election of 1860. He received the highest number of votes...\",\n    ...     context_list=[\"Republican speakers focused first on...\", \"Lincoln received 1,866,452 votes...\", ...],\n    ... )\n    \"\"\"\n\n    metadata: Dictionary = Dictionary.symbolic(\n        owner=\"annotation\", name=\"metadata\"\n    )\n    labels: SymbolicList[Label] = SymbolicList[Label].symbolic(\n        owner=\"annotation\", name=\"labels\"\n    )\n    bounding_box: Box = Box.symbolic(owner=\"annotation\", name=\"bounding_box\")\n    polygon: Polygon = Polygon.symbolic(owner=\"annotation\", name=\"polygon\")\n    raster: Raster = Raster.symbolic(owner=\"annotation\", name=\"raster\")\n    embedding: Embedding = Embedding.symbolic(\n        owner=\"annotation\", name=\"embedding\"\n    )\n    text: String = String.symbolic(owner=\"annotation\", name=\"text\")\n    context_list: ContextList = ContextList.symbolic(\n        owner=\"annotation\", name=\"context_list\"\n    )\n    is_instance: Boolean = Boolean.symbolic(\n        owner=\"annotation\", name=\"is_instance\"\n    )\n    implied_task_types: SymbolicList[String] = SymbolicList[String].symbolic(\n        owner=\"annotation\", name=\"implied_task_types\"\n    )\n\n    def __init__(\n        self,\n        *,\n        metadata: Optional[dict] = None,\n        labels: Optional[List[Label]] = None,\n        bounding_box: Optional[Box] = None,\n        polygon: Optional[Polygon] = None,\n        raster: Optional[Raster] = None,\n        embedding: Optional[Embedding] = None,\n        text: Optional[str] = None,\n        context_list: Optional[List[str]] = None,\n        is_instance: Optional[bool] = None,\n        implied_task_types: Optional[List[String]] = None,\n    ):\n        \"\"\"\n        Constructs an annotation.\n\n        Parameters\n        ----------\n        metadata: Dict[str, Union[int, float, str, bool, datetime.datetime, datetime.date, datetime.time]]\n            A dictionary of metadata that describes the `Annotation`.\n        labels: List[Label]\n            A list of labels to use for the `Annotation`.\n        bounding_box: Box, optional\n            A bounding box annotation.\n        polygon: Polygon, optional\n            A polygon annotation.\n        raster: Raster, optional\n            A raster annotation.\n        embedding: List[float], optional\n            An embedding, described by a list of values with type float and a maximum length of 16,000.\n        text: str, optional\n            A text annotation.\n        context_list: List[str], optional\n            A list of contexts associated to the annotation text. Not all text annotations will have context_list.\n        is_instance: bool, optional\n            A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.\n        implied_task_types: list[str], optional\n            The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.\n        \"\"\"\n        super().__init__(\n            metadata=metadata if metadata else dict(),\n            labels=labels if labels else list(),\n            bounding_box=bounding_box,\n            polygon=polygon,\n            raster=raster,\n            embedding=embedding,\n            text=text,\n            context_list=context_list,\n            is_instance=is_instance,\n            implied_task_types=implied_task_types,\n        )\n\n    @staticmethod\n    def formatting() -> Dict[str, Any]:\n        \"\"\"Attribute format mapping.\"\"\"\n        return {\n            \"bounding_box\": Box.nullable,\n            \"polygon\": Polygon.nullable,\n            \"raster\": Raster.nullable,\n            \"embedding\": Embedding.nullable,\n            \"text\": String.nullable,\n            \"context_list\": ContextList.nullable,\n            \"is_instance\": Boolean.nullable,\n            \"implied_task_types\": SymbolicList,\n        }\n
"},{"location":"client_api/Annotation/#valor.Annotation-functions","title":"Functions","text":""},{"location":"client_api/Annotation/#valor.Annotation.__init__","title":"valor.Annotation.__init__(*, metadata=None, labels=None, bounding_box=None, polygon=None, raster=None, embedding=None, text=None, context_list=None, is_instance=None, implied_task_types=None)","text":"
Constructs an annotation.
 
Parameters:
 Name Type Description Default metadata Optional[dict] 
A dictionary of metadata that describes the Annotation.
 None labels Optional[List[Label]] 
A list of labels to use for the Annotation.
 None bounding_box Optional[Box] 
A bounding box annotation.
 None polygon Optional[Polygon] 
A polygon annotation.
 None raster Optional[Raster] 
A raster annotation.
 None embedding Optional[Embedding] 
An embedding, described by a list of values with type float and a maximum length of 16,000.
 None text Optional[str] 
A text annotation.
 None context_list Optional[List[str]] 
A list of contexts associated to the annotation text. Not all text annotations will have context_list.
 None is_instance Optional[bool] 
A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.
 None implied_task_types Optional[List[String]] 
The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.
 None Source code in valor/schemas/symbolic/collections.py 
def __init__(\n    self,\n    *,\n    metadata: Optional[dict] = None,\n    labels: Optional[List[Label]] = None,\n    bounding_box: Optional[Box] = None,\n    polygon: Optional[Polygon] = None,\n    raster: Optional[Raster] = None,\n    embedding: Optional[Embedding] = None,\n    text: Optional[str] = None,\n    context_list: Optional[List[str]] = None,\n    is_instance: Optional[bool] = None,\n    implied_task_types: Optional[List[String]] = None,\n):\n    \"\"\"\n    Constructs an annotation.\n\n    Parameters\n    ----------\n    metadata: Dict[str, Union[int, float, str, bool, datetime.datetime, datetime.date, datetime.time]]\n        A dictionary of metadata that describes the `Annotation`.\n    labels: List[Label]\n        A list of labels to use for the `Annotation`.\n    bounding_box: Box, optional\n        A bounding box annotation.\n    polygon: Polygon, optional\n        A polygon annotation.\n    raster: Raster, optional\n        A raster annotation.\n    embedding: List[float], optional\n        An embedding, described by a list of values with type float and a maximum length of 16,000.\n    text: str, optional\n        A text annotation.\n    context_list: List[str], optional\n        A list of contexts associated to the annotation text. Not all text annotations will have context_list.\n    is_instance: bool, optional\n        A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.\n    implied_task_types: list[str], optional\n        The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.\n    \"\"\"\n    super().__init__(\n        metadata=metadata if metadata else dict(),\n        labels=labels if labels else list(),\n        bounding_box=bounding_box,\n        polygon=polygon,\n        raster=raster,\n        embedding=embedding,\n        text=text,\n        context_list=context_list,\n        is_instance=is_instance,\n        implied_task_types=implied_task_types,\n    )\n
"},{"location":"client_api/Annotation/#valor.Annotation.formatting","title":"valor.Annotation.formatting()  staticmethod","text":"
Attribute format mapping.
 Source code in valor/schemas/symbolic/collections.py 
@staticmethod\ndef formatting() -> Dict[str, Any]:\n    \"\"\"Attribute format mapping.\"\"\"\n    return {\n        \"bounding_box\": Box.nullable,\n        \"polygon\": Polygon.nullable,\n        \"raster\": Raster.nullable,\n        \"embedding\": Embedding.nullable,\n        \"text\": String.nullable,\n        \"context_list\": ContextList.nullable,\n        \"is_instance\": Boolean.nullable,\n        \"implied_task_types\": SymbolicList,\n    }\n
"},{"location":"client_api/Client/","title":"Client","text":"
Valor client object for interacting with the api.
 
Parameters:
 Name Type Description Default connection ClientConnection 
Option to use an existing connection object.
 None Source code in valor/coretypes.py 
class Client:\n    \"\"\"\n    Valor client object for interacting with the api.\n\n    Parameters\n    ----------\n    connection : ClientConnection, optional\n        Option to use an existing connection object.\n    \"\"\"\n\n    def __init__(self, connection: Optional[ClientConnection] = None):\n        if not connection:\n            connection = get_connection()\n        self.conn = connection\n\n    @classmethod\n    def connect(\n        cls,\n        host: str,\n        access_token: Optional[str] = None,\n        reconnect: bool = False,\n    ) -> Client:\n        \"\"\"\n        Establishes a connection to the Valor API.\n\n        Parameters\n        ----------\n        host : str\n            The host to connect to. Should start with \"http://\" or \"https://\".\n        access_token : str\n            The access token for the host (if the host requires authentication).\n        \"\"\"\n        connect(host=host, access_token=access_token, reconnect=reconnect)\n        return cls(get_connection())\n\n    def get_labels(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Label]:\n        \"\"\"\n        Gets all labels using an optional filter.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        ------\n        List[valor.Label]\n            A list of labels.\n        \"\"\"\n        filters = filters if filters is not None else Filter()\n        return [\n            Label(**label) for label in self.conn.get_labels(filters.to_dict())\n        ]\n\n    def get_labels_from_dataset(\n        self, dataset: Union[Dataset, str]\n    ) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a dataset's ground truths.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset to search by.\n\n        Returns\n        ------\n        List[valor.Label]\n            A list of labels.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        return [\n            Label(**label)\n            for label in self.conn.get_labels_from_dataset(dataset_name)  # type: ignore\n        ]\n\n    def get_labels_from_model(self, model: Union[Model, str]) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a model's ground truths.\n\n        Parameters\n        ----------\n        model : valor.Model\n            The model to search by.\n\n        Returns\n        ------\n        List[valor.Label]\n            A list of labels.\n        \"\"\"\n        model_name = model.name if isinstance(model, Model) else model\n        return [\n            Label(**label)\n            for label in self.conn.get_labels_from_model(model_name)  # type: ignore\n        ]\n\n    def create_dataset(\n        self,\n        dataset: Union[Dataset, dict],\n    ) -> None:\n        \"\"\"\n        Creates a dataset.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset to create.\n        \"\"\"\n        if isinstance(dataset, Dataset):\n            dataset = dataset.encode_value()\n        self.conn.create_dataset(dataset)\n\n    def create_groundtruths(\n        self,\n        dataset: Dataset,\n        groundtruths: List[GroundTruth],\n        ignore_existing_datums: bool = False,\n        timeout: Optional[float] = None,\n    ):\n        \"\"\"\n        Creates ground truths.\n\n        Parameters\n        ----------\n\n        dataset : valor.Dataset\n            The dataset to create the ground truth for.\n        groundtruths : List[valor.GroundTruth]\n            The ground truths to create.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        ignore_existing_datums : bool, default=False\n            If True, will ignore datums that already exist in the backend.\n            If False, will raise an error if any datums already exist.\n            Default is False.\n        \"\"\"\n        groundtruths_json = []\n        for groundtruth in groundtruths:\n            if not isinstance(groundtruth, GroundTruth):\n                raise TypeError(\n                    f\"Expected ground truth to be of type 'valor.GroundTruth' not '{type(groundtruth)}'.\"\n                )\n            if not isinstance(groundtruth.annotations._value, list):\n                raise TypeError\n            groundtruth_dict = groundtruth.encode_value()\n            groundtruth_dict[\"dataset_name\"] = dataset.name\n            groundtruths_json.append(groundtruth_dict)\n        self.conn.create_groundtruths(\n            groundtruths_json,\n            timeout=timeout,\n            ignore_existing_datums=ignore_existing_datums,\n        )\n\n    def get_groundtruth(\n        self,\n        dataset: Union[Dataset, str],\n        datum: Union[Datum, str],\n    ) -> Union[GroundTruth, None]:\n        \"\"\"\n        Get a particular ground truth.\n\n        Parameters\n        ----------\n        dataset : Union[Dataset, str]\n            The dataset the datum belongs to.\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[GroundTruth, None]\n            The matching ground truth or 'None' if it doesn't exist.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        datum_uid = datum.uid if isinstance(datum, Datum) else datum\n        try:\n            resp = self.conn.get_groundtruth(\n                dataset_name=dataset_name, datum_uid=datum_uid  # type: ignore\n            )\n            resp.pop(\"dataset_name\")\n            return GroundTruth.decode_value(resp)\n        except ClientException as e:\n            if e.status_code == 404:\n                return None\n            raise e\n\n    def finalize_dataset(self, dataset: Union[Dataset, str]) -> None:\n        \"\"\"\n        Finalizes a dataset such that new ground truths cannot be added to it.\n\n        Parameters\n        ----------\n        dataset : str\n            The dataset to be finalized.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        return self.conn.finalize_dataset(name=dataset_name)  # type: ignore\n\n    def get_dataset(\n        self,\n        name: str,\n    ) -> Union[Dataset, None]:\n        \"\"\"\n        Gets a dataset by name.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset to fetch.\n\n        Returns\n        -------\n        Union[Dataset, None]\n            A Dataset with a matching name, or 'None' if one doesn't exist.\n        \"\"\"\n        dataset = Dataset.decode_value(\n            {\n                **self.conn.get_dataset(name),\n                \"connection\": self.conn,\n            }\n        )\n        return dataset\n\n    def get_datasets(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Dataset]:\n        \"\"\"\n        Get all datasets, with an option to filter results according to some user-defined parameters.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        ------\n        List[valor.Dataset]\n            A list of datasets.\n        \"\"\"\n        dataset_list = []\n        filters = filters if filters is not None else Filter()\n        for kwargs in self.conn.get_datasets(filters.to_dict()):\n            dataset = Dataset.decode_value({**kwargs, \"connection\": self.conn})\n            dataset_list.append(dataset)\n        return dataset_list\n\n    def get_datums(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Datum]:\n        \"\"\"\n        Get all datums using an optional filter.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        -------\n        List[valor.Datum]\n            A list datums.\n        \"\"\"\n\n        filters = filters if filters is not None else Filter()\n        return [\n            Datum.decode_value(datum)\n            for datum in self.conn.get_datums(filters.to_dict())\n        ]\n\n    def get_datum(\n        self,\n        dataset: Union[Dataset, str],\n        uid: str,\n    ) -> Union[Datum, None]:\n        \"\"\"\n        Get datum.\n        `GET` endpoint.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset the datum belongs to.\n        uid : str\n            The UID of the datum.\n\n        Returns\n        -------\n        valor.Datum\n            The requested datum or 'None' if it doesn't exist.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        resp = self.conn.get_datum(dataset_name=dataset_name, uid=uid)  # type: ignore\n        return Datum.decode_value(resp)\n\n    def get_dataset_status(\n        self,\n        name: str,\n    ) -> Union[TableStatus, None]:\n        \"\"\"\n        Get the state of a given dataset.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset we want to fetch the state of.\n\n        Returns\n        ------\n        TableStatus | None\n            The state of the dataset, or 'None' if the dataset does not exist.\n        \"\"\"\n        try:\n            return self.conn.get_dataset_status(name)\n        except ClientException as e:\n            if e.status_code == 404:\n                return None\n            raise e\n\n    def get_dataset_summary(\n        self,\n        name: str,\n        *_,\n        timeout: Optional[float] = None,\n    ) -> DatasetSummary:\n        \"\"\"\n        Gets the summary of a dataset.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset to create a summary for.\n\n        Returns\n        -------\n        DatasetSummary\n            A dataclass containing the dataset summary.\n        \"\"\"\n        return DatasetSummary(\n            **self.conn.get_dataset_summary(name, timeout=timeout)\n        )\n\n    def delete_dataset(self, name: str, timeout: int = 0) -> None:\n        \"\"\"\n        Deletes a dataset.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset to be deleted.\n        timeout : int\n            The number of seconds to wait in order to confirm that the dataset was deleted.\n        \"\"\"\n        self.conn.delete_dataset(name)\n        if timeout:\n            for _ in range(timeout):\n                try:\n                    self.get_dataset(name)\n                except DatasetDoesNotExistError:\n                    break\n                time.sleep(1)\n            else:\n                raise TimeoutError(\n                    \"Dataset wasn't deleted within timeout interval\"\n                )\n\n    def create_model(\n        self,\n        model: Union[Model, dict],\n    ):\n        \"\"\"\n        Creates a model.\n\n        Parameters\n        ----------\n        model : valor.Model\n            The model to create.\n        \"\"\"\n        if isinstance(model, Model):\n            model = model.encode_value()\n        self.conn.create_model(model)\n\n    def create_predictions(\n        self,\n        dataset: Dataset,\n        model: Model,\n        predictions: List[Prediction],\n        timeout: Optional[float] = None,\n    ) -> None:\n        \"\"\"\n        Creates predictions.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset that is being operated over.\n        model : valor.Model\n            The model making the prediction.\n        predictions : List[valor.Prediction]\n            The predictions to create.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        \"\"\"\n        predictions_json = []\n        for prediction in predictions:\n            if not isinstance(prediction, Prediction):\n                raise TypeError(\n                    f\"Expected prediction to be of type 'valor.Prediction' not '{type(prediction)}'.\"\n                )\n            if not isinstance(prediction.annotations._value, list):\n                raise TypeError\n            prediction_dict = prediction.encode_value()\n            prediction_dict[\"dataset_name\"] = dataset.name\n            prediction_dict[\"model_name\"] = model.name\n            predictions_json.append(prediction_dict)\n        self.conn.create_predictions(predictions_json, timeout=timeout)\n\n    def get_prediction(\n        self,\n        dataset: Union[Dataset, str],\n        model: Union[Model, str],\n        datum: Union[Datum, str],\n    ) -> Union[Prediction, None]:\n        \"\"\"\n        Get a particular prediction.\n\n        Parameters\n        ----------\n        dataset : Union[Dataset, str]\n            The dataset the datum belongs to.\n        model : Union[Model, str]\n            The model that made the prediction.\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[Prediction, None]\n            The matching prediction or 'None' if it doesn't exist.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        model_name = model.name if isinstance(model, Model) else model\n        datum_uid = datum.uid if isinstance(datum, Datum) else datum\n\n        resp = self.conn.get_prediction(\n            dataset_name=dataset_name,  # type: ignore\n            model_name=model_name,  # type: ignore\n            datum_uid=datum_uid,  # type: ignore\n        )\n        resp.pop(\"dataset_name\")\n        resp.pop(\"model_name\")\n        return Prediction.decode_value(resp)\n\n    def finalize_inferences(\n        self, dataset: Union[Dataset, str], model: Union[Model, str]\n    ) -> None:\n        \"\"\"\n        Finalizes a model-dataset pairing such that new predictions cannot be added to it.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        model_name = model.name if isinstance(model, Model) else model\n        return self.conn.finalize_inferences(\n            dataset_name=dataset_name,  # type: ignore\n            model_name=model_name,  # type: ignore\n        )\n\n    def get_model(\n        self,\n        name: str,\n    ) -> Union[Model, None]:\n        \"\"\"\n        Gets a model by name.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model to fetch.\n\n        Returns\n        -------\n        Union[valor.Model, None]\n            A Model with matching name or 'None' if one doesn't exist.\n        \"\"\"\n        return Model.decode_value(\n            {\n                **self.conn.get_model(name),\n                \"connection\": self.conn,\n            }\n        )\n\n    def get_models(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Model]:\n        \"\"\"\n        Get all models using an optional filter.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        ------\n        List[valor.Model]\n            A list of models.\n        \"\"\"\n        model_list = []\n        filters = filters if filters is not None else Filter()\n        for kwargs in self.conn.get_models(filters.to_dict()):\n            model = Model.decode_value({**kwargs, \"connection\": self.conn})\n            model_list.append(model)\n        return model_list\n\n    def get_model_status(\n        self,\n        dataset_name: str,\n        model_name: str,\n    ) -> Optional[TableStatus]:\n        \"\"\"\n        Get the state of a given model over a dataset.\n\n        Parameters\n        ----------\n        dataset_name : str\n            The name of the dataset that the model is operating over.\n        model_name : str\n            The name of the model we want to fetch the state of.\n\n        Returns\n        ------\n        Union[TableStatus, None]\n            The state of the model or 'None' if the model doesn't exist.\n        \"\"\"\n        try:\n            return self.conn.get_model_status(dataset_name, model_name)\n        except ClientException as e:\n            if e.status_code == 404:\n                return None\n            raise e\n\n    def get_model_eval_requests(\n        self, model: Union[Model, str]\n    ) -> List[Evaluation]:\n        \"\"\"\n        Get all evaluations that have been created for a model.\n\n        This does not return evaluation results.\n\n        `GET` endpoint.\n\n        Parameters\n        ----------\n        model : str\n            The model to search by.\n\n        Returns\n        -------\n        List[Evaluation]\n            A list of evaluations.\n        \"\"\"\n        model_name = model.name if isinstance(model, Model) else model\n        return [\n            Evaluation(**evaluation, connection=self.conn)\n            for evaluation in self.conn.get_model_eval_requests(model_name)  # type: ignore\n        ]\n\n    def delete_model(self, name: str, timeout: int = 0) -> None:\n        \"\"\"\n        Deletes a model.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model to be deleted.\n        timeout : int\n            The number of seconds to wait in order to confirm that the model was deleted.\n        \"\"\"\n        self.conn.delete_model(name)\n        if timeout:\n            for _ in range(timeout):\n                try:\n                    self.get_model(name)\n                except ModelDoesNotExistError:\n                    break\n                time.sleep(1)\n            else:\n                raise TimeoutError(\n                    \"Model wasn't deleted within timeout interval\"\n                )\n\n    def get_evaluations(\n        self,\n        *,\n        evaluation_ids: Optional[List[int]] = None,\n        models: Union[List[Model], List[str], None] = None,\n        datasets: Union[List[Dataset], List[str], None] = None,\n        metrics_to_sort_by: Optional[\n            Dict[str, Union[Dict[str, str], str]]\n        ] = None,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Returns all evaluations associated with user-supplied dataset and/or model names.\n\n        Parameters\n        ----------\n        evaluation_ids : List[int], optional.\n            A list of job IDs to return metrics for.\n        models : Union[List[valor.Model], List[str]], optional\n            A list of model names that we want to return metrics for.\n        datasets : Union[List[valor.Dataset], List[str]], optional\n            A list of dataset names that we want to return metrics for.\n        metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n            An optional dict of metric types to sort the evaluations by.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        List[valor.Evaluation]\n            A list of evaluations.\n        \"\"\"\n        if isinstance(datasets, list):\n            datasets = [  # type: ignore\n                element.name if isinstance(element, Dataset) else element\n                for element in datasets\n            ]\n        if isinstance(models, list):\n            models = [  # type: ignore\n                element.name if isinstance(element, Model) else element\n                for element in models\n            ]\n        return [\n            Evaluation(connection=self.conn, **evaluation)\n            for evaluation in self.conn.get_evaluations(\n                evaluation_ids=evaluation_ids,\n                models=models,  # type: ignore\n                datasets=datasets,  # type: ignore\n                metrics_to_sort_by=metrics_to_sort_by,\n                timeout=timeout,\n            )\n        ]\n\n    def evaluate(\n        self,\n        request: EvaluationRequest,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Creates as many evaluations as necessary to fulfill the request.\n\n        Parameters\n        ----------\n        request : schemas.EvaluationRequest\n            The requested evaluation parameters.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        List[Evaluation]\n            A list of evaluations that meet the parameters.\n        \"\"\"\n        return [\n            Evaluation(**evaluation)\n            for evaluation in self.conn.evaluate(\n                request.to_dict(),\n                allow_retries=allow_retries,\n                timeout=timeout,\n            )\n        ]\n\n    def delete_evaluation(self, evaluation_id: int, timeout: int = 0) -> None:\n        \"\"\"\n        Deletes an evaluation.\n\n        Parameters\n        ----------\n        evaluation_id : int\n            The id of the evaluation to be deleted.\n        timeout : int, default=0\n            The number of seconds to wait in order to confirm that the model was deleted.\n        \"\"\"\n        self.conn.delete_evaluation(evaluation_id)\n        if timeout:\n            for _ in range(timeout):\n                try:\n                    self.get_evaluations(evaluation_ids=[evaluation_id])\n                except EvaluationDoesNotExist:\n                    break\n                time.sleep(1)\n            else:\n                raise TimeoutError(\n                    \"Evaluation wasn't deleted within timeout interval\"\n                )\n
"},{"location":"client_api/Client/#valor.Client-functions","title":"Functions","text":""},{"location":"client_api/Client/#valor.Client.connect","title":"valor.Client.connect(host, access_token=None, reconnect=False)  classmethod","text":"
Establishes a connection to the Valor API.
 
Parameters:
 Name Type Description Default host str 
The host to connect to. Should start with \"http://\" or \"https://\".
 required access_token str 
The access token for the host (if the host requires authentication).
 None Source code in valor/coretypes.py 
@classmethod\ndef connect(\n    cls,\n    host: str,\n    access_token: Optional[str] = None,\n    reconnect: bool = False,\n) -> Client:\n    \"\"\"\n    Establishes a connection to the Valor API.\n\n    Parameters\n    ----------\n    host : str\n        The host to connect to. Should start with \"http://\" or \"https://\".\n    access_token : str\n        The access token for the host (if the host requires authentication).\n    \"\"\"\n    connect(host=host, access_token=access_token, reconnect=reconnect)\n    return cls(get_connection())\n
"},{"location":"client_api/Client/#valor.Client.create_dataset","title":"valor.Client.create_dataset(dataset)","text":"
Creates a dataset.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset to create.
 required Source code in valor/coretypes.py 
def create_dataset(\n    self,\n    dataset: Union[Dataset, dict],\n) -> None:\n    \"\"\"\n    Creates a dataset.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset to create.\n    \"\"\"\n    if isinstance(dataset, Dataset):\n        dataset = dataset.encode_value()\n    self.conn.create_dataset(dataset)\n
"},{"location":"client_api/Client/#valor.Client.create_groundtruths","title":"valor.Client.create_groundtruths(dataset, groundtruths, ignore_existing_datums=False, timeout=None)","text":"
Creates ground truths.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset to create the ground truth for.
 required groundtruths List[GroundTruth] 
The ground truths to create.
 required timeout float 
The number of seconds the client should wait until raising a timeout.
 None ignore_existing_datums bool 
If True, will ignore datums that already exist in the backend. If False, will raise an error if any datums already exist. Default is False.
 False Source code in valor/coretypes.py 
def create_groundtruths(\n    self,\n    dataset: Dataset,\n    groundtruths: List[GroundTruth],\n    ignore_existing_datums: bool = False,\n    timeout: Optional[float] = None,\n):\n    \"\"\"\n    Creates ground truths.\n\n    Parameters\n    ----------\n\n    dataset : valor.Dataset\n        The dataset to create the ground truth for.\n    groundtruths : List[valor.GroundTruth]\n        The ground truths to create.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    ignore_existing_datums : bool, default=False\n        If True, will ignore datums that already exist in the backend.\n        If False, will raise an error if any datums already exist.\n        Default is False.\n    \"\"\"\n    groundtruths_json = []\n    for groundtruth in groundtruths:\n        if not isinstance(groundtruth, GroundTruth):\n            raise TypeError(\n                f\"Expected ground truth to be of type 'valor.GroundTruth' not '{type(groundtruth)}'.\"\n            )\n        if not isinstance(groundtruth.annotations._value, list):\n            raise TypeError\n        groundtruth_dict = groundtruth.encode_value()\n        groundtruth_dict[\"dataset_name\"] = dataset.name\n        groundtruths_json.append(groundtruth_dict)\n    self.conn.create_groundtruths(\n        groundtruths_json,\n        timeout=timeout,\n        ignore_existing_datums=ignore_existing_datums,\n    )\n
"},{"location":"client_api/Client/#valor.Client.create_model","title":"valor.Client.create_model(model)","text":"
Creates a model.
 
Parameters:
 Name Type Description Default model Model 
The model to create.
 required Source code in valor/coretypes.py 
def create_model(\n    self,\n    model: Union[Model, dict],\n):\n    \"\"\"\n    Creates a model.\n\n    Parameters\n    ----------\n    model : valor.Model\n        The model to create.\n    \"\"\"\n    if isinstance(model, Model):\n        model = model.encode_value()\n    self.conn.create_model(model)\n
"},{"location":"client_api/Client/#valor.Client.create_predictions","title":"valor.Client.create_predictions(dataset, model, predictions, timeout=None)","text":"
Creates predictions.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset that is being operated over.
 required model Model 
The model making the prediction.
 required predictions List[Prediction] 
The predictions to create.
 required timeout float 
The number of seconds the client should wait until raising a timeout.
 None Source code in valor/coretypes.py 
def create_predictions(\n    self,\n    dataset: Dataset,\n    model: Model,\n    predictions: List[Prediction],\n    timeout: Optional[float] = None,\n) -> None:\n    \"\"\"\n    Creates predictions.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset that is being operated over.\n    model : valor.Model\n        The model making the prediction.\n    predictions : List[valor.Prediction]\n        The predictions to create.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    \"\"\"\n    predictions_json = []\n    for prediction in predictions:\n        if not isinstance(prediction, Prediction):\n            raise TypeError(\n                f\"Expected prediction to be of type 'valor.Prediction' not '{type(prediction)}'.\"\n            )\n        if not isinstance(prediction.annotations._value, list):\n            raise TypeError\n        prediction_dict = prediction.encode_value()\n        prediction_dict[\"dataset_name\"] = dataset.name\n        prediction_dict[\"model_name\"] = model.name\n        predictions_json.append(prediction_dict)\n    self.conn.create_predictions(predictions_json, timeout=timeout)\n
"},{"location":"client_api/Client/#valor.Client.delete_dataset","title":"valor.Client.delete_dataset(name, timeout=0)","text":"
Deletes a dataset.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset to be deleted.
 required timeout int 
The number of seconds to wait in order to confirm that the dataset was deleted.
 0 Source code in valor/coretypes.py 
def delete_dataset(self, name: str, timeout: int = 0) -> None:\n    \"\"\"\n    Deletes a dataset.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset to be deleted.\n    timeout : int\n        The number of seconds to wait in order to confirm that the dataset was deleted.\n    \"\"\"\n    self.conn.delete_dataset(name)\n    if timeout:\n        for _ in range(timeout):\n            try:\n                self.get_dataset(name)\n            except DatasetDoesNotExistError:\n                break\n            time.sleep(1)\n        else:\n            raise TimeoutError(\n                \"Dataset wasn't deleted within timeout interval\"\n            )\n
"},{"location":"client_api/Client/#valor.Client.delete_evaluation","title":"valor.Client.delete_evaluation(evaluation_id, timeout=0)","text":"
Deletes an evaluation.
 
Parameters:
 Name Type Description Default evaluation_id int 
The id of the evaluation to be deleted.
 required timeout int 
The number of seconds to wait in order to confirm that the model was deleted.
 0 Source code in valor/coretypes.py 
def delete_evaluation(self, evaluation_id: int, timeout: int = 0) -> None:\n    \"\"\"\n    Deletes an evaluation.\n\n    Parameters\n    ----------\n    evaluation_id : int\n        The id of the evaluation to be deleted.\n    timeout : int, default=0\n        The number of seconds to wait in order to confirm that the model was deleted.\n    \"\"\"\n    self.conn.delete_evaluation(evaluation_id)\n    if timeout:\n        for _ in range(timeout):\n            try:\n                self.get_evaluations(evaluation_ids=[evaluation_id])\n            except EvaluationDoesNotExist:\n                break\n            time.sleep(1)\n        else:\n            raise TimeoutError(\n                \"Evaluation wasn't deleted within timeout interval\"\n            )\n
"},{"location":"client_api/Client/#valor.Client.delete_model","title":"valor.Client.delete_model(name, timeout=0)","text":"
Deletes a model.
 
Parameters:
 Name Type Description Default name str 
The name of the model to be deleted.
 required timeout int 
The number of seconds to wait in order to confirm that the model was deleted.
 0 Source code in valor/coretypes.py 
def delete_model(self, name: str, timeout: int = 0) -> None:\n    \"\"\"\n    Deletes a model.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model to be deleted.\n    timeout : int\n        The number of seconds to wait in order to confirm that the model was deleted.\n    \"\"\"\n    self.conn.delete_model(name)\n    if timeout:\n        for _ in range(timeout):\n            try:\n                self.get_model(name)\n            except ModelDoesNotExistError:\n                break\n            time.sleep(1)\n        else:\n            raise TimeoutError(\n                \"Model wasn't deleted within timeout interval\"\n            )\n
"},{"location":"client_api/Client/#valor.Client.evaluate","title":"valor.Client.evaluate(request, *_, allow_retries=False, timeout=None)","text":"
Creates as many evaluations as necessary to fulfill the request.
 
Parameters:
 Name Type Description Default request EvaluationRequest 
The requested evaluation parameters.
 required allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of evaluations that meet the parameters.
 Source code in valor/coretypes.py 
def evaluate(\n    self,\n    request: EvaluationRequest,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Creates as many evaluations as necessary to fulfill the request.\n\n    Parameters\n    ----------\n    request : schemas.EvaluationRequest\n        The requested evaluation parameters.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    List[Evaluation]\n        A list of evaluations that meet the parameters.\n    \"\"\"\n    return [\n        Evaluation(**evaluation)\n        for evaluation in self.conn.evaluate(\n            request.to_dict(),\n            allow_retries=allow_retries,\n            timeout=timeout,\n        )\n    ]\n
"},{"location":"client_api/Client/#valor.Client.finalize_dataset","title":"valor.Client.finalize_dataset(dataset)","text":"
Finalizes a dataset such that new ground truths cannot be added to it.
 
Parameters:
 Name Type Description Default dataset str 
The dataset to be finalized.
 required Source code in valor/coretypes.py 
def finalize_dataset(self, dataset: Union[Dataset, str]) -> None:\n    \"\"\"\n    Finalizes a dataset such that new ground truths cannot be added to it.\n\n    Parameters\n    ----------\n    dataset : str\n        The dataset to be finalized.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    return self.conn.finalize_dataset(name=dataset_name)  # type: ignore\n
"},{"location":"client_api/Client/#valor.Client.finalize_inferences","title":"valor.Client.finalize_inferences(dataset, model)","text":"
Finalizes a model-dataset pairing such that new predictions cannot be added to it.
 Source code in valor/coretypes.py 
def finalize_inferences(\n    self, dataset: Union[Dataset, str], model: Union[Model, str]\n) -> None:\n    \"\"\"\n    Finalizes a model-dataset pairing such that new predictions cannot be added to it.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    model_name = model.name if isinstance(model, Model) else model\n    return self.conn.finalize_inferences(\n        dataset_name=dataset_name,  # type: ignore\n        model_name=model_name,  # type: ignore\n    )\n
"},{"location":"client_api/Client/#valor.Client.get_dataset","title":"valor.Client.get_dataset(name)","text":"
Gets a dataset by name.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset to fetch.
 required 
Returns:
 Type Description Union[Dataset, None] 
A Dataset with a matching name, or 'None' if one doesn't exist.
 Source code in valor/coretypes.py 
def get_dataset(\n    self,\n    name: str,\n) -> Union[Dataset, None]:\n    \"\"\"\n    Gets a dataset by name.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset to fetch.\n\n    Returns\n    -------\n    Union[Dataset, None]\n        A Dataset with a matching name, or 'None' if one doesn't exist.\n    \"\"\"\n    dataset = Dataset.decode_value(\n        {\n            **self.conn.get_dataset(name),\n            \"connection\": self.conn,\n        }\n    )\n    return dataset\n
"},{"location":"client_api/Client/#valor.Client.get_dataset_status","title":"valor.Client.get_dataset_status(name)","text":"
Get the state of a given dataset.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset we want to fetch the state of.
 required 
Returns:
 Type Description TableStatus | None 
The state of the dataset, or 'None' if the dataset does not exist.
 Source code in valor/coretypes.py 
def get_dataset_status(\n    self,\n    name: str,\n) -> Union[TableStatus, None]:\n    \"\"\"\n    Get the state of a given dataset.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset we want to fetch the state of.\n\n    Returns\n    ------\n    TableStatus | None\n        The state of the dataset, or 'None' if the dataset does not exist.\n    \"\"\"\n    try:\n        return self.conn.get_dataset_status(name)\n    except ClientException as e:\n        if e.status_code == 404:\n            return None\n        raise e\n
"},{"location":"client_api/Client/#valor.Client.get_dataset_summary","title":"valor.Client.get_dataset_summary(name, *_, timeout=None)","text":"
Gets the summary of a dataset.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset to create a summary for.
 required 
Returns:
 Type Description DatasetSummary 
A dataclass containing the dataset summary.
 Source code in valor/coretypes.py 
def get_dataset_summary(\n    self,\n    name: str,\n    *_,\n    timeout: Optional[float] = None,\n) -> DatasetSummary:\n    \"\"\"\n    Gets the summary of a dataset.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset to create a summary for.\n\n    Returns\n    -------\n    DatasetSummary\n        A dataclass containing the dataset summary.\n    \"\"\"\n    return DatasetSummary(\n        **self.conn.get_dataset_summary(name, timeout=timeout)\n    )\n
"},{"location":"client_api/Client/#valor.Client.get_datasets","title":"valor.Client.get_datasets(filters=None)","text":"
Get all datasets, with an option to filter results according to some user-defined parameters.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Dataset] 
A list of datasets.
 Source code in valor/coretypes.py 
def get_datasets(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Dataset]:\n    \"\"\"\n    Get all datasets, with an option to filter results according to some user-defined parameters.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    ------\n    List[valor.Dataset]\n        A list of datasets.\n    \"\"\"\n    dataset_list = []\n    filters = filters if filters is not None else Filter()\n    for kwargs in self.conn.get_datasets(filters.to_dict()):\n        dataset = Dataset.decode_value({**kwargs, \"connection\": self.conn})\n        dataset_list.append(dataset)\n    return dataset_list\n
"},{"location":"client_api/Client/#valor.Client.get_datum","title":"valor.Client.get_datum(dataset, uid)","text":"
Get datum. GET endpoint.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset the datum belongs to.
 required uid str 
The UID of the datum.
 required 
Returns:
 Type Description Datum 
The requested datum or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_datum(\n    self,\n    dataset: Union[Dataset, str],\n    uid: str,\n) -> Union[Datum, None]:\n    \"\"\"\n    Get datum.\n    `GET` endpoint.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset the datum belongs to.\n    uid : str\n        The UID of the datum.\n\n    Returns\n    -------\n    valor.Datum\n        The requested datum or 'None' if it doesn't exist.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    resp = self.conn.get_datum(dataset_name=dataset_name, uid=uid)  # type: ignore\n    return Datum.decode_value(resp)\n
"},{"location":"client_api/Client/#valor.Client.get_datums","title":"valor.Client.get_datums(filters=None)","text":"
Get all datums using an optional filter.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Datum] 
A list datums.
 Source code in valor/coretypes.py 
def get_datums(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Datum]:\n    \"\"\"\n    Get all datums using an optional filter.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    -------\n    List[valor.Datum]\n        A list datums.\n    \"\"\"\n\n    filters = filters if filters is not None else Filter()\n    return [\n        Datum.decode_value(datum)\n        for datum in self.conn.get_datums(filters.to_dict())\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_evaluations","title":"valor.Client.get_evaluations(*, evaluation_ids=None, models=None, datasets=None, metrics_to_sort_by=None, timeout=None)","text":"
Returns all evaluations associated with user-supplied dataset and/or model names.
 
Parameters:
 Name Type Description Default evaluation_ids List[int], optional. 
A list of job IDs to return metrics for.
 None models Union[List[Model], List[str]] 
A list of model names that we want to return metrics for.
 None datasets Union[List[Dataset], List[str]] 
A list of dataset names that we want to return metrics for.
 None metrics_to_sort_by dict[str, str | dict[str, str]] 
An optional dict of metric types to sort the evaluations by.
 None timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of evaluations.
 Source code in valor/coretypes.py 
def get_evaluations(\n    self,\n    *,\n    evaluation_ids: Optional[List[int]] = None,\n    models: Union[List[Model], List[str], None] = None,\n    datasets: Union[List[Dataset], List[str], None] = None,\n    metrics_to_sort_by: Optional[\n        Dict[str, Union[Dict[str, str], str]]\n    ] = None,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Returns all evaluations associated with user-supplied dataset and/or model names.\n\n    Parameters\n    ----------\n    evaluation_ids : List[int], optional.\n        A list of job IDs to return metrics for.\n    models : Union[List[valor.Model], List[str]], optional\n        A list of model names that we want to return metrics for.\n    datasets : Union[List[valor.Dataset], List[str]], optional\n        A list of dataset names that we want to return metrics for.\n    metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n        An optional dict of metric types to sort the evaluations by.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    List[valor.Evaluation]\n        A list of evaluations.\n    \"\"\"\n    if isinstance(datasets, list):\n        datasets = [  # type: ignore\n            element.name if isinstance(element, Dataset) else element\n            for element in datasets\n        ]\n    if isinstance(models, list):\n        models = [  # type: ignore\n            element.name if isinstance(element, Model) else element\n            for element in models\n        ]\n    return [\n        Evaluation(connection=self.conn, **evaluation)\n        for evaluation in self.conn.get_evaluations(\n            evaluation_ids=evaluation_ids,\n            models=models,  # type: ignore\n            datasets=datasets,  # type: ignore\n            metrics_to_sort_by=metrics_to_sort_by,\n            timeout=timeout,\n        )\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_groundtruth","title":"valor.Client.get_groundtruth(dataset, datum)","text":"
Get a particular ground truth.
 
Parameters:
 Name Type Description Default dataset Union[Dataset, str] 
The dataset the datum belongs to.
 required datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[GroundTruth, None] 
The matching ground truth or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_groundtruth(\n    self,\n    dataset: Union[Dataset, str],\n    datum: Union[Datum, str],\n) -> Union[GroundTruth, None]:\n    \"\"\"\n    Get a particular ground truth.\n\n    Parameters\n    ----------\n    dataset : Union[Dataset, str]\n        The dataset the datum belongs to.\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[GroundTruth, None]\n        The matching ground truth or 'None' if it doesn't exist.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    datum_uid = datum.uid if isinstance(datum, Datum) else datum\n    try:\n        resp = self.conn.get_groundtruth(\n            dataset_name=dataset_name, datum_uid=datum_uid  # type: ignore\n        )\n        resp.pop(\"dataset_name\")\n        return GroundTruth.decode_value(resp)\n    except ClientException as e:\n        if e.status_code == 404:\n            return None\n        raise e\n
"},{"location":"client_api/Client/#valor.Client.get_labels","title":"valor.Client.get_labels(filters=None)","text":"
Gets all labels using an optional filter.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Label] 
A list of labels.
 Source code in valor/coretypes.py 
def get_labels(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Label]:\n    \"\"\"\n    Gets all labels using an optional filter.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    ------\n    List[valor.Label]\n        A list of labels.\n    \"\"\"\n    filters = filters if filters is not None else Filter()\n    return [\n        Label(**label) for label in self.conn.get_labels(filters.to_dict())\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_labels_from_dataset","title":"valor.Client.get_labels_from_dataset(dataset)","text":"
Get all labels associated with a dataset's ground truths.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset to search by.
 required 
Returns:
 Type Description List[Label] 
A list of labels.
 Source code in valor/coretypes.py 
def get_labels_from_dataset(\n    self, dataset: Union[Dataset, str]\n) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a dataset's ground truths.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset to search by.\n\n    Returns\n    ------\n    List[valor.Label]\n        A list of labels.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    return [\n        Label(**label)\n        for label in self.conn.get_labels_from_dataset(dataset_name)  # type: ignore\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_labels_from_model","title":"valor.Client.get_labels_from_model(model)","text":"
Get all labels associated with a model's ground truths.
 
Parameters:
 Name Type Description Default model Model 
The model to search by.
 required 
Returns:
 Type Description List[Label] 
A list of labels.
 Source code in valor/coretypes.py 
def get_labels_from_model(self, model: Union[Model, str]) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a model's ground truths.\n\n    Parameters\n    ----------\n    model : valor.Model\n        The model to search by.\n\n    Returns\n    ------\n    List[valor.Label]\n        A list of labels.\n    \"\"\"\n    model_name = model.name if isinstance(model, Model) else model\n    return [\n        Label(**label)\n        for label in self.conn.get_labels_from_model(model_name)  # type: ignore\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_model","title":"valor.Client.get_model(name)","text":"
Gets a model by name.
 
Parameters:
 Name Type Description Default name str 
The name of the model to fetch.
 required 
Returns:
 Type Description Union[Model, None] 
A Model with matching name or 'None' if one doesn't exist.
 Source code in valor/coretypes.py 
def get_model(\n    self,\n    name: str,\n) -> Union[Model, None]:\n    \"\"\"\n    Gets a model by name.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model to fetch.\n\n    Returns\n    -------\n    Union[valor.Model, None]\n        A Model with matching name or 'None' if one doesn't exist.\n    \"\"\"\n    return Model.decode_value(\n        {\n            **self.conn.get_model(name),\n            \"connection\": self.conn,\n        }\n    )\n
"},{"location":"client_api/Client/#valor.Client.get_model_eval_requests","title":"valor.Client.get_model_eval_requests(model)","text":"
Get all evaluations that have been created for a model.
 
This does not return evaluation results.
 
GET endpoint.
 
Parameters:
 Name Type Description Default model str 
The model to search by.
 required 
Returns:
 Type Description List[Evaluation] 
A list of evaluations.
 Source code in valor/coretypes.py 
def get_model_eval_requests(\n    self, model: Union[Model, str]\n) -> List[Evaluation]:\n    \"\"\"\n    Get all evaluations that have been created for a model.\n\n    This does not return evaluation results.\n\n    `GET` endpoint.\n\n    Parameters\n    ----------\n    model : str\n        The model to search by.\n\n    Returns\n    -------\n    List[Evaluation]\n        A list of evaluations.\n    \"\"\"\n    model_name = model.name if isinstance(model, Model) else model\n    return [\n        Evaluation(**evaluation, connection=self.conn)\n        for evaluation in self.conn.get_model_eval_requests(model_name)  # type: ignore\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_model_status","title":"valor.Client.get_model_status(dataset_name, model_name)","text":"
Get the state of a given model over a dataset.
 
Parameters:
 Name Type Description Default dataset_name str 
The name of the dataset that the model is operating over.
 required model_name str 
The name of the model we want to fetch the state of.
 required 
Returns:
 Type Description Union[TableStatus, None] 
The state of the model or 'None' if the model doesn't exist.
 Source code in valor/coretypes.py 
def get_model_status(\n    self,\n    dataset_name: str,\n    model_name: str,\n) -> Optional[TableStatus]:\n    \"\"\"\n    Get the state of a given model over a dataset.\n\n    Parameters\n    ----------\n    dataset_name : str\n        The name of the dataset that the model is operating over.\n    model_name : str\n        The name of the model we want to fetch the state of.\n\n    Returns\n    ------\n    Union[TableStatus, None]\n        The state of the model or 'None' if the model doesn't exist.\n    \"\"\"\n    try:\n        return self.conn.get_model_status(dataset_name, model_name)\n    except ClientException as e:\n        if e.status_code == 404:\n            return None\n        raise e\n
"},{"location":"client_api/Client/#valor.Client.get_models","title":"valor.Client.get_models(filters=None)","text":"
Get all models using an optional filter.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Model] 
A list of models.
 Source code in valor/coretypes.py 
def get_models(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Model]:\n    \"\"\"\n    Get all models using an optional filter.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    ------\n    List[valor.Model]\n        A list of models.\n    \"\"\"\n    model_list = []\n    filters = filters if filters is not None else Filter()\n    for kwargs in self.conn.get_models(filters.to_dict()):\n        model = Model.decode_value({**kwargs, \"connection\": self.conn})\n        model_list.append(model)\n    return model_list\n
"},{"location":"client_api/Client/#valor.Client.get_prediction","title":"valor.Client.get_prediction(dataset, model, datum)","text":"
Get a particular prediction.
 
Parameters:
 Name Type Description Default dataset Union[Dataset, str] 
The dataset the datum belongs to.
 required model Union[Model, str] 
The model that made the prediction.
 required datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[Prediction, None] 
The matching prediction or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_prediction(\n    self,\n    dataset: Union[Dataset, str],\n    model: Union[Model, str],\n    datum: Union[Datum, str],\n) -> Union[Prediction, None]:\n    \"\"\"\n    Get a particular prediction.\n\n    Parameters\n    ----------\n    dataset : Union[Dataset, str]\n        The dataset the datum belongs to.\n    model : Union[Model, str]\n        The model that made the prediction.\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[Prediction, None]\n        The matching prediction or 'None' if it doesn't exist.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    model_name = model.name if isinstance(model, Model) else model\n    datum_uid = datum.uid if isinstance(datum, Datum) else datum\n\n    resp = self.conn.get_prediction(\n        dataset_name=dataset_name,  # type: ignore\n        model_name=model_name,  # type: ignore\n        datum_uid=datum_uid,  # type: ignore\n    )\n    resp.pop(\"dataset_name\")\n    resp.pop(\"model_name\")\n    return Prediction.decode_value(resp)\n
"},{"location":"client_api/Dataset/","title":"Dataset","text":"
               Bases: StaticCollection
 
A class describing a given dataset.
 
Attributes:
 Name Type Description name String 
The name of the dataset.
 metadata Dictionary 
A dictionary of metadata that describes the dataset.
 
Examples:
 
>>> Dataset.create(name=\"dataset1\")\n>>> Dataset.create(name=\"dataset1\", metadata={})\n>>> Dataset.create(name=\"dataset1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n
 Source code in valor/coretypes.py 
class Dataset(StaticCollection):\n    \"\"\"\n    A class describing a given dataset.\n\n    Attributes\n    ----------\n    name : String\n        The name of the dataset.\n    metadata : Dictionary\n        A dictionary of metadata that describes the dataset.\n\n    Examples\n    --------\n    >>> Dataset.create(name=\"dataset1\")\n    >>> Dataset.create(name=\"dataset1\", metadata={})\n    >>> Dataset.create(name=\"dataset1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n    \"\"\"\n\n    name: String = String.symbolic(owner=\"dataset\", name=\"name\")\n    metadata: Dictionary = Dictionary.symbolic(\n        owner=\"dataset\", name=\"metadata\"\n    )\n\n    def __init__(\n        self,\n        *,\n        name: str,\n        metadata: Optional[dict] = None,\n        connection: Optional[ClientConnection] = None,\n    ):\n        \"\"\"\n        Creates a local instance of a dataset.\n\n        Use 'Dataset.create' classmethod to create a dataset with persistence.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset.\n        metadata : dict, optional\n            A dictionary of metadata that describes the dataset.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        self.conn = connection\n        super().__init__(name=name, metadata=metadata if metadata else dict())\n\n    @classmethod\n    def create(\n        cls,\n        name: str,\n        metadata: Optional[Dict[str, Any]] = None,\n        connection: Optional[ClientConnection] = None,\n    ) -> Dataset:\n        \"\"\"\n        Creates a dataset that persists in the back end.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset.\n        metadata : dict, optional\n            A dictionary of metadata that describes the dataset.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        dataset = cls(name=name, metadata=metadata, connection=connection)\n        Client(dataset.conn).create_dataset(dataset)\n        return dataset\n\n    @classmethod\n    def get(\n        cls,\n        name: str,\n        connection: Optional[ClientConnection] = None,\n    ) -> Union[Dataset, None]:\n        \"\"\"\n        Retrieves a dataset from the back end database.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset.\n\n        Returns\n        -------\n        Union[valor.Dataset, None]\n            The dataset or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(connection).get_dataset(name)\n\n    def add_groundtruth(\n        self,\n        groundtruth: GroundTruth,\n    ) -> None:\n        \"\"\"\n        Add a ground truth to the dataset.\n\n        Parameters\n        ----------\n        groundtruth : GroundTruth\n            The ground truth to create.\n        \"\"\"\n        Client(self.conn).create_groundtruths(\n            dataset=self,\n            groundtruths=[groundtruth],\n        )\n\n    def add_groundtruths(\n        self,\n        groundtruths: List[GroundTruth],\n        ignore_existing_datums: bool = False,\n        timeout: Optional[float] = 10.0,\n    ) -> None:\n        \"\"\"\n        Add multiple ground truths to the dataset.\n\n        Parameters\n        ----------\n        groundtruths : List[GroundTruth]\n            The ground truths to create.\n        ignore_existing_datums : bool, default=False\n            If True, will ignore datums that already exist in the backend.\n            If False, will raise an error if any datums already exist.\n            Default is False.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        \"\"\"\n        Client(self.conn).create_groundtruths(\n            dataset=self,\n            groundtruths=groundtruths,\n            ignore_existing_datums=ignore_existing_datums,\n            timeout=timeout,\n        )\n\n    def get_groundtruth(\n        self,\n        datum: Union[Datum, str],\n    ) -> Union[GroundTruth, None]:\n        \"\"\"\n        Get a particular ground truth.\n\n        Parameters\n        ----------\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[GroundTruth, None]\n            The matching ground truth or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(self.conn).get_groundtruth(dataset=self, datum=datum)\n\n    def get_labels(\n        self,\n    ) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a given dataset.\n\n        Returns\n        ----------\n        List[Label]\n            A list of `Labels` associated with the dataset.\n        \"\"\"\n        return Client(self.conn).get_labels_from_dataset(self)\n\n    def get_datums(self, filters: Optional[Filter] = None) -> List[Datum]:\n        \"\"\"\n        Get all datums associated with a given dataset.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            An optional datum filter.\n\n        Returns\n        ----------\n        List[Datum]\n            A list of `Datums` associated with the dataset.\n        \"\"\"\n        if filters is None:\n            filters = Filter()\n        filters.datasets = Dataset.name == self.name  # type: ignore - #issue 605\n        return Client(self.conn).get_datums(filters=filters)\n\n    def get_evaluations(\n        self,\n        metrics_to_sort_by: Optional[\n            Dict[str, Union[Dict[str, str], str]]\n        ] = None,\n        *_,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Get all evaluations associated with a given dataset.\n\n        Parameters\n        ----------\n        metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n            An optional dict of metric types to sort the evaluations by.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        ----------\n        List[Evaluation]\n            A list of `Evaluations` associated with the dataset.\n        \"\"\"\n        return Client(self.conn).get_evaluations(\n            datasets=[self],\n            metrics_to_sort_by=metrics_to_sort_by,\n            timeout=timeout,\n        )\n\n    def get_summary(\n        self, *_, timeout: Optional[float] = None\n    ) -> DatasetSummary:\n        \"\"\"\n        Get the summary of a given dataset.\n\n        Parameters\n        ----------\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        DatasetSummary\n            The summary of the dataset. This class has the following fields:\n\n            name : name of the dataset\n\n            num_datums : total number of datums in the dataset\n\n            num_annotations : total number of labeled annotations in the dataset; if an\n            object (such as a bounding box) has multiple labels, then each label is counted separately\n\n            num_bounding_boxes : total number of bounding boxes in the dataset\n\n            num_polygons : total number of polygons in the dataset\n\n            num_rasters : total number of rasters in the dataset\n\n            labels : list of the unique labels in the dataset\n\n            datum_metadata : list of the unique metadata dictionaries in the dataset that are associated\n            to datums\n\n            groundtruth_annotation_metadata : list of the unique metadata dictionaries in the dataset that are\n            associated to annotations\n        \"\"\"\n        return Client(self.conn).get_dataset_summary(self.name, timeout=timeout)  # type: ignore\n\n    def finalize(\n        self,\n    ):\n        \"\"\"\n        Finalizes the dataset such that new ground truths cannot be added to it.\n        \"\"\"\n        return Client(self.conn).finalize_dataset(self)\n\n    def delete(\n        self,\n        timeout: int = 0,\n    ):\n        \"\"\"\n        Delete the dataset from the back end.\n\n        Parameters\n        ----------\n        timeout : int, default=0\n            Sets a timeout in seconds.\n        \"\"\"\n        Client(self.conn).delete_dataset(self.name, timeout)  # type: ignore\n
"},{"location":"client_api/Dataset/#valor.Dataset-functions","title":"Functions","text":""},{"location":"client_api/Dataset/#valor.Dataset.__init__","title":"valor.Dataset.__init__(*, name, metadata=None, connection=None)","text":"
Creates a local instance of a dataset.
 
Use 'Dataset.create' classmethod to create a dataset with persistence.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset.
 required metadata dict 
A dictionary of metadata that describes the dataset.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    name: str,\n    metadata: Optional[dict] = None,\n    connection: Optional[ClientConnection] = None,\n):\n    \"\"\"\n    Creates a local instance of a dataset.\n\n    Use 'Dataset.create' classmethod to create a dataset with persistence.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset.\n    metadata : dict, optional\n        A dictionary of metadata that describes the dataset.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    self.conn = connection\n    super().__init__(name=name, metadata=metadata if metadata else dict())\n
"},{"location":"client_api/Dataset/#valor.Dataset.add_groundtruth","title":"valor.Dataset.add_groundtruth(groundtruth)","text":"
Add a ground truth to the dataset.
 
Parameters:
 Name Type Description Default groundtruth GroundTruth 
The ground truth to create.
 required Source code in valor/coretypes.py 
def add_groundtruth(\n    self,\n    groundtruth: GroundTruth,\n) -> None:\n    \"\"\"\n    Add a ground truth to the dataset.\n\n    Parameters\n    ----------\n    groundtruth : GroundTruth\n        The ground truth to create.\n    \"\"\"\n    Client(self.conn).create_groundtruths(\n        dataset=self,\n        groundtruths=[groundtruth],\n    )\n
"},{"location":"client_api/Dataset/#valor.Dataset.add_groundtruths","title":"valor.Dataset.add_groundtruths(groundtruths, ignore_existing_datums=False, timeout=10.0)","text":"
Add multiple ground truths to the dataset.
 
Parameters:
 Name Type Description Default groundtruths List[GroundTruth] 
The ground truths to create.
 required ignore_existing_datums bool 
If True, will ignore datums that already exist in the backend. If False, will raise an error if any datums already exist. Default is False.
 False timeout float 
The number of seconds the client should wait until raising a timeout.
 10.0 Source code in valor/coretypes.py 
def add_groundtruths(\n    self,\n    groundtruths: List[GroundTruth],\n    ignore_existing_datums: bool = False,\n    timeout: Optional[float] = 10.0,\n) -> None:\n    \"\"\"\n    Add multiple ground truths to the dataset.\n\n    Parameters\n    ----------\n    groundtruths : List[GroundTruth]\n        The ground truths to create.\n    ignore_existing_datums : bool, default=False\n        If True, will ignore datums that already exist in the backend.\n        If False, will raise an error if any datums already exist.\n        Default is False.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    \"\"\"\n    Client(self.conn).create_groundtruths(\n        dataset=self,\n        groundtruths=groundtruths,\n        ignore_existing_datums=ignore_existing_datums,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Dataset/#valor.Dataset.create","title":"valor.Dataset.create(name, metadata=None, connection=None)  classmethod","text":"
Creates a dataset that persists in the back end.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset.
 required metadata dict 
A dictionary of metadata that describes the dataset.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
@classmethod\ndef create(\n    cls,\n    name: str,\n    metadata: Optional[Dict[str, Any]] = None,\n    connection: Optional[ClientConnection] = None,\n) -> Dataset:\n    \"\"\"\n    Creates a dataset that persists in the back end.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset.\n    metadata : dict, optional\n        A dictionary of metadata that describes the dataset.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    dataset = cls(name=name, metadata=metadata, connection=connection)\n    Client(dataset.conn).create_dataset(dataset)\n    return dataset\n
"},{"location":"client_api/Dataset/#valor.Dataset.delete","title":"valor.Dataset.delete(timeout=0)","text":"
Delete the dataset from the back end.
 
Parameters:
 Name Type Description Default timeout int 
Sets a timeout in seconds.
 0 Source code in valor/coretypes.py 
def delete(\n    self,\n    timeout: int = 0,\n):\n    \"\"\"\n    Delete the dataset from the back end.\n\n    Parameters\n    ----------\n    timeout : int, default=0\n        Sets a timeout in seconds.\n    \"\"\"\n    Client(self.conn).delete_dataset(self.name, timeout)  # type: ignore\n
"},{"location":"client_api/Dataset/#valor.Dataset.finalize","title":"valor.Dataset.finalize()","text":"
Finalizes the dataset such that new ground truths cannot be added to it.
 Source code in valor/coretypes.py 
def finalize(\n    self,\n):\n    \"\"\"\n    Finalizes the dataset such that new ground truths cannot be added to it.\n    \"\"\"\n    return Client(self.conn).finalize_dataset(self)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get","title":"valor.Dataset.get(name, connection=None)  classmethod","text":"
Retrieves a dataset from the back end database.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset.
 required 
Returns:
 Type Description Union[Dataset, None] 
The dataset or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
@classmethod\ndef get(\n    cls,\n    name: str,\n    connection: Optional[ClientConnection] = None,\n) -> Union[Dataset, None]:\n    \"\"\"\n    Retrieves a dataset from the back end database.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset.\n\n    Returns\n    -------\n    Union[valor.Dataset, None]\n        The dataset or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(connection).get_dataset(name)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_datums","title":"valor.Dataset.get_datums(filters=None)","text":"
Get all datums associated with a given dataset.
 
Parameters:
 Name Type Description Default filters Filter 
An optional datum filter.
 None 
Returns:
 Type Description List[Datum] 
A list of Datums associated with the dataset.
 Source code in valor/coretypes.py 
def get_datums(self, filters: Optional[Filter] = None) -> List[Datum]:\n    \"\"\"\n    Get all datums associated with a given dataset.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        An optional datum filter.\n\n    Returns\n    ----------\n    List[Datum]\n        A list of `Datums` associated with the dataset.\n    \"\"\"\n    if filters is None:\n        filters = Filter()\n    filters.datasets = Dataset.name == self.name  # type: ignore - #issue 605\n    return Client(self.conn).get_datums(filters=filters)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_evaluations","title":"valor.Dataset.get_evaluations(metrics_to_sort_by=None, *_, timeout=None)","text":"
Get all evaluations associated with a given dataset.
 
Parameters:
 Name Type Description Default metrics_to_sort_by dict[str, str | dict[str, str]] 
An optional dict of metric types to sort the evaluations by.
 None timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of Evaluations associated with the dataset.
 Source code in valor/coretypes.py 
def get_evaluations(\n    self,\n    metrics_to_sort_by: Optional[\n        Dict[str, Union[Dict[str, str], str]]\n    ] = None,\n    *_,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Get all evaluations associated with a given dataset.\n\n    Parameters\n    ----------\n    metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n        An optional dict of metric types to sort the evaluations by.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    ----------\n    List[Evaluation]\n        A list of `Evaluations` associated with the dataset.\n    \"\"\"\n    return Client(self.conn).get_evaluations(\n        datasets=[self],\n        metrics_to_sort_by=metrics_to_sort_by,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_groundtruth","title":"valor.Dataset.get_groundtruth(datum)","text":"
Get a particular ground truth.
 
Parameters:
 Name Type Description Default datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[GroundTruth, None] 
The matching ground truth or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_groundtruth(\n    self,\n    datum: Union[Datum, str],\n) -> Union[GroundTruth, None]:\n    \"\"\"\n    Get a particular ground truth.\n\n    Parameters\n    ----------\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[GroundTruth, None]\n        The matching ground truth or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(self.conn).get_groundtruth(dataset=self, datum=datum)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_labels","title":"valor.Dataset.get_labels()","text":"
Get all labels associated with a given dataset.
 
Returns:
 Type Description List[Label] 
A list of Labels associated with the dataset.
 Source code in valor/coretypes.py 
def get_labels(\n    self,\n) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a given dataset.\n\n    Returns\n    ----------\n    List[Label]\n        A list of `Labels` associated with the dataset.\n    \"\"\"\n    return Client(self.conn).get_labels_from_dataset(self)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_summary","title":"valor.Dataset.get_summary(*_, timeout=None)","text":"
Get the summary of a given dataset.
 
Parameters:
 Name Type Description Default timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description DatasetSummary 
The summary of the dataset. This class has the following fields:
 
name : name of the dataset
 
num_datums : total number of datums in the dataset
 
num_annotations : total number of labeled annotations in the dataset; if an object (such as a bounding box) has multiple labels, then each label is counted separately
 
num_bounding_boxes : total number of bounding boxes in the dataset
 
num_polygons : total number of polygons in the dataset
 
num_rasters : total number of rasters in the dataset
 
labels : list of the unique labels in the dataset
 
datum_metadata : list of the unique metadata dictionaries in the dataset that are associated to datums
 
groundtruth_annotation_metadata : list of the unique metadata dictionaries in the dataset that are associated to annotations
 Source code in valor/coretypes.py 
def get_summary(\n    self, *_, timeout: Optional[float] = None\n) -> DatasetSummary:\n    \"\"\"\n    Get the summary of a given dataset.\n\n    Parameters\n    ----------\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    DatasetSummary\n        The summary of the dataset. This class has the following fields:\n\n        name : name of the dataset\n\n        num_datums : total number of datums in the dataset\n\n        num_annotations : total number of labeled annotations in the dataset; if an\n        object (such as a bounding box) has multiple labels, then each label is counted separately\n\n        num_bounding_boxes : total number of bounding boxes in the dataset\n\n        num_polygons : total number of polygons in the dataset\n\n        num_rasters : total number of rasters in the dataset\n\n        labels : list of the unique labels in the dataset\n\n        datum_metadata : list of the unique metadata dictionaries in the dataset that are associated\n        to datums\n\n        groundtruth_annotation_metadata : list of the unique metadata dictionaries in the dataset that are\n        associated to annotations\n    \"\"\"\n    return Client(self.conn).get_dataset_summary(self.name, timeout=timeout)  # type: ignore\n
"},{"location":"client_api/Datum/","title":"Datum","text":"
               Bases: StaticCollection
 
A class used to store information about a datum for either a 'GroundTruth' or a 'Prediction'.
 
Attributes:
 Name Type Description uid String 
The UID of the datum.
 text (String, optional) 
The text of the datum, if the datum is a piece of text, otherwise None.
 metadata Dictionary 
A dictionary of metadata that describes the datum.
 
Examples:
 
>>> Datum(uid=\"uid1\")\n>>> Datum(uid=\"uid1\", metadata={})\n>>> Datum(uid=\"uid1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n>>> Datum(uid=\"uid2\", text=\"Did Lincoln win the election of 1860?\", metadata={\"query_created_by\": \"Alice\"})\n
 Source code in valor/schemas/symbolic/collections.py 
class Datum(StaticCollection):\n    \"\"\"\n    A class used to store information about a datum for either a 'GroundTruth' or a 'Prediction'.\n\n    Attributes\n    ----------\n    uid : String\n        The UID of the datum.\n    text : String, optional\n        The text of the datum, if the datum is a piece of text, otherwise None.\n    metadata : Dictionary\n        A dictionary of metadata that describes the datum.\n\n    Examples\n    --------\n    >>> Datum(uid=\"uid1\")\n    >>> Datum(uid=\"uid1\", metadata={})\n    >>> Datum(uid=\"uid1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n    >>> Datum(uid=\"uid2\", text=\"Did Lincoln win the election of 1860?\", metadata={\"query_created_by\": \"Alice\"})\n    \"\"\"\n\n    uid: String = String.symbolic(owner=\"datum\", name=\"uid\")\n    text: String = String.symbolic(owner=\"datum\", name=\"text\")\n    metadata: Dictionary = Dictionary.symbolic(owner=\"datum\", name=\"metadata\")\n\n    def __init__(\n        self,\n        *,\n        uid: str,\n        text: Optional[str] = None,\n        metadata: Optional[dict] = None,\n    ):\n        \"\"\"\n        Constructs a datum.\n\n        Parameters\n        ----------\n        uid : str\n            The UID of the datum.\n        text : str, optional\n            The text of the datum, if the datum is a piece of text, otherwise None.\n        metadata : dict, optional\n            A dictionary of metadata that describes the datum.\n        \"\"\"\n        super().__init__(\n            uid=uid, text=text, metadata=metadata if metadata else dict()\n        )\n\n    @staticmethod\n    def formatting() -> Dict[str, Any]:\n        \"\"\"Attribute format mapping.\"\"\"\n        return {\n            \"text\": String.nullable,\n        }\n
"},{"location":"client_api/Datum/#valor.Datum-functions","title":"Functions","text":""},{"location":"client_api/Datum/#valor.Datum.__init__","title":"valor.Datum.__init__(*, uid, text=None, metadata=None)","text":"
Constructs a datum.
 
Parameters:
 Name Type Description Default uid str 
The UID of the datum.
 required text str 
The text of the datum, if the datum is a piece of text, otherwise None.
 None metadata dict 
A dictionary of metadata that describes the datum.
 None Source code in valor/schemas/symbolic/collections.py 
def __init__(\n    self,\n    *,\n    uid: str,\n    text: Optional[str] = None,\n    metadata: Optional[dict] = None,\n):\n    \"\"\"\n    Constructs a datum.\n\n    Parameters\n    ----------\n    uid : str\n        The UID of the datum.\n    text : str, optional\n        The text of the datum, if the datum is a piece of text, otherwise None.\n    metadata : dict, optional\n        A dictionary of metadata that describes the datum.\n    \"\"\"\n    super().__init__(\n        uid=uid, text=text, metadata=metadata if metadata else dict()\n    )\n
"},{"location":"client_api/Datum/#valor.Datum.formatting","title":"valor.Datum.formatting()  staticmethod","text":"
Attribute format mapping.
 Source code in valor/schemas/symbolic/collections.py 
@staticmethod\ndef formatting() -> Dict[str, Any]:\n    \"\"\"Attribute format mapping.\"\"\"\n    return {\n        \"text\": String.nullable,\n    }\n
"},{"location":"client_api/Evaluation/","title":"Evaluation","text":"
Wraps valor.client.Job to provide evaluation-specifc members.
 Source code in valor/coretypes.py 
class Evaluation:\n    \"\"\"\n    Wraps `valor.client.Job` to provide evaluation-specifc members.\n    \"\"\"\n\n    def __init__(\n        self, connection: Optional[ClientConnection] = None, **kwargs\n    ):\n        \"\"\"\n        Defines important attributes of the API's `EvaluationResult`.\n\n        Attributes\n        ----------\n        id : int\n            The ID of the evaluation.\n        dataset_names : list[str]\n            The names of the datasets the model was evaluated over.\n        model_name : str\n            The name of the evaluated model.\n        filters : dict\n            The filter used to select data partitions for evaluation.\n        status : EvaluationStatus\n            The status of the evaluation.\n        metrics : List[dict]\n            A list of metric dictionaries returned by the job.\n        confusion_matrices : List[dict]\n            A list of confusion matrix dictionaries returned by the job.\n        meta : dict[str, str | float | dict], optional\n            A dictionary of metadata describing the evaluation run.\n        \"\"\"\n        if not connection:\n            connection = get_connection()\n        self.conn = connection\n        self.update(**kwargs)\n\n    def update(\n        self,\n        *_,\n        id: int,\n        dataset_names: list[str],\n        model_name: str,\n        filters: dict,\n        parameters: EvaluationParameters,\n        status: EvaluationStatus,\n        metrics: List[Dict],\n        confusion_matrices: List[Dict],\n        created_at: str,\n        meta: dict[str, str | float | dict] | None,\n        **kwargs,\n    ):\n        self.id = id\n        self.dataset_names = dataset_names\n        self.model_name = model_name\n        self.filters = filters\n        self.parameters = (\n            EvaluationParameters(**parameters)\n            if isinstance(parameters, dict)\n            else parameters\n        )\n        self.status = EvaluationStatus(status)\n        self.metrics = metrics\n        self.meta = meta\n        self.confusion_matrices = confusion_matrices\n        self.kwargs = kwargs\n        self.ignored_pred_labels: Optional[List[Label]] = None\n        self.missing_pred_labels: Optional[List[Label]] = None\n        self.created_at = datetime.datetime.strptime(\n            created_at, \"%Y-%m-%dT%H:%M:%S.%fZ\"\n        ).replace(tzinfo=datetime.timezone.utc)\n\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n\n    def poll(self) -> EvaluationStatus:\n        \"\"\"\n        Poll the back end.\n\n        Updates the evaluation with the latest state from the back end.\n\n        Returns\n        -------\n        enums.EvaluationStatus\n            The status of the evaluation.\n\n        Raises\n        ----------\n        ClientException\n            If an Evaluation with the given `evaluation_id` is not found.\n        \"\"\"\n        response = self.conn.get_evaluations(\n            evaluation_ids=[self.id],\n            models=None,\n            datasets=None,\n            metrics_to_sort_by=None,\n        )\n        if not response:\n            raise EvaluationDoesNotExist(self.id)\n        self.update(**response[0])\n        return self.status\n\n    def wait_for_completion(\n        self,\n        *,\n        timeout: Optional[int] = None,\n        interval: float = 1.0,\n    ) -> EvaluationStatus:\n        \"\"\"\n        Blocking function that waits for evaluation to finish.\n\n        Parameters\n        ----------\n        timeout : int, optional\n            Length of timeout in seconds.\n        interval : float, default=1.0\n            Polling interval in seconds.\n        \"\"\"\n        t_start = time.time()\n        while self.poll() not in [\n            EvaluationStatus.DONE,\n            EvaluationStatus.FAILED,\n        ]:\n            time.sleep(interval)\n            if timeout and time.time() - t_start > timeout:\n                raise TimeoutError\n        return self.status\n\n    def __str__(self) -> str:\n        \"\"\"Dumps the object into a JSON formatted string.\"\"\"\n        return json.dumps(self.to_dict(), indent=4)\n\n    def to_dict(self) -> dict:\n        \"\"\"\n        Defines how a `valor.Evaluation` object is serialized into a dictionary.\n\n        Returns\n        ----------\n        dict\n            A dictionary describing an evaluation.\n        \"\"\"\n        return {\n            \"id\": self.id,\n            \"dataset_names\": self.dataset_names,\n            \"model_name\": self.model_name,\n            \"filters\": self.filters,\n            \"parameters\": asdict(self.parameters),\n            \"status\": self.status.value,\n            \"metrics\": self.metrics,\n            \"confusion_matrices\": self.confusion_matrices,\n            \"meta\": self.meta,\n            **self.kwargs,\n        }\n\n    def to_dataframe(\n        self,\n        stratify_by: Optional[Tuple[str, str]] = None,\n    ):\n        \"\"\"\n        Get all metrics associated with a Model and return them in a `pd.DataFrame`.\n\n        Returns\n        ----------\n        pd.DataFrame\n            Evaluation metrics being displayed in a `pd.DataFrame`.\n\n        Raises\n        ------\n        ModuleNotFoundError\n            This function requires the use of `pandas.DataFrame`.\n\n        \"\"\"\n        try:\n            import pandas as pd\n        except ModuleNotFoundError:\n            raise ModuleNotFoundError(\n                \"Must have pandas installed to use `get_metric_dataframes`.\"\n            )\n\n        if not stratify_by:\n            column_type = \"evaluation\"\n            column_name = self.id\n        else:\n            column_type = stratify_by[0]\n            column_name = stratify_by[1]\n\n        metrics = [\n            {**metric, column_type: column_name} for metric in self.metrics\n        ]\n        df = pd.DataFrame(metrics)\n        for k in [\"label\", \"parameters\"]:\n            df[k] = df[k].fillna(\"n/a\")\n        df[\"parameters\"] = df[\"parameters\"].apply(json.dumps)\n        df[\"label\"] = df[\"label\"].apply(\n            lambda x: f\"{x['key']}: {x['value']}\" if x != \"n/a\" else x\n        )\n        df = df.pivot(\n            index=[\"type\", \"parameters\", \"label\"], columns=[column_type]\n        )\n        return df\n
"},{"location":"client_api/Evaluation/#valor.Evaluation-functions","title":"Functions","text":""},{"location":"client_api/Evaluation/#valor.Evaluation.__init__","title":"valor.Evaluation.__init__(connection=None, **kwargs)","text":"
Defines important attributes of the API's EvaluationResult.
 
Attributes:
 Name Type Description id int 
The ID of the evaluation.
 dataset_names list[str] 
The names of the datasets the model was evaluated over.
 model_name str 
The name of the evaluated model.
 filters dict 
The filter used to select data partitions for evaluation.
 status EvaluationStatus 
The status of the evaluation.
 metrics List[dict] 
A list of metric dictionaries returned by the job.
 confusion_matrices List[dict] 
A list of confusion matrix dictionaries returned by the job.
 meta (dict[str, str | float | dict], optional) 
A dictionary of metadata describing the evaluation run.
 Source code in valor/coretypes.py 
def __init__(\n    self, connection: Optional[ClientConnection] = None, **kwargs\n):\n    \"\"\"\n    Defines important attributes of the API's `EvaluationResult`.\n\n    Attributes\n    ----------\n    id : int\n        The ID of the evaluation.\n    dataset_names : list[str]\n        The names of the datasets the model was evaluated over.\n    model_name : str\n        The name of the evaluated model.\n    filters : dict\n        The filter used to select data partitions for evaluation.\n    status : EvaluationStatus\n        The status of the evaluation.\n    metrics : List[dict]\n        A list of metric dictionaries returned by the job.\n    confusion_matrices : List[dict]\n        A list of confusion matrix dictionaries returned by the job.\n    meta : dict[str, str | float | dict], optional\n        A dictionary of metadata describing the evaluation run.\n    \"\"\"\n    if not connection:\n        connection = get_connection()\n    self.conn = connection\n    self.update(**kwargs)\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.__str__","title":"valor.Evaluation.__str__()","text":"
Dumps the object into a JSON formatted string.
 Source code in valor/coretypes.py 
def __str__(self) -> str:\n    \"\"\"Dumps the object into a JSON formatted string.\"\"\"\n    return json.dumps(self.to_dict(), indent=4)\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.poll","title":"valor.Evaluation.poll()","text":"
Poll the back end.
 
Updates the evaluation with the latest state from the back end.
 
Returns:
 Type Description EvaluationStatus 
The status of the evaluation.
 
Raises:
 Type Description ClientException 
If an Evaluation with the given evaluation_id is not found.
 Source code in valor/coretypes.py 
def poll(self) -> EvaluationStatus:\n    \"\"\"\n    Poll the back end.\n\n    Updates the evaluation with the latest state from the back end.\n\n    Returns\n    -------\n    enums.EvaluationStatus\n        The status of the evaluation.\n\n    Raises\n    ----------\n    ClientException\n        If an Evaluation with the given `evaluation_id` is not found.\n    \"\"\"\n    response = self.conn.get_evaluations(\n        evaluation_ids=[self.id],\n        models=None,\n        datasets=None,\n        metrics_to_sort_by=None,\n    )\n    if not response:\n        raise EvaluationDoesNotExist(self.id)\n    self.update(**response[0])\n    return self.status\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.to_dataframe","title":"valor.Evaluation.to_dataframe(stratify_by=None)","text":"
Get all metrics associated with a Model and return them in a pd.DataFrame.
 
Returns:
 Type Description DataFrame 
Evaluation metrics being displayed in a pd.DataFrame.
 
Raises:
 Type Description ModuleNotFoundError 
This function requires the use of pandas.DataFrame.
 Source code in valor/coretypes.py 
def to_dataframe(\n    self,\n    stratify_by: Optional[Tuple[str, str]] = None,\n):\n    \"\"\"\n    Get all metrics associated with a Model and return them in a `pd.DataFrame`.\n\n    Returns\n    ----------\n    pd.DataFrame\n        Evaluation metrics being displayed in a `pd.DataFrame`.\n\n    Raises\n    ------\n    ModuleNotFoundError\n        This function requires the use of `pandas.DataFrame`.\n\n    \"\"\"\n    try:\n        import pandas as pd\n    except ModuleNotFoundError:\n        raise ModuleNotFoundError(\n            \"Must have pandas installed to use `get_metric_dataframes`.\"\n        )\n\n    if not stratify_by:\n        column_type = \"evaluation\"\n        column_name = self.id\n    else:\n        column_type = stratify_by[0]\n        column_name = stratify_by[1]\n\n    metrics = [\n        {**metric, column_type: column_name} for metric in self.metrics\n    ]\n    df = pd.DataFrame(metrics)\n    for k in [\"label\", \"parameters\"]:\n        df[k] = df[k].fillna(\"n/a\")\n    df[\"parameters\"] = df[\"parameters\"].apply(json.dumps)\n    df[\"label\"] = df[\"label\"].apply(\n        lambda x: f\"{x['key']}: {x['value']}\" if x != \"n/a\" else x\n    )\n    df = df.pivot(\n        index=[\"type\", \"parameters\", \"label\"], columns=[column_type]\n    )\n    return df\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.to_dict","title":"valor.Evaluation.to_dict()","text":"
Defines how a valor.Evaluation object is serialized into a dictionary.
 
Returns:
 Type Description dict 
A dictionary describing an evaluation.
 Source code in valor/coretypes.py 
def to_dict(self) -> dict:\n    \"\"\"\n    Defines how a `valor.Evaluation` object is serialized into a dictionary.\n\n    Returns\n    ----------\n    dict\n        A dictionary describing an evaluation.\n    \"\"\"\n    return {\n        \"id\": self.id,\n        \"dataset_names\": self.dataset_names,\n        \"model_name\": self.model_name,\n        \"filters\": self.filters,\n        \"parameters\": asdict(self.parameters),\n        \"status\": self.status.value,\n        \"metrics\": self.metrics,\n        \"confusion_matrices\": self.confusion_matrices,\n        \"meta\": self.meta,\n        **self.kwargs,\n    }\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.wait_for_completion","title":"valor.Evaluation.wait_for_completion(*, timeout=None, interval=1.0)","text":"
Blocking function that waits for evaluation to finish.
 
Parameters:
 Name Type Description Default timeout int 
Length of timeout in seconds.
 None interval float 
Polling interval in seconds.
 1.0 Source code in valor/coretypes.py 
def wait_for_completion(\n    self,\n    *,\n    timeout: Optional[int] = None,\n    interval: float = 1.0,\n) -> EvaluationStatus:\n    \"\"\"\n    Blocking function that waits for evaluation to finish.\n\n    Parameters\n    ----------\n    timeout : int, optional\n        Length of timeout in seconds.\n    interval : float, default=1.0\n        Polling interval in seconds.\n    \"\"\"\n    t_start = time.time()\n    while self.poll() not in [\n        EvaluationStatus.DONE,\n        EvaluationStatus.FAILED,\n    ]:\n        time.sleep(interval)\n        if timeout and time.time() - t_start > timeout:\n            raise TimeoutError\n    return self.status\n
"},{"location":"client_api/Groundtruth/","title":"Groundtruth","text":"
               Bases: StaticCollection
 
An object describing a ground truth (e.g., a human-drawn bounding box on an image).
 
Attributes:
 Name Type Description datum Datum 
The datum associated with the groundtruth.
 annotations List[Annotation] 
The list of annotations associated with the groundtruth.
 
Examples:
 
>>> GroundTruth(\n...     datum=Datum(uid=\"uid1\"),\n...     annotations=[\n...         Annotation(\n...             labels=[Label(key=\"k1\", value=\"v1\")],\n...         )\n...     ]\n... )\n
 Source code in valor/coretypes.py 
class GroundTruth(StaticCollection):\n    \"\"\"\n    An object describing a ground truth (e.g., a human-drawn bounding box on an image).\n\n    Attributes\n    ----------\n    datum : Datum\n        The datum associated with the groundtruth.\n    annotations : List[Annotation]\n        The list of annotations associated with the groundtruth.\n\n    Examples\n    --------\n    >>> GroundTruth(\n    ...     datum=Datum(uid=\"uid1\"),\n    ...     annotations=[\n    ...         Annotation(\n    ...             labels=[Label(key=\"k1\", value=\"v1\")],\n    ...         )\n    ...     ]\n    ... )\n    \"\"\"\n\n    datum: Datum = Datum.symbolic(owner=\"groundtruth\", name=\"datum\")\n    annotations: SymbolicList[Annotation] = SymbolicList[Annotation].symbolic(\n        owner=\"groundtruth\", name=\"annotations\"\n    )\n\n    def __init__(\n        self,\n        *,\n        datum: Datum,\n        annotations: List[Annotation],\n    ):\n        \"\"\"\n        Creates a ground truth.\n\n        Parameters\n        ----------\n        datum : Datum\n            The datum that the ground truth is operating over.\n        annotations : List[Annotation]\n            The list of ground truth annotations.\n        \"\"\"\n        super().__init__(datum=datum, annotations=annotations)\n\n        for annotation in self.annotations:\n            for label in annotation.labels:\n                if label.score is not None:\n                    raise ValueError(\n                        \"GroundTruth labels should not have scores.\"\n                    )\n
"},{"location":"client_api/Groundtruth/#valor.GroundTruth-functions","title":"Functions","text":""},{"location":"client_api/Groundtruth/#valor.GroundTruth.__init__","title":"valor.GroundTruth.__init__(*, datum, annotations)","text":"
Creates a ground truth.
 
Parameters:
 Name Type Description Default datum Datum 
The datum that the ground truth is operating over.
 required annotations List[Annotation] 
The list of ground truth annotations.
 required Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    datum: Datum,\n    annotations: List[Annotation],\n):\n    \"\"\"\n    Creates a ground truth.\n\n    Parameters\n    ----------\n    datum : Datum\n        The datum that the ground truth is operating over.\n    annotations : List[Annotation]\n        The list of ground truth annotations.\n    \"\"\"\n    super().__init__(datum=datum, annotations=annotations)\n\n    for annotation in self.annotations:\n        for label in annotation.labels:\n            if label.score is not None:\n                raise ValueError(\n                    \"GroundTruth labels should not have scores.\"\n                )\n
"},{"location":"client_api/Label/","title":"Label","text":"
               Bases: StaticCollection
 
An object for labeling datasets, models, and annotations.
 
Attributes:
 Name Type Description key String 
The class label key.
 value String 
The class label value.
 score Score 
The label score.
 
Examples:
 
>>> Label(key=\"k1\", value=\"v1\")\n>>> Label(key=\"k1\", value=\"v1\", score=None)\n>>> Label(key=\"k1\", value=\"v1\", score=0.9)\n
 Source code in valor/schemas/symbolic/collections.py 
class Label(StaticCollection):\n    \"\"\"\n    An object for labeling datasets, models, and annotations.\n\n    Attributes\n    ----------\n    key : String\n        The class label key.\n    value : String\n        The class label value.\n    score : Score\n        The label score.\n\n    Examples\n    --------\n    >>> Label(key=\"k1\", value=\"v1\")\n    >>> Label(key=\"k1\", value=\"v1\", score=None)\n    >>> Label(key=\"k1\", value=\"v1\", score=0.9)\n    \"\"\"\n\n    key: String = String.symbolic(owner=\"label\", name=\"key\")\n    value: String = String.symbolic(owner=\"label\", name=\"value\")\n    score: Float = Float.symbolic(owner=\"label\", name=\"score\")\n\n    def __init__(\n        self,\n        *,\n        key: str,\n        value: str,\n        score: Union[float, np.floating, None] = None,\n    ):\n        \"\"\"\n        Initializes an instance of a label.\n\n        Attributes\n        ----------\n        key : str\n            The class label key.\n        value : str\n            The class label value.\n        score : float, optional\n            The label score.\n        \"\"\"\n        super().__init__(key=key, value=value, score=score)\n\n    @staticmethod\n    def formatting() -> Dict[str, Any]:\n        \"\"\"Attribute format mapping.\"\"\"\n        return {\n            \"score\": Float.nullable,\n        }\n\n    def tuple(self):\n        \"\"\"\n        Defines how the `Label` is turned into a tuple.\n\n        Returns\n        ----------\n        tuple\n            A tuple of the `Label's` arguments.\n        \"\"\"\n        return (self.key, self.value, self.score)\n
"},{"location":"client_api/Label/#valor.Label-functions","title":"Functions","text":""},{"location":"client_api/Label/#valor.Label.__init__","title":"valor.Label.__init__(*, key, value, score=None)","text":"
Initializes an instance of a label.
 
Attributes:
 Name Type Description key str 
The class label key.
 value str 
The class label value.
 score (float, optional) 
The label score.
 Source code in valor/schemas/symbolic/collections.py 
def __init__(\n    self,\n    *,\n    key: str,\n    value: str,\n    score: Union[float, np.floating, None] = None,\n):\n    \"\"\"\n    Initializes an instance of a label.\n\n    Attributes\n    ----------\n    key : str\n        The class label key.\n    value : str\n        The class label value.\n    score : float, optional\n        The label score.\n    \"\"\"\n    super().__init__(key=key, value=value, score=score)\n
"},{"location":"client_api/Label/#valor.Label.formatting","title":"valor.Label.formatting()  staticmethod","text":"
Attribute format mapping.
 Source code in valor/schemas/symbolic/collections.py 
@staticmethod\ndef formatting() -> Dict[str, Any]:\n    \"\"\"Attribute format mapping.\"\"\"\n    return {\n        \"score\": Float.nullable,\n    }\n
"},{"location":"client_api/Label/#valor.Label.tuple","title":"valor.Label.tuple()","text":"
Defines how the Label is turned into a tuple.
 
Returns:
 Type Description tuple 
A tuple of the Label's arguments.
 Source code in valor/schemas/symbolic/collections.py 
def tuple(self):\n    \"\"\"\n    Defines how the `Label` is turned into a tuple.\n\n    Returns\n    ----------\n    tuple\n        A tuple of the `Label's` arguments.\n    \"\"\"\n    return (self.key, self.value, self.score)\n
"},{"location":"client_api/Model/","title":"Model","text":"
               Bases: StaticCollection
 
A class describing a model that was trained on a particular dataset.
 
Attributes:
 Name Type Description name String 
The name of the model.
 metadata Dictionary 
A dictionary of metadata that describes the model.
 
Examples:
 
>>> Model.create(name=\"model1\")\n>>> Model.create(name=\"model1\", metadata={})\n>>> Model.create(name=\"model1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n
 Source code in valor/coretypes.py 
class Model(StaticCollection):\n    \"\"\"\n    A class describing a model that was trained on a particular dataset.\n\n    Attributes\n    ----------\n    name : String\n        The name of the model.\n    metadata : Dictionary\n        A dictionary of metadata that describes the model.\n\n    Examples\n    --------\n    >>> Model.create(name=\"model1\")\n    >>> Model.create(name=\"model1\", metadata={})\n    >>> Model.create(name=\"model1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n    \"\"\"\n\n    name: String = String.symbolic(owner=\"model\", name=\"name\")\n    metadata: Dictionary = Dictionary.symbolic(owner=\"model\", name=\"metadata\")\n\n    def __init__(\n        self,\n        *,\n        name: str,\n        metadata: Optional[dict] = None,\n        connection: Optional[ClientConnection] = None,\n    ):\n        \"\"\"\n        Creates a local instance of a model.\n\n        Use 'Model.create' classmethod to create a model with persistence.\n\n        Parameters\n        ----------\n        name : String\n            The name of the model.\n        metadata : Dictionary\n            A dictionary of metadata that describes the model.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        self.conn = connection\n        super().__init__(name=name, metadata=metadata if metadata else dict())\n\n    @classmethod\n    def create(\n        cls,\n        name: str,\n        metadata: Optional[Dict[str, Any]] = None,\n        connection: Optional[ClientConnection] = None,\n        **_,\n    ) -> Model:\n        \"\"\"\n        Creates a model that persists in the back end.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model.\n        metadata : dict, optional\n            A dictionary of metadata that describes the model.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        model = cls(name=name, metadata=metadata, connection=connection)\n        Client(connection).create_model(model)\n        return model\n\n    @classmethod\n    def get(\n        cls,\n        name: str,\n        connection: Optional[ClientConnection] = None,\n    ) -> Union[Model, None]:\n        \"\"\"\n        Retrieves a model from the back end database.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model.\n        connection : ClientConnnetion, optional\n            An optional Valor client object for interacting with the API.\n\n        Returns\n        -------\n        Union[valor.Model, None]\n            The model or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(connection).get_model(name)\n\n    def add_prediction(\n        self,\n        dataset: Dataset,\n        prediction: Prediction,\n    ) -> None:\n        \"\"\"\n        Add a prediction to the model.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset that is being operated over.\n        prediction : valor.Prediction\n            The prediction to create.\n        \"\"\"\n        Client(self.conn).create_predictions(\n            dataset=dataset,\n            model=self,\n            predictions=[prediction],\n        )\n\n    def add_predictions(\n        self,\n        dataset: Dataset,\n        predictions: List[Prediction],\n        timeout: Optional[float] = 10.0,\n    ) -> None:\n        \"\"\"\n        Add multiple predictions to the model.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset that is being operated over.\n        predictions : List[valor.Prediction]\n            The predictions to create.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        \"\"\"\n        Client(self.conn).create_predictions(\n            dataset=dataset,\n            model=self,\n            predictions=predictions,\n            timeout=timeout,\n        )\n\n    def get_prediction(\n        self, dataset: Union[Dataset, str], datum: Union[Datum, str]\n    ) -> Union[Prediction, None]:\n        \"\"\"\n        Get a particular prediction.\n\n        Parameters\n        ----------\n        dataset : Union[Dataset, str]\n            The dataset the datum belongs to.\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[Prediction, None]\n            The matching prediction or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(self.conn).get_prediction(\n            dataset=dataset, model=self, datum=datum\n        )\n\n    def finalize_inferences(self, dataset: Union[Dataset, str]) -> None:\n        \"\"\"\n        Finalizes the model over a dataset such that new predictions cannot be added to it.\n        \"\"\"\n        return Client(self.conn).finalize_inferences(\n            dataset=dataset, model=self\n        )\n\n    def _create_label_map(\n        self,\n        label_map: Optional[Dict[Label, Label]],\n    ) -> Union[List[List[List[str]]], None]:\n        \"\"\"Convert a dictionary of label maps to a serializable list format.\"\"\"\n        if not label_map:\n            return None\n\n        if not isinstance(label_map, dict) or not all(\n            [\n                isinstance(key, Label) and isinstance(value, Label)\n                for key, value in label_map.items()\n            ]\n        ):\n            raise TypeError(\n                \"label_map should be a dictionary with valid Labels for both the key and value.\"\n            )\n\n        return_value = []\n        for key, value in label_map.items():\n            if not all(\n                [\n                    (isinstance(v.key, str) and isinstance(v.value, str))\n                    for v in [key, value]\n                ]\n            ):\n                raise TypeError\n            return_value.append(\n                [\n                    [key.key, key.value],\n                    [value.key, value.value],\n                ]\n            )\n        return return_value\n\n    def evaluate_classification(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        filters: Optional[Filter] = None,\n        label_map: Optional[Dict[Label, Label]] = None,\n        pr_curve_max_examples: int = 1,\n        metrics_to_return: Optional[List[MetricType]] = None,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start a classification evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        label_map : Dict[Label, Label], optional\n            Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n        metrics_to_return : List[MetricType], optional\n            The list of metrics to compute, store, and return to the user.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion.\n        \"\"\"\n        if metrics_to_return and not set(metrics_to_return).issubset(\n            MetricType.classification()\n        ):\n            raise ValueError(\n                f\"The following metrics are not supported for classification: '{set(metrics_to_return) - MetricType.classification()}'\"\n            )\n\n        # format request\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=EvaluationParameters(\n                task_type=TaskType.CLASSIFICATION,\n                label_map=self._create_label_map(label_map=label_map),\n                pr_curve_max_examples=pr_curve_max_examples,\n                metrics_to_return=metrics_to_return,\n            ),\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(\n            request, allow_retries=allow_retries, timeout=timeout\n        )\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def evaluate_detection(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        filters: Optional[Filter] = None,\n        convert_annotations_to_type: Optional[AnnotationType] = None,\n        iou_thresholds_to_compute: Optional[List[float]] = None,\n        iou_thresholds_to_return: Optional[List[float]] = None,\n        label_map: Optional[Dict[Label, Label]] = None,\n        recall_score_threshold: float = 0,\n        metrics_to_return: Optional[List[MetricType]] = None,\n        pr_curve_iou_threshold: float = 0.5,\n        pr_curve_max_examples: int = 1,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start an object-detection evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        convert_annotations_to_type : enums.AnnotationType, optional\n            Forces the object detection evaluation to compute over this type.\n        iou_thresholds_to_compute : List[float], optional\n            Thresholds to compute mAP against.\n        iou_thresholds_to_return : List[float], optional\n            Thresholds to return AP for. Must be subset of `iou_thresholds_to_compute`.\n        label_map : Dict[Label, Label], optional\n            Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n        recall_score_threshold : float, default=0\n            The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\n        metrics_to_return : List[MetricType], optional\n            The list of metrics to compute, store, and return to the user.\n        pr_curve_iou_threshold : float, optional\n            The IOU threshold to use when calculating precision-recall curves. Defaults to 0.5.\n        pr_curve_max_examples : int, optional\n            The maximum number of datum examples to store when calculating PR curves.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion.\n        \"\"\"\n        if metrics_to_return and not set(metrics_to_return).issubset(\n            MetricType.object_detection()\n        ):\n            raise ValueError(\n                f\"The following metrics are not supported for object detection: '{set(metrics_to_return) - MetricType.object_detection()}'\"\n            )\n\n        if iou_thresholds_to_compute is None:\n            iou_thresholds_to_compute = [\n                round(0.5 + 0.05 * i, 2) for i in range(10)\n            ]\n        if iou_thresholds_to_return is None:\n            iou_thresholds_to_return = [0.5, 0.75]\n\n        # format request\n        parameters = EvaluationParameters(\n            task_type=TaskType.OBJECT_DETECTION,\n            convert_annotations_to_type=convert_annotations_to_type,\n            iou_thresholds_to_compute=iou_thresholds_to_compute,\n            iou_thresholds_to_return=iou_thresholds_to_return,\n            label_map=self._create_label_map(label_map=label_map),\n            recall_score_threshold=recall_score_threshold,\n            metrics_to_return=metrics_to_return,\n            pr_curve_iou_threshold=pr_curve_iou_threshold,\n            pr_curve_max_examples=pr_curve_max_examples,\n        )\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=parameters,\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(\n            request, allow_retries=allow_retries, timeout=timeout\n        )\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def evaluate_segmentation(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        filters: Optional[Filter] = None,\n        label_map: Optional[Dict[Label, Label]] = None,\n        metrics_to_return: Optional[List[MetricType]] = None,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start a semantic-segmentation evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        label_map : Dict[Label, Label], optional\n            Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n        metrics_to_return : List[MetricType], optional\n            The list of metrics to compute, store, and return to the user.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion\n        \"\"\"\n        if metrics_to_return and not set(metrics_to_return).issubset(\n            MetricType.semantic_segmentation()\n        ):\n            raise ValueError(\n                f\"The following metrics are not supported for semantic segmentation: '{set(metrics_to_return) - MetricType.semantic_segmentation()}'\"\n            )\n\n        # format request\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=EvaluationParameters(\n                task_type=TaskType.SEMANTIC_SEGMENTATION,\n                label_map=self._create_label_map(label_map=label_map),\n                metrics_to_return=metrics_to_return,\n            ),\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(\n            request, allow_retries=allow_retries, timeout=timeout\n        )\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def evaluate_text_generation(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        metrics_to_return: List[MetricType],\n        filters: Optional[Filter] = None,\n        llm_api_params: Optional[Dict[str, Union[str, dict]]] = None,\n        metric_params: Optional[Dict[MetricType, Dict[str, Any]]] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start a classification evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        metrics_to_return : List[MetricType]\n            The list of metrics to compute, store, and return to the user. This is not optional for text generation evaluations.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        llm_api_params : Dict[str, Union[str,dict]], optional\n            A dictionary of parameters for the LLM API.\n        metric_params : Dict[MetricType, Dict[str,Any]], optional\n            A dictionary of parameters for the metrics used in the evaluation. The keys should be the metrics and the values should be dictionaries of parameters for those metrics.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion.\n        \"\"\"\n        if not set(metrics_to_return).issubset(MetricType.text_generation()):\n            raise ValueError(\n                f\"The following metrics are not supported for text generation: '{set(metrics_to_return) - MetricType.text_generation()}'\"\n            )\n\n        # If no api_key is provided, check the environment variables for an api key.\n        if llm_api_params is not None:\n            if \"api_key\" not in llm_api_params:\n                if \"client\" not in llm_api_params:\n                    raise ValueError(\n                        \"The client must be specified in the llm_api_params.\"\n                    )\n\n                if llm_api_params[\"client\"] == \"openai\":\n                    api_key = os.getenv(\"OPENAI_API_KEY\", None)\n                elif llm_api_params[\"client\"] == \"mistral\":\n                    api_key = os.getenv(\"MISTRAL_API_KEY\", None)\n                elif llm_api_params[\"client\"] == \"mock\":\n                    api_key = \"\"\n                else:\n                    raise ValueError(\n                        \"The client specified in llm_api_params is not supported.\"\n                    )\n\n                if api_key is not None:\n                    llm_api_params[\"api_key\"] = api_key\n\n        bleu_weights = None\n        rouge_types = None\n        rouge_use_stemmer = None\n        if metric_params is not None:\n            if not all(\n                metric in metrics_to_return for metric in metric_params.keys()\n            ):\n                raise ValueError(\n                    \"All metrics in metric_params must be in metrics_to_return.\"\n                )\n\n            if MetricType.BLEU in metric_params:\n                bleu_weights = metric_params[MetricType.BLEU].get(\"weights\")\n\n            if MetricType.ROUGE in metric_params:\n                rouge_types = metric_params[MetricType.ROUGE].get(\n                    \"rouge_types\"\n                )\n                rouge_use_stemmer = metric_params[MetricType.ROUGE].get(\n                    \"use_stemmer\"\n                )\n\n        # format request\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=EvaluationParameters(\n                task_type=TaskType.TEXT_GENERATION,\n                metrics_to_return=metrics_to_return,\n                llm_api_params=llm_api_params,\n                bleu_weights=bleu_weights,\n                rouge_types=rouge_types,\n                rouge_use_stemmer=rouge_use_stemmer,\n            ),\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(request)\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def delete(self, timeout: int = 0):\n        \"\"\"\n        Delete the `Model` object from the back end.\n\n        Parameters\n        ----------\n        timeout : int, default=0\n            Sets a timeout in seconds.\n        \"\"\"\n        Client(self.conn).delete_model(self.name, timeout)  # type: ignore\n\n    def get_labels(\n        self,\n    ) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a given model.\n\n        Returns\n        ----------\n        List[Label]\n            A list of `Labels` associated with the model.\n        \"\"\"\n        return Client(self.conn).get_labels_from_model(self)\n\n    def get_evaluations(\n        self,\n        metrics_to_sort_by: Optional[\n            Dict[str, Union[Dict[str, str], str]]\n        ] = None,\n        *_,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Get all evaluations associated with a given model.\n\n        Parameters\n        ----------\n        metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n            An optional dict of metric types to sort the evaluations by.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        ----------\n        List[Evaluation]\n            A list of `Evaluations` associated with the model.\n        \"\"\"\n        return Client(self.conn).get_evaluations(\n            models=[self],\n            metrics_to_sort_by=metrics_to_sort_by,\n            timeout=timeout,\n        )\n
"},{"location":"client_api/Model/#valor.Model-functions","title":"Functions","text":""},{"location":"client_api/Model/#valor.Model.__init__","title":"valor.Model.__init__(*, name, metadata=None, connection=None)","text":"
Creates a local instance of a model.
 
Use 'Model.create' classmethod to create a model with persistence.
 
Parameters:
 Name Type Description Default name String 
The name of the model.
 required metadata Dictionary 
A dictionary of metadata that describes the model.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    name: str,\n    metadata: Optional[dict] = None,\n    connection: Optional[ClientConnection] = None,\n):\n    \"\"\"\n    Creates a local instance of a model.\n\n    Use 'Model.create' classmethod to create a model with persistence.\n\n    Parameters\n    ----------\n    name : String\n        The name of the model.\n    metadata : Dictionary\n        A dictionary of metadata that describes the model.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    self.conn = connection\n    super().__init__(name=name, metadata=metadata if metadata else dict())\n
"},{"location":"client_api/Model/#valor.Model.add_prediction","title":"valor.Model.add_prediction(dataset, prediction)","text":"
Add a prediction to the model.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset that is being operated over.
 required prediction Prediction 
The prediction to create.
 required Source code in valor/coretypes.py 
def add_prediction(\n    self,\n    dataset: Dataset,\n    prediction: Prediction,\n) -> None:\n    \"\"\"\n    Add a prediction to the model.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset that is being operated over.\n    prediction : valor.Prediction\n        The prediction to create.\n    \"\"\"\n    Client(self.conn).create_predictions(\n        dataset=dataset,\n        model=self,\n        predictions=[prediction],\n    )\n
"},{"location":"client_api/Model/#valor.Model.add_predictions","title":"valor.Model.add_predictions(dataset, predictions, timeout=10.0)","text":"
Add multiple predictions to the model.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset that is being operated over.
 required predictions List[Prediction] 
The predictions to create.
 required timeout float 
The number of seconds the client should wait until raising a timeout.
 10.0 Source code in valor/coretypes.py 
def add_predictions(\n    self,\n    dataset: Dataset,\n    predictions: List[Prediction],\n    timeout: Optional[float] = 10.0,\n) -> None:\n    \"\"\"\n    Add multiple predictions to the model.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset that is being operated over.\n    predictions : List[valor.Prediction]\n        The predictions to create.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    \"\"\"\n    Client(self.conn).create_predictions(\n        dataset=dataset,\n        model=self,\n        predictions=predictions,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Model/#valor.Model.create","title":"valor.Model.create(name, metadata=None, connection=None, **_)  classmethod","text":"
Creates a model that persists in the back end.
 
Parameters:
 Name Type Description Default name str 
The name of the model.
 required metadata dict 
A dictionary of metadata that describes the model.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
@classmethod\ndef create(\n    cls,\n    name: str,\n    metadata: Optional[Dict[str, Any]] = None,\n    connection: Optional[ClientConnection] = None,\n    **_,\n) -> Model:\n    \"\"\"\n    Creates a model that persists in the back end.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model.\n    metadata : dict, optional\n        A dictionary of metadata that describes the model.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    model = cls(name=name, metadata=metadata, connection=connection)\n    Client(connection).create_model(model)\n    return model\n
"},{"location":"client_api/Model/#valor.Model.delete","title":"valor.Model.delete(timeout=0)","text":"
Delete the Model object from the back end.
 
Parameters:
 Name Type Description Default timeout int 
Sets a timeout in seconds.
 0 Source code in valor/coretypes.py 
def delete(self, timeout: int = 0):\n    \"\"\"\n    Delete the `Model` object from the back end.\n\n    Parameters\n    ----------\n    timeout : int, default=0\n        Sets a timeout in seconds.\n    \"\"\"\n    Client(self.conn).delete_model(self.name, timeout)  # type: ignore\n
"},{"location":"client_api/Model/#valor.Model.evaluate_classification","title":"valor.Model.evaluate_classification(datasets, filters=None, label_map=None, pr_curve_max_examples=1, metrics_to_return=None, *_, allow_retries=False, timeout=None)","text":"
Start a classification evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None label_map Dict[Label, Label] 
Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 None metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user.
 None allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion.
 Source code in valor/coretypes.py 
def evaluate_classification(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    filters: Optional[Filter] = None,\n    label_map: Optional[Dict[Label, Label]] = None,\n    pr_curve_max_examples: int = 1,\n    metrics_to_return: Optional[List[MetricType]] = None,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> Evaluation:\n    \"\"\"\n    Start a classification evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    label_map : Dict[Label, Label], optional\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    metrics_to_return : List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion.\n    \"\"\"\n    if metrics_to_return and not set(metrics_to_return).issubset(\n        MetricType.classification()\n    ):\n        raise ValueError(\n            f\"The following metrics are not supported for classification: '{set(metrics_to_return) - MetricType.classification()}'\"\n        )\n\n    # format request\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=EvaluationParameters(\n            task_type=TaskType.CLASSIFICATION,\n            label_map=self._create_label_map(label_map=label_map),\n            pr_curve_max_examples=pr_curve_max_examples,\n            metrics_to_return=metrics_to_return,\n        ),\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(\n        request, allow_retries=allow_retries, timeout=timeout\n    )\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.evaluate_detection","title":"valor.Model.evaluate_detection(datasets, filters=None, convert_annotations_to_type=None, iou_thresholds_to_compute=None, iou_thresholds_to_return=None, label_map=None, recall_score_threshold=0, metrics_to_return=None, pr_curve_iou_threshold=0.5, pr_curve_max_examples=1, *_, allow_retries=False, timeout=None)","text":"
Start an object-detection evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None convert_annotations_to_type AnnotationType 
Forces the object detection evaluation to compute over this type.
 None iou_thresholds_to_compute List[float] 
Thresholds to compute mAP against.
 None iou_thresholds_to_return List[float] 
Thresholds to return AP for. Must be subset of iou_thresholds_to_compute.
 None label_map Dict[Label, Label] 
Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 None recall_score_threshold float 
The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.
 0 metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user.
 None pr_curve_iou_threshold float 
The IOU threshold to use when calculating precision-recall curves. Defaults to 0.5.
 0.5 pr_curve_max_examples int 
The maximum number of datum examples to store when calculating PR curves.
 1 allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion.
 Source code in valor/coretypes.py 
def evaluate_detection(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    filters: Optional[Filter] = None,\n    convert_annotations_to_type: Optional[AnnotationType] = None,\n    iou_thresholds_to_compute: Optional[List[float]] = None,\n    iou_thresholds_to_return: Optional[List[float]] = None,\n    label_map: Optional[Dict[Label, Label]] = None,\n    recall_score_threshold: float = 0,\n    metrics_to_return: Optional[List[MetricType]] = None,\n    pr_curve_iou_threshold: float = 0.5,\n    pr_curve_max_examples: int = 1,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> Evaluation:\n    \"\"\"\n    Start an object-detection evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    convert_annotations_to_type : enums.AnnotationType, optional\n        Forces the object detection evaluation to compute over this type.\n    iou_thresholds_to_compute : List[float], optional\n        Thresholds to compute mAP against.\n    iou_thresholds_to_return : List[float], optional\n        Thresholds to return AP for. Must be subset of `iou_thresholds_to_compute`.\n    label_map : Dict[Label, Label], optional\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    recall_score_threshold : float, default=0\n        The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\n    metrics_to_return : List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    pr_curve_iou_threshold : float, optional\n        The IOU threshold to use when calculating precision-recall curves. Defaults to 0.5.\n    pr_curve_max_examples : int, optional\n        The maximum number of datum examples to store when calculating PR curves.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion.\n    \"\"\"\n    if metrics_to_return and not set(metrics_to_return).issubset(\n        MetricType.object_detection()\n    ):\n        raise ValueError(\n            f\"The following metrics are not supported for object detection: '{set(metrics_to_return) - MetricType.object_detection()}'\"\n        )\n\n    if iou_thresholds_to_compute is None:\n        iou_thresholds_to_compute = [\n            round(0.5 + 0.05 * i, 2) for i in range(10)\n        ]\n    if iou_thresholds_to_return is None:\n        iou_thresholds_to_return = [0.5, 0.75]\n\n    # format request\n    parameters = EvaluationParameters(\n        task_type=TaskType.OBJECT_DETECTION,\n        convert_annotations_to_type=convert_annotations_to_type,\n        iou_thresholds_to_compute=iou_thresholds_to_compute,\n        iou_thresholds_to_return=iou_thresholds_to_return,\n        label_map=self._create_label_map(label_map=label_map),\n        recall_score_threshold=recall_score_threshold,\n        metrics_to_return=metrics_to_return,\n        pr_curve_iou_threshold=pr_curve_iou_threshold,\n        pr_curve_max_examples=pr_curve_max_examples,\n    )\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=parameters,\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(\n        request, allow_retries=allow_retries, timeout=timeout\n    )\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.evaluate_segmentation","title":"valor.Model.evaluate_segmentation(datasets, filters=None, label_map=None, metrics_to_return=None, *_, allow_retries=False, timeout=None)","text":"
Start a semantic-segmentation evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None label_map Dict[Label, Label] 
Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 None metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user.
 None allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion
 Source code in valor/coretypes.py 
def evaluate_segmentation(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    filters: Optional[Filter] = None,\n    label_map: Optional[Dict[Label, Label]] = None,\n    metrics_to_return: Optional[List[MetricType]] = None,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> Evaluation:\n    \"\"\"\n    Start a semantic-segmentation evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    label_map : Dict[Label, Label], optional\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    metrics_to_return : List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion\n    \"\"\"\n    if metrics_to_return and not set(metrics_to_return).issubset(\n        MetricType.semantic_segmentation()\n    ):\n        raise ValueError(\n            f\"The following metrics are not supported for semantic segmentation: '{set(metrics_to_return) - MetricType.semantic_segmentation()}'\"\n        )\n\n    # format request\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=EvaluationParameters(\n            task_type=TaskType.SEMANTIC_SEGMENTATION,\n            label_map=self._create_label_map(label_map=label_map),\n            metrics_to_return=metrics_to_return,\n        ),\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(\n        request, allow_retries=allow_retries, timeout=timeout\n    )\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.evaluate_text_generation","title":"valor.Model.evaluate_text_generation(datasets, metrics_to_return, filters=None, llm_api_params=None, metric_params=None)","text":"
Start a classification evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user. This is not optional for text generation evaluations.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None llm_api_params Dict[str, Union[str, dict]] 
A dictionary of parameters for the LLM API.
 None metric_params Dict[MetricType, Dict[str, Any]] 
A dictionary of parameters for the metrics used in the evaluation. The keys should be the metrics and the values should be dictionaries of parameters for those metrics.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion.
 Source code in valor/coretypes.py 
def evaluate_text_generation(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    metrics_to_return: List[MetricType],\n    filters: Optional[Filter] = None,\n    llm_api_params: Optional[Dict[str, Union[str, dict]]] = None,\n    metric_params: Optional[Dict[MetricType, Dict[str, Any]]] = None,\n) -> Evaluation:\n    \"\"\"\n    Start a classification evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    metrics_to_return : List[MetricType]\n        The list of metrics to compute, store, and return to the user. This is not optional for text generation evaluations.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    llm_api_params : Dict[str, Union[str,dict]], optional\n        A dictionary of parameters for the LLM API.\n    metric_params : Dict[MetricType, Dict[str,Any]], optional\n        A dictionary of parameters for the metrics used in the evaluation. The keys should be the metrics and the values should be dictionaries of parameters for those metrics.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion.\n    \"\"\"\n    if not set(metrics_to_return).issubset(MetricType.text_generation()):\n        raise ValueError(\n            f\"The following metrics are not supported for text generation: '{set(metrics_to_return) - MetricType.text_generation()}'\"\n        )\n\n    # If no api_key is provided, check the environment variables for an api key.\n    if llm_api_params is not None:\n        if \"api_key\" not in llm_api_params:\n            if \"client\" not in llm_api_params:\n                raise ValueError(\n                    \"The client must be specified in the llm_api_params.\"\n                )\n\n            if llm_api_params[\"client\"] == \"openai\":\n                api_key = os.getenv(\"OPENAI_API_KEY\", None)\n            elif llm_api_params[\"client\"] == \"mistral\":\n                api_key = os.getenv(\"MISTRAL_API_KEY\", None)\n            elif llm_api_params[\"client\"] == \"mock\":\n                api_key = \"\"\n            else:\n                raise ValueError(\n                    \"The client specified in llm_api_params is not supported.\"\n                )\n\n            if api_key is not None:\n                llm_api_params[\"api_key\"] = api_key\n\n    bleu_weights = None\n    rouge_types = None\n    rouge_use_stemmer = None\n    if metric_params is not None:\n        if not all(\n            metric in metrics_to_return for metric in metric_params.keys()\n        ):\n            raise ValueError(\n                \"All metrics in metric_params must be in metrics_to_return.\"\n            )\n\n        if MetricType.BLEU in metric_params:\n            bleu_weights = metric_params[MetricType.BLEU].get(\"weights\")\n\n        if MetricType.ROUGE in metric_params:\n            rouge_types = metric_params[MetricType.ROUGE].get(\n                \"rouge_types\"\n            )\n            rouge_use_stemmer = metric_params[MetricType.ROUGE].get(\n                \"use_stemmer\"\n            )\n\n    # format request\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=EvaluationParameters(\n            task_type=TaskType.TEXT_GENERATION,\n            metrics_to_return=metrics_to_return,\n            llm_api_params=llm_api_params,\n            bleu_weights=bleu_weights,\n            rouge_types=rouge_types,\n            rouge_use_stemmer=rouge_use_stemmer,\n        ),\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(request)\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.finalize_inferences","title":"valor.Model.finalize_inferences(dataset)","text":"
Finalizes the model over a dataset such that new predictions cannot be added to it.
 Source code in valor/coretypes.py 
def finalize_inferences(self, dataset: Union[Dataset, str]) -> None:\n    \"\"\"\n    Finalizes the model over a dataset such that new predictions cannot be added to it.\n    \"\"\"\n    return Client(self.conn).finalize_inferences(\n        dataset=dataset, model=self\n    )\n
"},{"location":"client_api/Model/#valor.Model.get","title":"valor.Model.get(name, connection=None)  classmethod","text":"
Retrieves a model from the back end database.
 
Parameters:
 Name Type Description Default name str 
The name of the model.
 required connection ClientConnnetion 
An optional Valor client object for interacting with the API.
 None 
Returns:
 Type Description Union[Model, None] 
The model or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
@classmethod\ndef get(\n    cls,\n    name: str,\n    connection: Optional[ClientConnection] = None,\n) -> Union[Model, None]:\n    \"\"\"\n    Retrieves a model from the back end database.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model.\n    connection : ClientConnnetion, optional\n        An optional Valor client object for interacting with the API.\n\n    Returns\n    -------\n    Union[valor.Model, None]\n        The model or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(connection).get_model(name)\n
"},{"location":"client_api/Model/#valor.Model.get_evaluations","title":"valor.Model.get_evaluations(metrics_to_sort_by=None, *_, timeout=None)","text":"
Get all evaluations associated with a given model.
 
Parameters:
 Name Type Description Default metrics_to_sort_by dict[str, str | dict[str, str]] 
An optional dict of metric types to sort the evaluations by.
 None timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of Evaluations associated with the model.
 Source code in valor/coretypes.py 
def get_evaluations(\n    self,\n    metrics_to_sort_by: Optional[\n        Dict[str, Union[Dict[str, str], str]]\n    ] = None,\n    *_,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Get all evaluations associated with a given model.\n\n    Parameters\n    ----------\n    metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n        An optional dict of metric types to sort the evaluations by.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    ----------\n    List[Evaluation]\n        A list of `Evaluations` associated with the model.\n    \"\"\"\n    return Client(self.conn).get_evaluations(\n        models=[self],\n        metrics_to_sort_by=metrics_to_sort_by,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Model/#valor.Model.get_labels","title":"valor.Model.get_labels()","text":"
Get all labels associated with a given model.
 
Returns:
 Type Description List[Label] 
A list of Labels associated with the model.
 Source code in valor/coretypes.py 
def get_labels(\n    self,\n) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a given model.\n\n    Returns\n    ----------\n    List[Label]\n        A list of `Labels` associated with the model.\n    \"\"\"\n    return Client(self.conn).get_labels_from_model(self)\n
"},{"location":"client_api/Model/#valor.Model.get_prediction","title":"valor.Model.get_prediction(dataset, datum)","text":"
Get a particular prediction.
 
Parameters:
 Name Type Description Default dataset Union[Dataset, str] 
The dataset the datum belongs to.
 required datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[Prediction, None] 
The matching prediction or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_prediction(\n    self, dataset: Union[Dataset, str], datum: Union[Datum, str]\n) -> Union[Prediction, None]:\n    \"\"\"\n    Get a particular prediction.\n\n    Parameters\n    ----------\n    dataset : Union[Dataset, str]\n        The dataset the datum belongs to.\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[Prediction, None]\n        The matching prediction or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(self.conn).get_prediction(\n        dataset=dataset, model=self, datum=datum\n    )\n
"},{"location":"client_api/Prediction/","title":"Prediction","text":"
               Bases: StaticCollection
 
An object describing a prediction (e.g., a machine-drawn bounding box on an image).
 
Attributes:
 Name Type Description datum Datum 
The datum associated with the prediction.
 annotations List[Annotation] 
The list of annotations associated with the prediction.
 
Examples:
 
>>> Prediction(\n...     datum=Datum(uid=\"uid1\"),\n...     annotations=[\n...         Annotation(\n...             labels=[\n...                 Label(key=\"k1\", value=\"v1\", score=0.9),\n...                 Label(key=\"k1\", value=\"v1\", score=0.1)\n...             ],\n...         )\n...     ]\n... )\n
 Source code in valor/coretypes.py 
class Prediction(StaticCollection):\n    \"\"\"\n    An object describing a prediction (e.g., a machine-drawn bounding box on an image).\n\n    Attributes\n    ----------\n    datum : Datum\n        The datum associated with the prediction.\n    annotations : List[Annotation]\n        The list of annotations associated with the prediction.\n\n    Examples\n    --------\n    >>> Prediction(\n    ...     datum=Datum(uid=\"uid1\"),\n    ...     annotations=[\n    ...         Annotation(\n    ...             labels=[\n    ...                 Label(key=\"k1\", value=\"v1\", score=0.9),\n    ...                 Label(key=\"k1\", value=\"v1\", score=0.1)\n    ...             ],\n    ...         )\n    ...     ]\n    ... )\n    \"\"\"\n\n    datum: Datum = Datum.symbolic(owner=\"prediction\", name=\"datum\")\n    annotations: SymbolicList[Annotation] = SymbolicList[Annotation].symbolic(\n        owner=\"prediction\", name=\"annotations\"\n    )\n\n    def __init__(\n        self,\n        *,\n        datum: Datum,\n        annotations: List[Annotation],\n    ):\n        \"\"\"\n        Creates a prediction.\n\n        Parameters\n        ----------\n        datum : Datum\n            The datum that the prediction is operating over.\n        annotations : List[Annotation]\n            The list of predicted annotations.\n        \"\"\"\n        super().__init__(datum=datum, annotations=annotations)\n
"},{"location":"client_api/Prediction/#valor.Prediction-functions","title":"Functions","text":""},{"location":"client_api/Prediction/#valor.Prediction.__init__","title":"valor.Prediction.__init__(*, datum, annotations)","text":"
Creates a prediction.
 
Parameters:
 Name Type Description Default datum Datum 
The datum that the prediction is operating over.
 required annotations List[Annotation] 
The list of predicted annotations.
 required Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    datum: Datum,\n    annotations: List[Annotation],\n):\n    \"\"\"\n    Creates a prediction.\n\n    Parameters\n    ----------\n    datum : Datum\n        The datum that the prediction is operating over.\n    annotations : List[Annotation]\n        The list of predicted annotations.\n    \"\"\"\n    super().__init__(datum=datum, annotations=annotations)\n
"},{"location":"client_api/Viz/","title":"Viz","text":""},{"location":"client_api/Viz/#valor.viz-classes","title":"Classes","text":""},{"location":"client_api/Viz/#valor.viz-functions","title":"Functions","text":""},{"location":"client_api/Viz/#valor.viz.create_combined_segmentation_mask","title":"valor.viz.create_combined_segmentation_mask(annotated_datum, label_key, filter_on_instance_segmentations=False)","text":"
Creates a combined segmentation mask from a list of segmentations.
 
Parameters:
 Name Type Description Default annotated_datum Union[GroundTruth, Prediction] 
A list of segmentations. These all must have the same image attribute.
 required label_key str 
The label key to use.
 required filter_on_instance_segmentations bool 
Whether to filter on instance segmentations or not.
 False 
Returns:
 Type Description tuple 
The first element of the tuple is the combined mask, as an RGB PIL image. The second element is a color legend: it's a dict with the unique labels as keys and the PIL image swatches as values.
 
Raises:
 Type Description RuntimeError 
If all segmentations don't belong to the same image or there is a segmentation that doesn't have label_key as the key of one of its labels.
 ValueError 
If there aren't any segmentations.
 Source code in valor/viz.py 
def create_combined_segmentation_mask(\n    annotated_datum: Union[GroundTruth, Prediction],\n    label_key: str,\n    filter_on_instance_segmentations: bool = False,\n) -> Tuple[Image.Image, Dict[str, Image.Image]]:\n    \"\"\"\n    Creates a combined segmentation mask from a list of segmentations.\n\n    Parameters\n    -------\n    annotated_datum : Union[GroundTruth, Prediction]\n        A list of segmentations. These all must have the same `image` attribute.\n    label_key : str\n        The label key to use.\n    filter_on_instance_segmentations : bool, optional\n        Whether to filter on instance segmentations or not.\n\n    Returns\n    -------\n    tuple\n        The first element of the tuple is the combined mask, as an RGB PIL image. The second\n        element is a color legend: it's a dict with the unique labels as keys and the\n        PIL image swatches as values.\n\n    Raises\n    ------\n    RuntimeError\n        If all segmentations don't belong to the same image or there is a\n        segmentation that doesn't have `label_key` as the key of one of its labels.\n    ValueError\n        If there aren't any segmentations.\n    \"\"\"\n\n    # validate input type\n    if not isinstance(annotated_datum, (GroundTruth, Prediction)):\n        raise ValueError(\"Expected either a 'GroundTruth' or 'Prediction'\")\n\n    # verify there are a nonzero number of annotations\n    if len(annotated_datum.annotations) == 0:\n        raise ValueError(\"annotations cannot be empty.\")\n\n    # validate raster size\n    img_h = None\n    img_w = None\n    for annotation in annotated_datum.annotations:\n        raster = annotation.raster\n        if raster.get_value() is None:\n            raise ValueError(\"No raster exists.\")\n        if img_h is None:\n            img_h = raster.height\n        if img_w is None:\n            img_w = raster.width\n        if (img_h != raster.height) or (img_w != raster.width):\n            raise ValueError(\n                f\"Size mismatch between rasters. {(img_h, img_w)} != {(raster.height, raster.width)}\"\n            )\n    if img_h is None or img_w is None:\n        raise ValueError(\n            f\"Segmentation bounds not properly defined. {(img_h, img_w)}\"\n        )\n\n    # unpack raster annotations\n    annotations: List[Annotation] = []\n    for annotation in annotated_datum.annotations:\n        if (\n            annotation.is_instance or False\n        ) == filter_on_instance_segmentations:\n            annotations.append(annotation)\n\n    # unpack label values\n    label_values = []\n    for annotation in annotations:\n        for label in annotation.labels:\n            if label.key == label_key:\n                label_values.append(label.value)\n    if not label_values:\n        raise RuntimeError(\n            f\"Annotation doesn't have a label with key `{label_key}`\"\n        )\n\n    # assign label coloring\n    unique_label_values = list(set(label_values))\n    label_value_to_color = {\n        v: COLOR_MAP[i] for i, v in enumerate(unique_label_values)\n    }\n    seg_colors = [label_value_to_color[v] for v in label_values]\n\n    # create mask\n    combined_mask = np.zeros((img_h, img_w, 3), dtype=np.uint8)\n    for annotation, color in zip(annotations, seg_colors):\n        raster = annotation.raster\n        if raster.get_value() is None:\n            raise ValueError(\"No raster exists.\")\n        if raster.array is not None:\n            if raster.geometry is None:\n                mask = raster.array\n            elif isinstance(raster.geometry, schemas.MultiPolygon):\n                mask = _polygons_to_binary_mask(\n                    raster.geometry.to_polygons(),\n                    img_w=img_w,\n                    img_h=img_h,\n                )\n            elif isinstance(raster.geometry, (schemas.Box, schemas.Polygon)):\n                mask = _polygons_to_binary_mask(\n                    [raster.geometry],\n                    img_w=img_w,\n                    img_h=img_h,\n                )\n            else:\n                continue\n            combined_mask[np.where(mask)] = color\n        else:\n            continue\n\n    legend = {\n        v: Image.new(\"RGB\", (20, 20), color)\n        for v, color in label_value_to_color.items()\n    }\n\n    return Image.fromarray(combined_mask), legend\n
"},{"location":"client_api/Viz/#valor.viz.draw_bounding_box_on_image","title":"valor.viz.draw_bounding_box_on_image(bounding_box, img, color=(255, 0, 0))","text":"
Draws a bounding polygon on an image. This operation is not done in place.
 
Parameters:
 Name Type Description Default bounding_box Box 
Bounding box to draw on the image.
 required img Image 
Pillow image to draw on.
 required color Tuple[int, int, int] 
RGB tuple of the color to use.
 (255, 0, 0) 
Returns:
 Type Description img 
Pillow image with bounding box drawn on it.
 Source code in valor/viz.py 
def draw_bounding_box_on_image(\n    bounding_box: schemas.Box,\n    img: Image.Image,\n    color: Tuple[int, int, int] = (255, 0, 0),\n) -> Image.Image:\n    \"\"\"Draws a bounding polygon on an image. This operation is not done in place.\n\n    Parameters\n    ----------\n    bounding_box\n        Bounding box to draw on the image.\n    img\n        Pillow image to draw on.\n    color\n        RGB tuple of the color to use.\n\n    Returns\n    -------\n    img\n        Pillow image with bounding box drawn on it.\n    \"\"\"\n    coords = bounding_box.get_value()\n    return _draw_bounding_polygon_on_image(\n        schemas.Polygon(coords), img, color=color, inplace=False\n    )\n
"},{"location":"client_api/Viz/#valor.viz.draw_detections_on_image","title":"valor.viz.draw_detections_on_image(detections, img)","text":"
Draws detections (bounding boxes and labels) on an image.
 
Parameters:
 Name Type Description Default detections List[Union[GroundTruth, Prediction]] 
A list of GroundTruths or Predictions to draw on the image.
 required img Image 
The image to draw the detections on.
 required 
Returns:
 Name Type Description img Image 
An image with the detections drawn on.
 Source code in valor/viz.py 
def draw_detections_on_image(\n    detections: Sequence[Union[GroundTruth, Prediction]],\n    img: Image.Image,\n) -> Image.Image:\n    \"\"\"\n    Draws detections (bounding boxes and labels) on an image.\n    Parameters\n    -------\n    detections : List[Union[GroundTruth, Prediction]]\n        A list of `GroundTruths` or `Predictions` to draw on the image.\n    img : Image.Image\n        The image to draw the detections on.\n    Returns\n    -------\n    img : Image.Image\n        An image with the detections drawn on.\n    \"\"\"\n\n    annotations = []\n    for datum in detections:\n        annotations.extend(datum.annotations)\n\n    for i, detection in enumerate(annotations):\n        if detection.raster and detection.is_instance is True:\n            img = _draw_detection_on_image(detection, img, inplace=i != 0)\n    return img\n
"},{"location":"client_api/Viz/#valor.viz.draw_raster_on_image","title":"valor.viz.draw_raster_on_image(raster, img, color=(255, 0, 0), alpha=0.4)","text":"
Draws the raster on top of an image. This operation is not done in place.
 
Parameters:
 Name Type Description Default img Image 
pillow image to draw on.
 required color Tuple[int, int, int] 
RGB tuple of the color to use
 (255, 0, 0) alpha float 
alpha (transparency) value of the mask. 0 is fully transparent, 1 is fully opaque
 0.4 Source code in valor/viz.py 
def draw_raster_on_image(\n    raster: schemas.Raster,\n    img: Image.Image,\n    color: Tuple[int, int, int] = (255, 0, 0),\n    alpha: float = 0.4,\n) -> Image.Image:\n    \"\"\"Draws the raster on top of an image. This operation is not done in place.\n\n    Parameters\n    ----------\n    img\n        pillow image to draw on.\n    color\n        RGB tuple of the color to use\n    alpha\n        alpha (transparency) value of the mask. 0 is fully transparent, 1 is fully opaque\n    \"\"\"\n    img = img.copy()\n    binary_mask = raster.array\n    mask_arr = np.zeros(\n        (binary_mask.shape[0], binary_mask.shape[1], 3), dtype=np.uint8\n    )\n    mask_arr[binary_mask] = color\n    mask_img = Image.fromarray(mask_arr)\n\n    if mask_img.size != img.size:\n        raise ValueError(\"Input image and raster must be the same size.\")\n    blend = Image.blend(img, mask_img, alpha=alpha)\n    img.paste(blend, (0, 0), mask=Image.fromarray(binary_mask))\n\n    return img\n
"},{"location":"client_api/Schemas/Filters/","title":"Filters","text":""},{"location":"client_api/Schemas/Filters/#valor.schemas.filters-classes","title":"Classes","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/","title":"EvaluationParameters","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation-classes","title":"Classes","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationParameters","title":"valor.schemas.evaluation.EvaluationParameters  dataclass","text":"
Defines parameters for evaluation methods.\n
 
Attributes\n
 
task_type: TaskType\n    The task type of a given evaluation.\nlabel_map: Optional[List[List[List[str]]]]\n    Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\nmetrics_to_return: List[MetricType], optional\n    The list of metrics to compute, store, and return to the user.\nllm_api_params: Dict[str, str | dict], optional\n    A dictionary of parameters for the LLM API.\nconvert_annotations_to_type: AnnotationType | None = None\n    The type to convert all annotations to.\niou_thresholds_to_compute: List[float], optional\n    A list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP).\niou_thresholds_to_return: List[float], optional\n    A list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`.\nrecall_score_threshold: float, default=0\n    The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\npr_curve_iou_threshold: float, optional\n    The IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5.\npr_curve_max_examples: int\n    The maximum number of datum examples to store when calculating PR curves.\nbleu_weights: list[float], optional\n    The weights to use when calculating BLEU scores.\nrouge_types: list[ROUGEType]\n    A list of rouge types to calculate. Options are ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'], where `rouge1` is unigram-based scoring, `rouge2` is bigram-based scoring, `rougeL` is scoring based on sentences (i.e., splitting on \".\" and ignoring \"\n
 
\"), and rougeLsum is scoring based on splitting the text using \" \".     rouge_use_stemmer: bool         If True, uses Porter stemmer to strip word suffixes.
 Source code in valor/schemas/evaluation.py 
@dataclass\nclass EvaluationParameters:\n    \"\"\"\n    Defines parameters for evaluation methods.\n\n    Attributes\n    ----------\n    task_type: TaskType\n        The task type of a given evaluation.\n    label_map: Optional[List[List[List[str]]]]\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    metrics_to_return: List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    llm_api_params: Dict[str, str | dict], optional\n        A dictionary of parameters for the LLM API.\n    convert_annotations_to_type: AnnotationType | None = None\n        The type to convert all annotations to.\n    iou_thresholds_to_compute: List[float], optional\n        A list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP).\n    iou_thresholds_to_return: List[float], optional\n        A list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`.\n    recall_score_threshold: float, default=0\n        The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\n    pr_curve_iou_threshold: float, optional\n        The IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5.\n    pr_curve_max_examples: int\n        The maximum number of datum examples to store when calculating PR curves.\n    bleu_weights: list[float], optional\n        The weights to use when calculating BLEU scores.\n    rouge_types: list[ROUGEType]\n        A list of rouge types to calculate. Options are ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'], where `rouge1` is unigram-based scoring, `rouge2` is bigram-based scoring, `rougeL` is scoring based on sentences (i.e., splitting on \".\" and ignoring \"\\n\"), and `rougeLsum` is scoring based on splitting the text using \"\\n\".\n    rouge_use_stemmer: bool\n        If True, uses Porter stemmer to strip word suffixes.\n    \"\"\"\n\n    task_type: TaskType\n    label_map: Optional[List[List[List[str]]]] = None\n    metrics_to_return: Optional[List[MetricType]] = None\n    llm_api_params: Optional[Dict[str, Union[str, dict]]] = None\n\n    convert_annotations_to_type: Optional[AnnotationType] = None\n    iou_thresholds_to_compute: Optional[List[float]] = None\n    iou_thresholds_to_return: Optional[List[float]] = None\n    recall_score_threshold: float = 0\n    pr_curve_iou_threshold: float = 0.5\n    pr_curve_max_examples: int = 1\n    bleu_weights: Optional[List[float]] = None\n    rouge_types: Optional[List[ROUGEType]] = None\n    rouge_use_stemmer: Optional[bool] = None\n
"},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationRequest","title":"valor.schemas.evaluation.EvaluationRequest  dataclass","text":"
An evaluation request.
 
Defines important attributes of the API's EvaluationRequest.
 
Attributes:
 Name Type Description dataset_names List[str] 
The list of datasets we want to evaluate by name.
 model_names List[str] 
The list of models we want to evaluate by name.
 filters dict 
The filter object used to define what the model(s) is evaluating against.
 parameters EvaluationParameters 
Any parameters that are used to modify an evaluation method.
 Source code in valor/schemas/evaluation.py 
@dataclass\nclass EvaluationRequest:\n    \"\"\"\n    An evaluation request.\n\n    Defines important attributes of the API's `EvaluationRequest`.\n\n    Attributes\n    ----------\n    dataset_names : List[str]\n        The list of datasets we want to evaluate by name.\n    model_names : List[str]\n        The list of models we want to evaluate by name.\n    filters : dict\n        The filter object used to define what the model(s) is evaluating against.\n    parameters : EvaluationParameters\n        Any parameters that are used to modify an evaluation method.\n    \"\"\"\n\n    dataset_names: Union[str, List[str]]\n    model_names: Union[str, List[str]]\n    parameters: EvaluationParameters\n    filters: Filter = field(default_factory=Filter)\n\n    def __post_init__(self):\n        if isinstance(self.filters, dict):\n            self.filters = Filter(**self.filters)\n        elif self.filters is None:\n            self.filters = Filter()\n\n        if isinstance(self.parameters, dict):\n            self.parameters = EvaluationParameters(**self.parameters)\n\n    def to_dict(self) -> dict:\n        \"\"\"\n        Converts the request into a JSON-compatible dictionary.\n        \"\"\"\n        return {\n            \"dataset_names\": self.dataset_names,\n            \"model_names\": self.model_names,\n            \"parameters\": asdict(self.parameters),\n            \"filters\": self.filters.to_dict(),\n        }\n
"},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationRequest-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationRequest.to_dict","title":"valor.schemas.evaluation.EvaluationRequest.to_dict()","text":"
Converts the request into a JSON-compatible dictionary.
 Source code in valor/schemas/evaluation.py 
def to_dict(self) -> dict:\n    \"\"\"\n    Converts the request into a JSON-compatible dictionary.\n    \"\"\"\n    return {\n        \"dataset_names\": self.dataset_names,\n        \"model_names\": self.model_names,\n        \"parameters\": asdict(self.parameters),\n        \"filters\": self.filters.to_dict(),\n    }\n
"},{"location":"client_api/Schemas/Spatial/Box/","title":"Box","text":"
               Bases: Polygon
 
A Box is a polygon that is constrained to 4 unique points.
 
Note that this does not need to be axis-aligned.
 
Parameters:
 Name Type Description Default value List[List[Tuple[float, float]]] 
An polygon value representing a box.
 required 
Attributes:
 Name Type Description area polygon boundary holes xmin xmax ymin ymax 
Examples:
 
>>> Box([[(0,0), (0,1), (1,1), (1,0), (0,0)]])\n
 
Create a Box using extrema.
 
>>> Box.from_extrema(\n...     xmin=0, xmax=1,\n...     ymin=0, ymax=1,\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class Box(Polygon):\n    \"\"\"\n    A Box is a polygon that is constrained to 4 unique points.\n\n    Note that this does not need to be axis-aligned.\n\n    Parameters\n    ----------\n    value : List[List[Tuple[float, float]]], optional\n        An polygon value representing a box.\n\n    Attributes\n    ----------\n    area\n    polygon\n    boundary\n    holes\n    xmin\n    xmax\n    ymin\n    ymax\n\n    Examples\n    --------\n    >>> Box([[(0,0), (0,1), (1,1), (1,0), (0,0)]])\n\n    Create a Box using extrema.\n    >>> Box.from_extrema(\n    ...     xmin=0, xmax=1,\n    ...     ymin=0, ymax=1,\n    ... )\n    \"\"\"\n\n    def __init__(\n        self, value: typing.List[typing.List[typing.Tuple[float, float]]]\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        \"\"\"\n        Validates typing.\n\n        Parameters\n        ----------\n        value : typing.Any\n            The value to validate.\n\n        Raises\n        ------\n        TypeError\n            If the value type is not supported.\n        \"\"\"\n        Polygon.__validate__(value)\n        if len(value) != 1:\n            raise ValueError(\"Box should not contain holes.\")\n        elif len(value[0]) != 5:\n            raise ValueError(\"Box should consist of four unique points.\")\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return super().decode_value(value)\n\n    @classmethod\n    def from_extrema(\n        cls,\n        xmin: float,\n        xmax: float,\n        ymin: float,\n        ymax: float,\n    ):\n        \"\"\"\n        Create a Box from extrema values.\n\n        Parameters\n        ----------\n        xmin : float\n            Minimum x-coordinate of the bounding box.\n        xmax : float\n            Maximum x-coordinate of the bounding box.\n        ymin : float\n            Minimum y-coordinate of the bounding box.\n        ymax : float\n            Maximum y-coordinate of the bounding box.\n\n        Returns\n        -------\n        Box\n            A Box created from the provided extrema values.\n        \"\"\"\n        points = [\n            [\n                (xmin, ymin),\n                (xmax, ymin),\n                (xmax, ymax),\n                (xmin, ymax),\n                (xmin, ymin),\n            ]\n        ]\n        return cls(value=points)\n\n    def to_polygon(self) -> Polygon:\n        \"\"\"\n        Converts box to a generic polygon.\n\n        Returns\n        -------\n        Polygon\n            The box as a Polygon.\n        \"\"\"\n        return Polygon(self.get_value())\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.__validate__","title":"valor.schemas.Box.__validate__(value)  classmethod","text":"
Validates typing.
 
Parameters:
 Name Type Description Default value Any 
The value to validate.
 required 
Raises:
 Type Description TypeError 
If the value type is not supported.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef __validate__(cls, value: typing.Any):\n    \"\"\"\n    Validates typing.\n\n    Parameters\n    ----------\n    value : typing.Any\n        The value to validate.\n\n    Raises\n    ------\n    TypeError\n        If the value type is not supported.\n    \"\"\"\n    Polygon.__validate__(value)\n    if len(value) != 1:\n        raise ValueError(\"Box should not contain holes.\")\n    elif len(value[0]) != 5:\n        raise ValueError(\"Box should consist of four unique points.\")\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.decode_value","title":"valor.schemas.Box.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return super().decode_value(value)\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.from_extrema","title":"valor.schemas.Box.from_extrema(xmin, xmax, ymin, ymax)  classmethod","text":"
Create a Box from extrema values.
 
Parameters:
 Name Type Description Default xmin float 
Minimum x-coordinate of the bounding box.
 required xmax float 
Maximum x-coordinate of the bounding box.
 required ymin float 
Minimum y-coordinate of the bounding box.
 required ymax float 
Maximum y-coordinate of the bounding box.
 required 
Returns:
 Type Description Box 
A Box created from the provided extrema values.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_extrema(\n    cls,\n    xmin: float,\n    xmax: float,\n    ymin: float,\n    ymax: float,\n):\n    \"\"\"\n    Create a Box from extrema values.\n\n    Parameters\n    ----------\n    xmin : float\n        Minimum x-coordinate of the bounding box.\n    xmax : float\n        Maximum x-coordinate of the bounding box.\n    ymin : float\n        Minimum y-coordinate of the bounding box.\n    ymax : float\n        Maximum y-coordinate of the bounding box.\n\n    Returns\n    -------\n    Box\n        A Box created from the provided extrema values.\n    \"\"\"\n    points = [\n        [\n            (xmin, ymin),\n            (xmax, ymin),\n            (xmax, ymax),\n            (xmin, ymax),\n            (xmin, ymin),\n        ]\n    ]\n    return cls(value=points)\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.to_polygon","title":"valor.schemas.Box.to_polygon()","text":"
Converts box to a generic polygon.
 
Returns:
 Type Description Polygon 
The box as a Polygon.
 Source code in valor/schemas/symbolic/types.py 
def to_polygon(self) -> Polygon:\n    \"\"\"\n    Converts box to a generic polygon.\n\n    Returns\n    -------\n    Polygon\n        The box as a Polygon.\n    \"\"\"\n    return Polygon(self.get_value())\n
"},{"location":"client_api/Schemas/Spatial/LineString/","title":"LineString","text":"
               Bases: Spatial
 
Represents a line.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[Tuple[float, float]] 
A linestring.
 required 
Methods:
 Name Description colorspace 
Represent the photo in the given colorspace.
 gamma 
Change the photo's gamma exposure.
 
Examples:
 
Create a line.
 
>>> LineString([(0,0), (0,1), (1,1)])\n
 Source code in valor/schemas/symbolic/types.py 
class LineString(Spatial):\n    \"\"\"\n    Represents a line.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[Tuple[float, float]], optional\n        A linestring.\n\n    Methods\n    -------\n    colorspace(c='rgb')\n        Represent the photo in the given colorspace.\n    gamma(n=1.0)\n        Change the photo's gamma exposure.\n\n    Examples\n    --------\n    Create a line.\n    >>> LineString([(0,0), (0,1), (1,1)])\n    \"\"\"\n\n    def __init__(self, value: typing.List[typing.Tuple[float, float]]):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        MultiPoint.__validate__(value)\n        if len(value) < 2:\n            raise ValueError(\n                \"At least two points are required to make a line.\"\n            )\n\n    @classmethod\n    def decode_value(\n        cls, value: typing.Optional[typing.List[typing.List[float]]]\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/LineString/#valor.schemas.LineString-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/LineString/#valor.schemas.LineString.decode_value","title":"valor.schemas.LineString.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls, value: typing.Optional[typing.List[typing.List[float]]]\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/MultiLineString/","title":"MultiLineString","text":"
               Bases: Spatial
 
Represents a list of lines.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[List[Tuple[float, float]]] 
A multilinestring.
 required 
Examples:
 
Create a single line.
 
>>> MultiLineString([[(0,0), (0,1), (1,1), (0,0)]])\n
 
Create 3 lines.
 
>>> MultiLineString(\n...     [\n...         [(0,0), (0,1), (1,1)],\n...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2)],\n...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7)],\n...     ]\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class MultiLineString(Spatial):\n    \"\"\"\n    Represents a list of lines.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[List[Tuple[float, float]]], optional\n        A multilinestring.\n\n    Examples\n    --------\n    Create a single line.\n    >>> MultiLineString([[(0,0), (0,1), (1,1), (0,0)]])\n\n    Create 3 lines.\n    >>> MultiLineString(\n    ...     [\n    ...         [(0,0), (0,1), (1,1)],\n    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2)],\n    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7)],\n    ...     ]\n    ... )\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.List[typing.List[typing.Tuple[float, float]]],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, list):\n            raise TypeError(\n                f\"Expected type 'List[List[Tuple[float, float]]]' received type '{type(value).__name__}'\"\n            )\n        for line in value:\n            LineString.__validate__(line)\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls(\n            [[(point[0], point[1]) for point in line] for line in value]\n        )\n
"},{"location":"client_api/Schemas/Spatial/MultiLineString/#valor.schemas.MultiLineString-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/MultiLineString/#valor.schemas.MultiLineString.decode_value","title":"valor.schemas.MultiLineString.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls(\n        [[(point[0], point[1]) for point in line] for line in value]\n    )\n
"},{"location":"client_api/Schemas/Spatial/MultiPoint/","title":"MultiPoint","text":"
               Bases: Spatial
 
Represents a list of points.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[Tuple[float, float]] 
A multipoint.
 required 
Examples:
 
>>> MultiPoint([(0,0), (0,1), (1,1)])\n
 Source code in valor/schemas/symbolic/types.py 
class MultiPoint(Spatial):\n    \"\"\"\n    Represents a list of points.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[Tuple[float, float]], optional\n        A multipoint.\n\n    Examples\n    --------\n    >>> MultiPoint([(0,0), (0,1), (1,1)])\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.List[typing.Tuple[float, float]],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, list):\n            raise TypeError(\n                f\"Expected 'typing.List[typing.Tuple[float, float]]' received type '{type(value).__name__}'\"\n            )\n        for point in value:\n            Point.__validate__(point)\n\n    @classmethod\n    def decode_value(\n        cls, value: typing.Optional[typing.List[typing.List[float]]]\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/MultiPoint/#valor.schemas.MultiPoint-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/MultiPoint/#valor.schemas.MultiPoint.decode_value","title":"valor.schemas.MultiPoint.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls, value: typing.Optional[typing.List[typing.List[float]]]\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/","title":"MultiPolygon","text":"
               Bases: Spatial
 
Represents a collection of polygons.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[List[List[Tuple[float, float]]]] 
A list of polygons.
 required 
Attributes:
 Name Type Description area polygons 
Examples:
 
>>> MultiPolygon(\n...     [\n...         [\n...             [(0,0), (0,1), (1,1), (0,0)]\n...         ],\n...         [\n...             [(0,0), (0,1), (1,1), (0,0)],\n...             [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n...             [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n...         ],\n...     ]\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class MultiPolygon(Spatial):\n    \"\"\"\n    Represents a collection of polygons.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[List[List[Tuple[float, float]]]], optional\n        A list of polygons.\n\n    Attributes\n    ----------\n    area\n    polygons\n\n    Examples\n    --------\n    >>> MultiPolygon(\n    ...     [\n    ...         [\n    ...             [(0,0), (0,1), (1,1), (0,0)]\n    ...         ],\n    ...         [\n    ...             [(0,0), (0,1), (1,1), (0,0)],\n    ...             [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n    ...             [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n    ...         ],\n    ...     ]\n    ... )\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.List[\n            typing.List[typing.List[typing.Tuple[float, float]]]\n        ],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, list):\n            raise TypeError(\n                f\"Expected type 'List[List[List[Tuple[float, float]]]]' received type '{type(value).__name__}'\"\n            )\n        for poly in value:\n            Polygon.__validate__(poly)\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[\n            typing.List[typing.List[typing.List[typing.List[float]]]]\n        ],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls(\n            [\n                [\n                    [(point[0], point[1]) for point in subpolygon]\n                    for subpolygon in polygon\n                ]\n                for polygon in value\n            ]\n        )\n\n    @property\n    def area(self):\n        \"\"\"\n        Symbolic representation of area.\n        \"\"\"\n        if not isinstance(self._value, Symbol):\n            raise ValueError(\n                \"attribute 'area' is reserved for symbolic variables.\"\n            )\n        return Float.symbolic(\n            name=self._value._name,\n            key=self._value._key,\n            attribute=\"area\",\n        )\n\n    def to_polygons(self) -> typing.List[Polygon]:\n        \"\"\"\n        Converts multipolygon to a list of Polygon instances.\n\n        Returns\n        -------\n        List[Polygon]\n        \"\"\"\n        return [Polygon(poly) for poly in self.get_value()]\n\n    @classmethod\n    def from_polygons(self, polygons: typing.List[Polygon]) -> \"MultiPolygon\":\n        \"\"\"\n        Converts a list of Polygon instances to a MultiPolygon.\n\n        Parameters\n        ----------\n        polygons : List[Polygon]\n            A list of Polygon instances.\n\n        Returns\n        -------\n        MultiPolygon\n            A MultiPolygon instance.\n        \"\"\"\n        return MultiPolygon([poly.get_value() for poly in polygons])\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon-attributes","title":"Attributes","text":""},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.area","title":"valor.schemas.MultiPolygon.area  property","text":"
Symbolic representation of area.
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.decode_value","title":"valor.schemas.MultiPolygon.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[\n        typing.List[typing.List[typing.List[typing.List[float]]]]\n    ],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls(\n        [\n            [\n                [(point[0], point[1]) for point in subpolygon]\n                for subpolygon in polygon\n            ]\n            for polygon in value\n        ]\n    )\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.from_polygons","title":"valor.schemas.MultiPolygon.from_polygons(polygons)  classmethod","text":"
Converts a list of Polygon instances to a MultiPolygon.
 
Parameters:
 Name Type Description Default polygons List[Polygon] 
A list of Polygon instances.
 required 
Returns:
 Type Description MultiPolygon 
A MultiPolygon instance.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_polygons(self, polygons: typing.List[Polygon]) -> \"MultiPolygon\":\n    \"\"\"\n    Converts a list of Polygon instances to a MultiPolygon.\n\n    Parameters\n    ----------\n    polygons : List[Polygon]\n        A list of Polygon instances.\n\n    Returns\n    -------\n    MultiPolygon\n        A MultiPolygon instance.\n    \"\"\"\n    return MultiPolygon([poly.get_value() for poly in polygons])\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.to_polygons","title":"valor.schemas.MultiPolygon.to_polygons()","text":"
Converts multipolygon to a list of Polygon instances.
 
Returns:
 Type Description List[Polygon] Source code in valor/schemas/symbolic/types.py 
def to_polygons(self) -> typing.List[Polygon]:\n    \"\"\"\n    Converts multipolygon to a list of Polygon instances.\n\n    Returns\n    -------\n    List[Polygon]\n    \"\"\"\n    return [Polygon(poly) for poly in self.get_value()]\n
"},{"location":"client_api/Schemas/Spatial/Point/","title":"Point","text":"
               Bases: Spatial, Equatable
 
Represents a point in 2D space.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value Tuple[float, float] 
A point.
 required 
Examples:
 
>>> Point((1,2))\n
 Source code in valor/schemas/symbolic/types.py 
class Point(Spatial, Equatable):\n    \"\"\"\n    Represents a point in 2D space.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : Tuple[float, float], optional\n        A point.\n\n    Examples\n    --------\n    >>> Point((1,2))\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.Tuple[float, float],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, tuple):\n            raise TypeError(\n                f\"Expected type 'typing.Tuple[float, float]' received type '{type(value).__name__}'\"\n            )\n        elif len(value) != 2:\n            raise ValueError(\"\")\n        for item in value:\n            if not isinstance(item, (int, float, np.floating)):\n                raise TypeError(\n                    f\"Expected type '{float.__name__}' received type '{type(item).__name__}'\"\n                )\n\n    @classmethod\n    def decode_value(cls, value: typing.Optional[typing.List[float]]):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls((value[0], value[1]))\n\n    def encode_value(self) -> typing.Any:\n        \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n        value = self.get_value()\n        if value is None:\n            return None\n        return (float(value[0]), float(value[1]))\n\n    def tuple(self):\n        return self.get_value()\n\n    def resize(\n        self,\n        og_img_h=10,\n        og_img_w=10,\n        new_img_h=100,\n        new_img_w=100,\n    ):\n        value = self.get_value()\n        h_ratio = new_img_h / og_img_h\n        w_ratio = new_img_w / og_img_w\n        return Point((value[0] * h_ratio, value[1] * w_ratio))\n\n    @property\n    def x(self):\n        return self.get_value()[0]\n\n    @property\n    def y(self):\n        return self.get_value()[1]\n
"},{"location":"client_api/Schemas/Spatial/Point/#valor.schemas.Point-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Point/#valor.schemas.Point.decode_value","title":"valor.schemas.Point.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(cls, value: typing.Optional[typing.List[float]]):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls((value[0], value[1]))\n
"},{"location":"client_api/Schemas/Spatial/Point/#valor.schemas.Point.encode_value","title":"valor.schemas.Point.encode_value()","text":"
Encode object to JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
def encode_value(self) -> typing.Any:\n    \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n    value = self.get_value()\n    if value is None:\n        return None\n    return (float(value[0]), float(value[1]))\n
"},{"location":"client_api/Schemas/Spatial/Polygon/","title":"Polygon","text":"
               Bases: Spatial
 
Represents a polygon with a boundary and optional holes.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[List[Tuple[float, float]]] 
A polygon.
 required 
Attributes:
 Name Type Description area Float boundary List[Tuple[float, float]] holes List[List[Tuple[float, float]]] xmin float xmax float ymin float ymax float 
Examples:
 
Create a polygon without any holes.
 
>>> Polygon([[(0,0), (0,1), (1,1), (0,0)]])\n
 
Create a polygon with 2 holes.
 
>>> Polygon(\n...     [\n...         [(0,0), (0,1), (1,1), (0,0)],\n...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n...     ]\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class Polygon(Spatial):\n    \"\"\"\n    Represents a polygon with a boundary and optional holes.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[List[Tuple[float, float]]], optional\n        A polygon.\n\n    Attributes\n    ----------\n    area\n    boundary\n    holes\n    xmin\n    xmax\n    ymin\n    ymax\n\n    Examples\n    --------\n    Create a polygon without any holes.\n    >>> Polygon([[(0,0), (0,1), (1,1), (0,0)]])\n\n    Create a polygon with 2 holes.\n    >>> Polygon(\n    ...     [\n    ...         [(0,0), (0,1), (1,1), (0,0)],\n    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n    ...     ]\n    ... )\n    \"\"\"\n\n    def __init__(\n        self, value: typing.List[typing.List[typing.Tuple[float, float]]]\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        MultiLineString.__validate__(value)\n        for line in value:\n            if not (len(line) >= 4 and line[0] == line[-1]):\n                raise ValueError(\n                    \"Polygons are defined by at least 4 points with the first point being repeated at the end.\"\n                )\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls(\n            [\n                [(point[0], point[1]) for point in subpolygon]\n                for subpolygon in value\n            ]\n        )\n\n    @property\n    def area(self) -> Float:\n        \"\"\"\n        Symbolic representation of area.\n        \"\"\"\n        if not isinstance(self._value, Symbol):\n            raise ValueError\n        return Float.symbolic(\n            name=self._value._name,\n            key=self._value._key,\n            attribute=\"area\",\n        )\n\n    @property\n    def boundary(self) -> typing.List[typing.Tuple[float, float]]:\n        \"\"\"\n        The boundary of the polygon.\n\n        Returns\n        -------\n        List[Tuple(float, float)]\n            A list of points.\n        \"\"\"\n        value = self.get_value()\n        if value is None:\n            raise ValueError(\"Polygon is 'None'\")\n        return value[0]\n\n    @property\n    def holes(self) -> typing.List[typing.List[typing.Tuple[float, float]]]:\n        \"\"\"\n        typing.Any holes in the polygon.\n\n        Returns\n        -------\n        List[List[Tuple(float, float)]]\n            A list of holes.\n        \"\"\"\n        value = self.get_value()\n        if value is None:\n            raise ValueError(\"Polygon is 'None'\")\n        return value[1:]\n\n    @property\n    def xmin(self) -> float:\n        \"\"\"\n        Minimum x-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return min([p[0] for p in self.boundary])\n\n    @property\n    def xmax(self) -> float:\n        \"\"\"\n        Maximum x-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return max([p[0] for p in self.boundary])\n\n    @property\n    def ymin(self) -> float:\n        \"\"\"\n        Minimum y-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return min([p[1] for p in self.boundary])\n\n    @property\n    def ymax(self) -> float:\n        \"\"\"\n        Maximum y-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return max([p[1] for p in self.boundary])\n
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon-attributes","title":"Attributes","text":""},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.area","title":"valor.schemas.Polygon.area: Float  property","text":"
Symbolic representation of area.
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.boundary","title":"valor.schemas.Polygon.boundary: typing.List[typing.Tuple[float, float]]  property","text":"
The boundary of the polygon.
 
Returns:
 Type Description List[Tuple(float, float)] 
A list of points.
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.holes","title":"valor.schemas.Polygon.holes: typing.List[typing.List[typing.Tuple[float, float]]]  property","text":"
typing.Any holes in the polygon.
 
Returns:
 Type Description List[List[Tuple(float, float)]] 
A list of holes.
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.xmax","title":"valor.schemas.Polygon.xmax: float  property","text":"
Maximum x-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.xmin","title":"valor.schemas.Polygon.xmin: float  property","text":"
Minimum x-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.ymax","title":"valor.schemas.Polygon.ymax: float  property","text":"
Maximum y-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.ymin","title":"valor.schemas.Polygon.ymin: float  property","text":"
Minimum y-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.decode_value","title":"valor.schemas.Polygon.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls(\n        [\n            [(point[0], point[1]) for point in subpolygon]\n            for subpolygon in value\n        ]\n    )\n
"},{"location":"client_api/Schemas/Spatial/Raster/","title":"Raster","text":"
               Bases: Spatial
 
Represents a binary mask.
 
Parameters:
 Name Type Description Default value Dict[str, Union[ndarray, str, None]] 
An raster value.
 required 
Attributes:
 Name Type Description area Float array ndarray geometry Union[Box, Polygon, MultiPolygon] height int width int 
Raises:
 Type Description TypeError 
If encoding is not a string.
 
Examples:
 
Generate a random mask.
 
>>> import numpy.random\n>>> height = 640\n>>> width = 480\n>>> array = numpy.random.rand(height, width)\n
 
Convert to binary mask.
 
>>> mask = (array > 0.5)\n
 
Create Raster.
 
>>> Raster.from_numpy(mask)\n
 Source code in valor/schemas/symbolic/types.py 
class Raster(Spatial):\n    \"\"\"\n    Represents a binary mask.\n\n    Parameters\n    ----------\n    value : Dict[str, typing.Union[np.ndarray, str, None]], optional\n        An raster value.\n\n    Attributes\n    ----------\n    area\n    array\n    geometry\n    height\n    width\n\n    Raises\n    ------\n    TypeError\n        If `encoding` is not a string.\n\n    Examples\n    --------\n    Generate a random mask.\n    >>> import numpy.random\n    >>> height = 640\n    >>> width = 480\n    >>> array = numpy.random.rand(height, width)\n\n    Convert to binary mask.\n    >>> mask = (array > 0.5)\n\n    Create Raster.\n    >>> Raster.from_numpy(mask)\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.Dict[\n            str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]\n        ],\n    ):\n        \"\"\"\n        Initialize and instance of a raster.\n\n        Parameters\n        ----------\n        value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]\n            The raster in dictionary format {\"mask\": <np.ndarray>, \"geometry\": <geometry | None>}.\n        \"\"\"\n        super().__init__(value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        \"\"\"\n        Validates typing.\n\n        Parameters\n        ----------\n        value : Any\n            The value to validate.\n\n        Raises\n        ------\n        TypeError\n            If the value type is not supported.\n        \"\"\"\n        if not isinstance(value, dict):\n            raise TypeError(\n                \"Raster should contain a dictionary describing a mask and optionally a geometry.\"\n            )\n        elif set(value.keys()) != {\"mask\", \"geometry\"}:\n            raise ValueError(\n                \"Raster should be described by a dictionary with keys 'mask' and 'geometry'\"\n            )\n        elif not isinstance(value[\"mask\"], np.ndarray):\n            raise TypeError(\n                f\"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'\"\n            )\n        elif len(value[\"mask\"].shape) != 2:\n            raise ValueError(\"raster only supports 2d arrays\")\n        elif value[\"mask\"].dtype != bool:\n            raise ValueError(\n                f\"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}\"\n            )\n        elif (\n            value[\"geometry\"] is not None\n            and not Polygon.supports(value[\"geometry\"])\n            and not MultiPolygon.supports(value[\"geometry\"])\n        ):\n            raise TypeError(\n                \"Expected geometry to conform to either Polygon or MultiPolygon or be 'None'\"\n            )\n\n    def encode_value(self) -> typing.Any:\n        \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n        value = self.get_value()\n        if value is None:\n            return None\n        f = io.BytesIO()\n        PIL.Image.fromarray(value[\"mask\"]).save(f, format=\"PNG\")\n        f.seek(0)\n        mask_bytes = f.read()\n        f.close()\n        return {\n            \"mask\": b64encode(mask_bytes).decode(),\n            \"geometry\": value[\"geometry\"],\n        }\n\n    @classmethod\n    def decode_value(cls, value: typing.Any):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        if not (\n            isinstance(value, dict)\n            and set(value.keys()) == {\"mask\", \"geometry\"}\n        ):\n            raise ValueError(\n                f\"Improperly formatted raster encoding. Received '{value}'\"\n            )\n        mask_bytes = b64decode(value[\"mask\"])\n        with io.BytesIO(mask_bytes) as f:\n            img = PIL.Image.open(f)\n            value = {\n                \"mask\": np.array(img),\n                \"geometry\": value[\"geometry\"],\n            }\n        return cls(value=value)\n\n    @classmethod\n    def from_numpy(cls, mask: np.ndarray):\n        \"\"\"\n        Create a Raster object from a NumPy array.\n\n        Parameters\n        ----------\n        mask : np.ndarray\n            The 2D binary array representing the mask.\n\n        Returns\n        -------\n        Raster\n\n        Raises\n        ------\n        ValueError\n            If the input array is not 2D or not of dtype bool.\n        \"\"\"\n        return cls(value={\"mask\": mask, \"geometry\": None})\n\n    @classmethod\n    def from_geometry(\n        cls,\n        geometry: typing.Union[Box, Polygon, MultiPolygon],\n        height: int,\n        width: int,\n    ):\n        \"\"\"\n        Create a Raster object from a geometric mask.\n\n        Parameters\n        ----------\n        geometry : Union[Box, Polygon, MultiPolygon]\n            Defines the bitmask as a geometry. Overrides any existing mask.\n        height : int\n            The intended height of the binary mask.\n        width : int\n            The intended width of the binary mask.\n\n        Returns\n        -------\n        Raster\n        \"\"\"\n        bitmask = np.full((int(height), int(width)), False)\n        return cls(value={\"mask\": bitmask, \"geometry\": geometry.get_value()})\n\n    @property\n    def area(self) -> Float:\n        \"\"\"\n        Symbolic representation of area.\n        \"\"\"\n        if not isinstance(self._value, Symbol):\n            raise ValueError\n        return Float.symbolic(\n            name=self._value._name,\n            key=self._value._key,\n            attribute=\"area\",\n        )\n\n    @property\n    def array(self) -> np.ndarray:\n        \"\"\"\n        The bitmask as a numpy array.\n\n        Returns\n        -------\n        Optional[np.ndarray]\n            A 2D binary array representing the mask if it exists.\n        \"\"\"\n        value = self.get_value()\n        if value[\"geometry\"] is not None:\n            warnings.warn(\n                \"Raster array does not contain bitmask as this is a geometry-defined raster.\",\n                RuntimeWarning,\n            )\n        return value[\"mask\"]\n\n    @property\n    def geometry(self) -> typing.Union[Box, Polygon, MultiPolygon]:\n        \"\"\"\n        The geometric mask if it exists.\n\n        Returns\n        -------\n        Box | Polygon | MultiPolygon | None\n            The geometry if it exists.\n        \"\"\"\n        return self.get_value()[\"geometry\"]\n\n    @property\n    def height(self) -> int:\n        \"\"\"Returns the height of the raster if it exists.\"\"\"\n        return self.array.shape[0]\n\n    @property\n    def width(self) -> int:\n        \"\"\"Returns the width of the raster if it exists.\"\"\"\n        return self.array.shape[1]\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster-attributes","title":"Attributes","text":""},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.area","title":"valor.schemas.Raster.area: Float  property","text":"
Symbolic representation of area.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.array","title":"valor.schemas.Raster.array: np.ndarray  property","text":"
The bitmask as a numpy array.
 
Returns:
 Type Description Optional[ndarray] 
A 2D binary array representing the mask if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.geometry","title":"valor.schemas.Raster.geometry: typing.Union[Box, Polygon, MultiPolygon]  property","text":"
The geometric mask if it exists.
 
Returns:
 Type Description Box | Polygon | MultiPolygon | None 
The geometry if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.height","title":"valor.schemas.Raster.height: int  property","text":"
Returns the height of the raster if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.width","title":"valor.schemas.Raster.width: int  property","text":"
Returns the width of the raster if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.__init__","title":"valor.schemas.Raster.__init__(value)","text":"
Initialize and instance of a raster.
 
Parameters:
 Name Type Description Default value Dict[str, Union[ndarray, Box, Polygon, MultiPolygon, None]] 
The raster in dictionary format {\"mask\": , \"geometry\": }. required Source code in valor/schemas/symbolic/types.py 
def __init__(\n    self,\n    value: typing.Dict[\n        str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]\n    ],\n):\n    \"\"\"\n    Initialize and instance of a raster.\n\n    Parameters\n    ----------\n    value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]\n        The raster in dictionary format {\"mask\": <np.ndarray>, \"geometry\": <geometry | None>}.\n    \"\"\"\n    super().__init__(value)\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.__validate__","title":"valor.schemas.Raster.__validate__(value)  classmethod","text":"
Validates typing.
 
Parameters:
 Name Type Description Default value Any 
The value to validate.
 required 
Raises:
 Type Description TypeError 
If the value type is not supported.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef __validate__(cls, value: typing.Any):\n    \"\"\"\n    Validates typing.\n\n    Parameters\n    ----------\n    value : Any\n        The value to validate.\n\n    Raises\n    ------\n    TypeError\n        If the value type is not supported.\n    \"\"\"\n    if not isinstance(value, dict):\n        raise TypeError(\n            \"Raster should contain a dictionary describing a mask and optionally a geometry.\"\n        )\n    elif set(value.keys()) != {\"mask\", \"geometry\"}:\n        raise ValueError(\n            \"Raster should be described by a dictionary with keys 'mask' and 'geometry'\"\n        )\n    elif not isinstance(value[\"mask\"], np.ndarray):\n        raise TypeError(\n            f\"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'\"\n        )\n    elif len(value[\"mask\"].shape) != 2:\n        raise ValueError(\"raster only supports 2d arrays\")\n    elif value[\"mask\"].dtype != bool:\n        raise ValueError(\n            f\"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}\"\n        )\n    elif (\n        value[\"geometry\"] is not None\n        and not Polygon.supports(value[\"geometry\"])\n        and not MultiPolygon.supports(value[\"geometry\"])\n    ):\n        raise TypeError(\n            \"Expected geometry to conform to either Polygon or MultiPolygon or be 'None'\"\n        )\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.decode_value","title":"valor.schemas.Raster.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(cls, value: typing.Any):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    if not (\n        isinstance(value, dict)\n        and set(value.keys()) == {\"mask\", \"geometry\"}\n    ):\n        raise ValueError(\n            f\"Improperly formatted raster encoding. Received '{value}'\"\n        )\n    mask_bytes = b64decode(value[\"mask\"])\n    with io.BytesIO(mask_bytes) as f:\n        img = PIL.Image.open(f)\n        value = {\n            \"mask\": np.array(img),\n            \"geometry\": value[\"geometry\"],\n        }\n    return cls(value=value)\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.encode_value","title":"valor.schemas.Raster.encode_value()","text":"
Encode object to JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
def encode_value(self) -> typing.Any:\n    \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n    value = self.get_value()\n    if value is None:\n        return None\n    f = io.BytesIO()\n    PIL.Image.fromarray(value[\"mask\"]).save(f, format=\"PNG\")\n    f.seek(0)\n    mask_bytes = f.read()\n    f.close()\n    return {\n        \"mask\": b64encode(mask_bytes).decode(),\n        \"geometry\": value[\"geometry\"],\n    }\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.from_geometry","title":"valor.schemas.Raster.from_geometry(geometry, height, width)  classmethod","text":"
Create a Raster object from a geometric mask.
 
Parameters:
 Name Type Description Default geometry Union[Box, Polygon, MultiPolygon] 
Defines the bitmask as a geometry. Overrides any existing mask.
 required height int 
The intended height of the binary mask.
 required width int 
The intended width of the binary mask.
 required 
Returns:
 Type Description Raster Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_geometry(\n    cls,\n    geometry: typing.Union[Box, Polygon, MultiPolygon],\n    height: int,\n    width: int,\n):\n    \"\"\"\n    Create a Raster object from a geometric mask.\n\n    Parameters\n    ----------\n    geometry : Union[Box, Polygon, MultiPolygon]\n        Defines the bitmask as a geometry. Overrides any existing mask.\n    height : int\n        The intended height of the binary mask.\n    width : int\n        The intended width of the binary mask.\n\n    Returns\n    -------\n    Raster\n    \"\"\"\n    bitmask = np.full((int(height), int(width)), False)\n    return cls(value={\"mask\": bitmask, \"geometry\": geometry.get_value()})\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.from_numpy","title":"valor.schemas.Raster.from_numpy(mask)  classmethod","text":"
Create a Raster object from a NumPy array.
 
Parameters:
 Name Type Description Default mask ndarray 
The 2D binary array representing the mask.
 required 
Returns:
 Type Description Raster 
Raises:
 Type Description ValueError 
If the input array is not 2D or not of dtype bool.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_numpy(cls, mask: np.ndarray):\n    \"\"\"\n    Create a Raster object from a NumPy array.\n\n    Parameters\n    ----------\n    mask : np.ndarray\n        The 2D binary array representing the mask.\n\n    Returns\n    -------\n    Raster\n\n    Raises\n    ------\n    ValueError\n        If the input array is not 2D or not of dtype bool.\n    \"\"\"\n    return cls(value={\"mask\": mask, \"geometry\": None})\n
"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Introduction","text":"
Valor is a centralized evaluation store that makes it easy to measure, explore, and rank model performance. Valor empowers data scientists and engineers to evaluate the performance of their machine learning pipelines and use those evaluations to make better modeling decisions in the future. To skip this textual introduction and dive right in, first go here for instructions to setup the Valor service, and then checkout the sample notebooks.
 
Valor is maintained by Striveworks, a cutting-edge machine learning operations (MLOps) company based out of Austin, Texas. We'd love to learn more about your interest in Valor and answer any questions you may have; please don't hesitate to reach out to us on Slack or GitHub.
 
These docs are organized as follows:
 
     
  • Overview (this page): Provides an overview of what Valor is, why it's important, and how it works.
    •  
  • Installation: Explains how to install Valor.
    •  
  • Getting Started Notebook: Details everything you need to get up and running with using Valor.
    •  
  • All Sample Notebooks: Collection of descriptive Jupyter notebooks giving examples of how to evaluate model performance using Valor.
    •  
  • Metadata and Filtering: Describes Valor's robust support for adding metadata to data, along with how to filter evaluations and Valor objects based on metadata and other attributes.
    •  
  • Metrics: Describes all of the metrics that you can calculate using Valor.
    •  
  • Endpoints: Documents Valor's various API endpoints.
    •  
  • Technical Concepts: Describes the technical concepts that underpin Valor.
    •  
  • Contributing and Development: Explains how you can build on and contribute to Valor.
    •  
  • Python Client API: Shares reference documentation for our Python client.
    •  
"},{"location":"#overview","title":"Overview","text":"
In this section, we'll explore what Valor is, why it's important, and provide a high-level description of how it works. This overview is also available in the following five-minute video:
"},{"location":"#use-cases-for-a-containerized-evaluation-store","title":"Use Cases for a Containerized Evaluation Store","text":"
As we've worked with dozens of data scientists and engineers on their MLOps pipelines, we have identified three important questions that an effective evaluation store could help them answer. First, they wanted to understand: \"Of the various models I tested for a given dataset, which one performs best?\". This is a very common and important use case\u2014and one that is often solved on a model-to-model basis in a local Jupyter notebook. This focus on bespoke implementations limits traceability and makes it difficult to create apples-to-apples comparisons between new model runs and prior model runs.
 
Second, our users wanted to understand: \"How does the performance of a particular model vary across datasets?\". We found that many practitioners use the same computer vision model (e.g., YOLOv8) for a variety of supervised learning tasks, and they needed a way to identify patterns where that particular model didn't meet expectations.
 
Finally, our users wanted to understand: \"How can I use my prior evaluations to pick the best model for a future ML pipeline?\". This last question requires the ability to filter previous evaluations on granular metadata (e.g., time of day, geospatial coordinates, etc.) in order to provide tailored recommendations regarding which model to pick in the future.
 
With these three use cases in mind, we set out to build a centralized evaluation store that we later named Valor.
"},{"location":"#introducing-valor","title":"Introducing Valor","text":"
Valor is a centralized evaluation store that makes it easy to measure, explore, and rank model performance. Our ultimate goal with Valor is to help data scientists and engineers pick the right ML model for their specific needs. To that end, we built Valor with three design principles in mind:
 
     
  • Valor works with any dataset or model: We believe Valor should be able to handle any supervised learning task that you want to throw at it. Just pass in your ground truth annotations and predictions, describe your learning task (i.e., object detection), and Valor will do the rest. (Note: At launch, Valor will only support classification and computer vision (i.e., image segmentation and object detection) tasks. We're confident this framework will abstract well to other supervised learning tasks and plan to support them in later releases).
    •  
  • Valor can handle any type of image, model, or dataset metadata you throw at it: Metadata is a critical component of any evaluation store as it enables the system to offer tailored model recommendations based on a user's specific needs. To that end, we built Valor to handle any metadata under the sun. Dates, geospatial coordinates, and even JSONs filled with configuration details are all on the table. This means you can slice and dice your evaluations any way you want: just pass in the right labels for your use case and define your filter (say a geographic bounding box), and you\u2019ll get back results for your specific needs.
    •  
  • Valor standardizes the evaluation process: The trickiest part of comparing two different model runs is avoiding apples-to-oranges comparisons. Valor helps you audit your metrics and avoid false comparisons by versioning your uploads, storing them in a centralized location, and ensuring that you only compare runs that used the exact same filters and metrics.
    •  
"},{"location":"#how-it-works-an-illustrative-example","title":"How It Works: An Illustrative Example","text":"
Let\u2019s walk through a quick example to bring Valor to life.
 
Say that you're interested in using computer vision models to detect forest fires around the world using satellite imagery. You've just been tasked with building a new ML pipeline to detect fires in an unfamiliar region of interest. How might you leverage your evaluation metrics from prior ML pipelines to understand which model will perform best for this particular use case?
 
 
To answer this question, we'll start by passing in three pieces of information from each of our prior modeling runs:
 
     
  • GroundTruths: First, we'll pass in human-annotated bounding boxes to tell Valor exactly where forest fires can be found across all of the satellite images used in prior runs.
    •  
  • Predictions: Next, we'll pass machine-generated predictions for each image (also in the form of bounding boxes) so that Valor can evaluate how well each model did at predicting forest fires.
    •  
  • Labels: Finally, we'll pass metadata to Valor describing each of our various images (e.g., the time of day the photo was taken, the geospatial coordinates of the forest in the photo, etc.). We'll use this metadata later on in order to identify the right model for our new use case.
    •  
 
Once we pass in these three ingredients, Valor will compare all of our GroundTruths and Predictions in order to calculate various evaluation metrics (i.e., mean average precision or mAP). These metrics, Labels, GroundTruths, and Predictions, will all be stored in Postgres, with PostGIS support for fast geospatial lookups and geometric comparisons at a later date.
 
Finally, once all of our previous pipeline runs and evaluations are stored in Valor, we can use Valor\u2019s API to specify our exact filter criteria and get back its model rankings. In this case, we can ask Valor to find us the best model for detecting forest fires at night in a 50 mile radius around (42.36, -71.03), sorted by mAP. Valor will then filter all of our stored evaluation metrics, rank each model with evaluations that meet our criteria, and send back all relevant evaluation metrics to help us determine which model to use for our new modeling pipeline.
 
"},{"location":"#next-steps","title":"Next Steps","text":"
We'd recommend reviewing our \"Getting Started\" sample notebook to become further acquainted with Valor. For more detailed explanations of Valor's technical underpinnings, see our technical concepts guide.
"},{"location":"#faq","title":"FAQ","text":"
Q. What is Valor?
 
A. Valor is a centralized evaluation store that makes it easy to measure, explore, and rank model performance. For an overview of what Valor is and why it's important, please refer to our high-level overview.
 
Q. What evaluation methods are supported?
 
A. Valor currently supports generic classification as well as object-detection and semantic-segmentation for images. The long-term goal for Valor is to support the most popular supervised learning methods.
 
Q. Does Valor store data?
 
A. Valor only stores ground truth annotations, model predictions, and user-defined metadata.
 
Q. What is a Datum?
 
A. A valor.Datum object is a generic type that represents a datum in the context of a machine learning workflow. The object stores a UID and related metadata in a dictionary. This metadata allows for the user to construct their own abstraction layer by mapping a real-world type (e.g., an image) into a valor.Datum type.
 
from valor.metatypes import ImageMetadata\nimage = ImageMetadata.create(\n  uid = \"1234\",\n  height = 100,\n  width = 100,\n)\n\n# access the datum\ndatum = image.datum\n
 
Q. What is a GroundTruth?
 
A. valor.GroundTruth objects in Valor each represent a singular datum and its associated annotations that provide a reference standard or the 'truth' against which predictions are compared. There cannot be multiple ground truths per datum.
 
Q. What is a Prediction?
 
A. valor.Prediction objects are similar to valor.GroundTruth objects in that they also contain a list of annotations over a datum. However, these annotations are generated by a model as inferences, and the object also includes the name of the model that was used for creating these inferences. There cannot be multiple predictions by the same model over a single datum.
 
Q. Can Valor handle multiple data types?
 
A. Valor abstracts data types through metadata. An example of this can be seen in valor.metatypes.ImageMetadata which describes the mapping of an image to a valor.Datum.
 
Q. Does Valor support geospatial queries?
 
A. Valor follows the GeoJSON specification (RFC 7946) in the implementation of Point, MulitPoint, LineString, MultiLineString, Polygon and MulitPolygon geometries. These objects are used to define annotations and facilitate the creation of geospatial metadata.
"},{"location":"#troubleshooting","title":"Troubleshooting","text":"
Q. Why am I getting NotFinalizedError when trying to run an evaluation?
 
A. Valor requires both dataset and model representations to be finalized before evaluation can take place. Finalization is crucial for auditability as it ensures that data finalized at a certain date is immutable.
 
Dataset finalization is accomplished through the valor.Dataset.finalize member function.
 
from valor import Client, Dataset\nclient = Client(...)\ndataset = Dataset(name=\"test_dataset\")\n...\ndataset.finalize()\n
 
Models are finalized automatically given two conditions.
 
     
  1. The working dataset is finalized.
    2.  
  2. There is a 1:1 mapping of predictions to ground truths.
    3.  
 
Models and their predictions can also be finalized prematurely using the valor.Model.finalize_inferences member function. This will generate empty predictions with task type enums.TaskType.SKIP to achieve the 1:1 ground truth mapping.
 
from valor import Client, Dataset, Model\nclient = Client(...)\ndataset = Dataset(name=\"test_dataset\")\nmodel = Model(name=\"test_model\")\n...\ndataset.finalize()\nmodel.finalize_inferences(dataset)\n
 
Q. Why am I getting GDAL driver errors?
 
A. For some computations (mostly involving rasters), Valor requires the PostGIS database to have all GDAL drivers enabled. The Valor back end attempts to enable these drivers, but it might not have permission depending on your specific setup. If you encounter this error, see here for ways to enable the drivers directly in the PostGIS instance.
"},{"location":"contributing/","title":"Contributing to Valor","text":"
We welcome all contributions, bug reports, bug fixes, documentation improvements, enhancements, and ideas aimed at improving Valor. This doc describes the high-level process for how to contribute to this repository. If you have any questions or comments about this process, please feel free to reach out to us on Slack.
"},{"location":"contributing/#on-github","title":"On GitHub","text":"
We use Git on GitHub to manage this repo, which means you will need to sign up for a free GitHub account to submit issues, ideas, and pull requests. We use Git for version control to allow contributors from all over the world to work together on this project.
 
If you are new to Git, these official resources can help bring you up to speed:
 
     
  • GitHub documentation for forking a repo
    •  
  • GitHub documentation for collaborating with pull requests
    •  
  • GitHub documentation for working with forks
    •  
"},{"location":"contributing/#contribution-workflow","title":"Contribution Workflow","text":"
Generally, the high-level workflow for contributing to this repo includes:
 
     
  1. Submitting an issue or enhancement request using the appropriate template on GitHub Issues.
    2.  
  2. Gathering feedback from devs and the broader community in your issue before starting to code.
    3.  
  3. Forking the Valor repo, making your proposed changes, and submitting a pull request (PR). When submitting a PR, please be sure to:
       
    1. Update the README.md and/or any relevant docstrings with details of your change.
      2.  
    2. Add tests where necessary.
      3.  
    3. Run pre-commit install on your local repo before your last commit to ensure your changes follow our formatting guidelines.
      4.  
    4. Double-check that your code passes all of the tests that are automated via GitHub Actions.
      5.  
    5. Ping us on Slack to ensure timely review.
      6.  
     
    4.  
  4. Working with repo maintainers to review and improve your PR before it is merged into the official repo.
    5.  
 
For questions or comments on this process, please reach out to us at any time on Slack.
"},{"location":"contributing/#development-tips-and-tricks","title":"Development Tips and Tricks","text":""},{"location":"contributing/#deploying-the-back-end-for-development","title":"Deploying the Back End for Development","text":""},{"location":"contributing/#docker-compose","title":"Docker Compose","text":"
The fastest way to test the API and Python client is via Docker Compose. Start by setting the environment variable POSTGRES_PASSWORD to your liking, and then start Docker and build the container:
 
export POSTGRES_PASSWORD=\"my_password\"\ndocker compose up\n
"},{"location":"contributing/#makefile-requires-docker","title":"Makefile (requires Docker)","text":"
Alternatively, you may want to run the API service from a terminal to enable faster debugging. To start the service, you can run:
 
pip install api # Install the API in your python environment\n\nexport POSTGRES_PASSWORD=password\nexport POSTGRES_HOST=localhost\nmake start-postgres-docker # Start the custom postgres service in Docker\nmake run-migrations # Instantiate the table schemas in Postgres\nmake start-server # Start the API service locally\n
"},{"location":"contributing/#setting-up-your-environment","title":"Setting Up Your Environment","text":"
Creating a Valor-specific Python environment at the start of development can help you avoid dependency and versioning issues later on. To start, we'd recommend activating a new Python environment:
 
# venv\npython3 -m venv .env-valor\nsource .env-valor/bin/activate\n\n# conda\nconda create --name valor python=3.11\nconda activate valor\n
 
Next, install pre-commit to ensure formatting consistency throughout your repo:
 
pip install pre-commit\npre-commit install\n
 
Finally, you're ready to install your client and API modules:
 
# Install the Client module\npython -m pip install -e client/.\n\n# Install the API module\npython -m pip install -e api/.\n
"},{"location":"contributing/#use-pgadmin-to-debug-postgis","title":"Use pgAdmin to Debug PostGIS","text":"
You can use the pgAdmin utility to debug your PostGIS tables as you code. Start by installing pgAdmin, and then select Object > Register > Server to connect to your PostGIS container. The default connection details are listed below for convenience:
 
- *Host name/address*: 0.0.0.0\n- *Port*: 5432\n- *Maintenance database*: postgres\n- *Username*: postgres\n
"},{"location":"contributing/#running-tests","title":"Running Tests","text":"
All of our tests are run automatically via GitHub Actions on every push, so it's important to double-check that your code passes all local tests before committing your code. All of the tests below require pytest:
 
pip install pytest\n
"},{"location":"contributing/#running-integration-tests","title":"Running integration tests","text":"
pytest integration_tests\n
"},{"location":"contributing/#running-back-end-unit-tests","title":"Running back end unit tests","text":"
pytest api/tests/unit-tests\n
"},{"location":"contributing/#running-back-end-functional-tests","title":"Running back end functional tests","text":"
Note: Functional tests require a running instance of PostgreSQL, which you can start using make start-postgres-docker.
 
POSTGRES_PASSWORD=password \\\nPOSTGRES_HOST=localhost \\\npytest api/tests/functional-tests/\n
"},{"location":"endpoints/","title":"Endpoints","text":""},{"location":"installation/","title":"Installation","text":"
Valor comprises two services: a back-end service (which consists of a REST API and a Postgres database with the PostGIS extension), and a Python client for interacting with the back-end service.
"},{"location":"installation/#setting-up-the-back-end-service","title":"Setting up the back-end service","text":""},{"location":"installation/#using-docker-compose","title":"Using Docker Compose","text":"
The easiest way to get up and running with Valor is to use Docker Compose with the docker-compose.yml file in the repository root:
 
git clone https://github.com/striveworks/valor\ncd valor\ndocker compose --env-file ./api/.env.testing up\n
 
This will set up the necessary environment variables, start both the API and database services, and run the database migration job. The endpoint localhost:8000/health should return {\"status\":\"ok\"} if all of Valor's services were started correctly.
 
Note: running Valor this way is not intended for production and scalable use and is only recommended for development and testing purposes.
"},{"location":"installation/#deploying-via-docker-and-a-hosted-database","title":"Deploying via Docker and a hosted database","text":"
For a more production-grade deployment, we publish the images ghcr.io/striveworks/valor/valor-service (used for the REST API) and ghcr.io/striveworks/valor/migrations (used for setting up the database and migrations). These can be paired with any Postgres database with the PostGIS extension.
 
The following environment variables are required for running these images:
 Variable Description Images that need it POSTGRES_HOST The host of the Postgres database valor-service, migrations POSTGRES_PORT The port of the Postgres database valor-service, migrations POSTGRES_DB The name of the Postgres database valor-service, migrations POSTGRES_USERNAME The user of the Postgres database valor-service, migrations POSTGRES_PASSWORD The password of the Postgres database valor-service, migrations POSTGRES_SSLMODE Sets the Postgres instance SSL mode (typically needs to be \"require\") migrations API_ROOT_PATH The root path of the API (if serving behind a proxy) valor-service 
Additionally, the Valor REST API has an optional single username/password/bearer token authentication. To enable this feature, the valor-service image requires the following environment variables:
 Variable Description VALOR_USERNAME The username to use VALOR_PASSWORD The password to use VALOR_SECRET_KEY A random, secret string used for signing JWT tokens"},{"location":"installation/#manual-deployment","title":"Manual deployment","text":"
If you would prefer to build your own image or if you want a debug console for the back-end, please see the deployment instructions in Contributing to Valor.
"},{"location":"installation/#setting-up-the-python-client","title":"Setting up the Python client","text":"
The Python client can be installed via pip:
 
pip install valor-client\n
"},{"location":"metadata_and_filtering/","title":"Metadata and Filtering","text":""},{"location":"metadata_and_filtering/#metadata","title":"Metadata","text":"
Valor offers rich support for attaching metadata to almost any object, which can then be used to filter, group, and organize objects in Valor.
 
The metadata types supported are:
 
     
  • simple data types (strings, numerics, boolean)
    •  
  • datetimes (via datetime.datetime, datetime.date, datetime.time, and datetime.timedelta in the Valor client)
    •  
  • geometries and geographies (via GeoJSON)
    •  
 
Metadata is added on object creation. For example, if you want to use metadata to organize models that come from training run checkpoints, this may look like:
 
run_name: str\nckpt: int\n\nModel.create(name=f\"{run_name}-ckpt{ckpt}\", metadata={\"run_name\": run_name, \"ckpt\": ckpt})\n
 
or if a datum has an associated datetime of capture, that can be added in the creation stage:
 
from datetime import datetime\n\nDatum(uid=fname, metadata={\"capture_day\": datetime.datetime(day=1, month=1, year=2021)})\n
"},{"location":"metadata_and_filtering/#filtering","title":"Filtering","text":"
Valor supports filtering objects based on metadata or other attributes (such as labels or bounding boxes). One of the most important use cases of filtering is to define a subset of a dataset to evaluate a model on.
"},{"location":"metadata_and_filtering/#filtering-by-metadata","title":"Filtering by metadata","text":"
For example, using the above example where capture_day was added as metadata, one way to test model drift could be to evaluate the model over different time periods. Such a workflow may look like:
 
import datetime\n\nimport valor\n\nmodel: valor.Model # classification model\ndset: valor.Dataset # dataset to evaluate on\n\n# compare performance on data captured before and after 2020\nd = datetime.datetime(day=5, month=10, year=2020)\neval1 = model.evaluate_classification(dset, filter_by=[Datum.metadata[\"capture_day\"] < d])\neval2 = model.evaluate_classification(dset, filter_by=[Datum.metadata[\"capture_day\"] > d])\n
"},{"location":"metadata_and_filtering/#filtering-by-geometric-attributes","title":"Filtering by geometric attributes","text":"
As an example for filtering by geometric attributes, consider evaluating an object detection model's performance on small objects, where we define small as being less than 500 square pixels in area. This can be achieved via:
 
import valor\n\nmodel: valor.Model # object detection model\ndset: valor.Dataset # dataset to evaluate on\n\ndset.evaluate_detection(dset, filter_by=[valor.Annotation.bounding_box.area < 500])\n
"},{"location":"metadata_and_filtering/#filtering-in-queries","title":"Filtering in queries","text":"
Filtering can also be used when querying for different objects. For example, taking the model section checkpoint example from above, we could query model checkpoints from a training run based on the checkpoint number greater than 100 by:
 
from valor import client\n\nrun_name: str # run name to query for\n\nclient.get_models([Model.metadata[\"run_name\"] == run_name, Model.metadata[\"ckpt\"] > 100])\n
"},{"location":"metrics/","title":"Metrics","text":"
Let's look at the various metrics you can calculate using Valor.
 
If we're missing an important metric for your particular use case, please write us a GitHub Issue ticket. We love hearing your suggestions.
"},{"location":"metrics/#classification-metrics","title":"Classification Metrics","text":"Name Description Equation Precision The number of true positives divided by the total number of positive predictions (i.e., the number of true positives plus the number of false positives). \\(\\dfrac{\\|TP\\|}{\\|TP\\|+\\|FP\\|}\\) Recall The number of true positives divided by the total count of the class of interest (i.e., the number of true positives plus the number of true negatives). \\(\\dfrac{\\|TP\\|}{\\|TP\\|+\\|FN\\|}\\) F1 A weighted average of precision and recall. \\(\\frac{2 * Precision * Recall}{Precision + Recall}\\) Accuracy The number of true predictions divided by the total number of predictions. \\(\\dfrac{\\|TP\\|+\\|TN\\|}{\\|TP\\|+\\|TN\\|+\\|FP\\|+\\|FN\\|}\\) ROC AUC The area under the Receiver Operating Characteristic (ROC) curve for the predictions generated by a given model. See ROCAUC methods. Precision-Recall Curves Outputs a nested dictionary containing the true positives, false positives, true negatives, false negatives, precision, recall, and F1 score for each (label key, label value, confidence threshold) combination. See precision-recall curve methods Detailed Precision-Recall Curves Similar to PrecisionRecallCurve, except this metric a) classifies false positives as hallucinations or misclassifications, b) classifies false negatives as misclassifications or missed_detections, and c) gives example datums for each observation, up to a maximum of pr_curve_max_examples. See detailed precision-recall curve methods"},{"location":"metrics/#object-detection-and-instance-segmentation-metrics","title":"Object Detection and Instance Segmentation Metrics**","text":"Name Description Equation Average Precision (AP) The weighted mean of precisions achieved at several different recall thresholds for a single Intersection over Union (IOU), grouped by class. See AP methods. AP Averaged Over IOUs The average of several AP metrics across IOU thresholds, grouped by class labels. \\(\\dfrac{1}{\\text{number of thresholds}} \\sum\\limits_{iou \\in thresholds} AP_{iou}\\) Mean Average Precision (mAP) The average of several AP metrics, grouped by label keys and IOU thresholds. \\(\\dfrac{1}{\\text{number of labels}} \\sum\\limits_{label \\in labels} AP_{c}\\) mAP Averaged Over IOUs The average of several mAP metrics grouped by label keys. \\(\\dfrac{1}{\\text{number of thresholds}} \\sum\\limits_{iou \\in thresholds} mAP_{iou}\\) Average Recall (AR) The average of several recall metrics across IOU thresholds, grouped by class labels. See AR methods. Mean Average Recall (mAR) The average of several AR metrics, grouped by label keys. \\(\\dfrac{1}{\\text{number of labels}} \\sum\\limits_{label \\in labels} AR_{class}\\) Precision-Recall Curves Outputs a nested dictionary containing the true positives, false positives, true negatives, false negatives, precision, recall, and F1 score for each (label key, label value, confidence threshold) combination. These curves are calculated using a default IOU threshold of 0.5; you can set your own threshold by passing a float between 0 and 1 to the pr_curve_iou_threshold parameter at evaluation time. See precision-recall curve methods Detailed Precision-Recall Curves Similar to PrecisionRecallCurve, except this metric a) classifies false positives as hallucinations or misclassifications, b) classifies false negatives as misclassifications or missed_detections, and c) gives example datums and bounding boxes for each observation, up to a maximum of pr_curve_max_examples. See detailed precision-recall curve methods 
**When calculating IOUs for object detection metrics, Valor handles the necessary conversion between different types of geometric annotations. For example, if your model prediction is a polygon and your groundtruth is a raster, then the raster will be converted to a polygon prior to calculating the IOU.
"},{"location":"metrics/#semantic-segmentation-metrics","title":"Semantic Segmentation Metrics","text":"Name Description Equation Intersection Over Union (IOU) A ratio between the groundtruth and predicted regions of an image, measured as a percentage, grouped by class. \\(\\dfrac{area( prediction \\cap groundtruth )}{area( prediction \\cup groundtruth )}\\) Mean IOU The average of IOU across labels, grouped by label key. \\(\\dfrac{1}{\\text{number of labels}} \\sum\\limits_{label \\in labels} IOU_{c}\\)"},{"location":"metrics/#text-generation-metrics","title":"Text Generation Metrics","text":"Name Description Equation Answer Relevance The number of statements in the answer that are relevant to the query, divided by the total number of statements in the answer See appendix for details. Coherence Rates the coherence of a textual summary relative to some source text using a score from 1 to 5, where 5 means \"This summary is extremely coherent based on the information provided in the source text\". See appendix for details. ROUGE A score between 0 and 1 indicating how often the words in the ground truth string appeared in the predicted string (i.e., measuring recall). See appendix for details. BLEU A score between 0 and 1 indicating how much the predicted string matches the ground truth string (i.e., measuring precision), with a penalty for brevity. See appendix for details."},{"location":"metrics/#appendix-metric-calculations","title":"Appendix: Metric Calculations","text":""},{"location":"metrics/#binary-roc-auc","title":"Binary ROC AUC","text":""},{"location":"metrics/#receiver-operating-characteristic-roc","title":"Receiver Operating Characteristic (ROC)","text":"
An ROC curve plots the True Positive Rate (TPR) vs. the False Positive Rate (FPR) at different confidence thresholds.
 
In Valor, we use the confidence scores sorted in decreasing order as our thresholds. Using these thresholds, we can calculate our TPR and FPR as follows:
"},{"location":"metrics/#determining-the-rate-of-correct-predictions","title":"Determining the Rate of Correct Predictions","text":"Element Description True Positive (TP) Prediction confidence score >= threshold and is correct. False Positive (FP) Prediction confidence score >= threshold and is incorrect. True Negative (TN) Prediction confidence score < threshold and is correct. False Negative (FN) Prediction confidence score < threshold and is incorrect. 
     
  •  
    \\(\\text{True Positive Rate (TPR)} = \\dfrac{|TP|}{|TP| + |FN|} = \\dfrac{|TP(threshold)|}{|TP(threshold)| + |FN(threshold)|}\\)
     
    •  
  •  
    \\(\\text{False Positive Rate (FPR)} = \\dfrac{|FP|}{|FP| + |TN|} = \\dfrac{|FP(threshold)|}{|FP(threshold)| + |TN(threshold)|}\\)
     
    •  
 
We now use the confidence scores, sorted in decreasing order, as our thresholds in order to generate points on a curve.
 
\\(Point(score) = (FPR(score), \\ TPR(score))\\)
"},{"location":"metrics/#area-under-the-roc-curve-roc-auc","title":"Area Under the ROC Curve (ROC AUC)","text":"
After calculating the ROC curve, we find the ROC AUC metric by approximating the integral using the trapezoidal rule formula.
 
\\(ROC AUC =  \\sum_{i=1}^{|scores|} \\frac{  \\lVert Point(score_{i-1}) - Point(score_i) \\rVert }{2}\\)
 
See Classification: ROC Curve and AUC for more information.
"},{"location":"metrics/#average-precision-ap","title":"Average Precision (AP)","text":"
For object detection and instance segmentation tasks, average precision is calculated from the intersection-over-union (IOU) of geometric predictions and ground truths.
"},{"location":"metrics/#multiclass-precision-and-recall","title":"Multiclass Precision and Recall","text":"
Tasks that predict geometries (such as object detection or instance segmentation) use the ratio intersection-over-union (IOU) to calculate precision and recall. IOU is the ratio of the intersecting area over the joint area spanned by the two geometries, and is defined in the following equation.
 
\\(Intersection \\ over \\ Union \\ (IOU) = \\dfrac{Area( prediction \\cap groundtruth )}{Area( prediction \\cup groundtruth )}\\)
 
Using different IOU thresholds, we can determine whether we count a pairing between a prediction and a ground truth pairing based on their overlap.
 Case Description True Positive (TP) Prediction-GroundTruth pair exists with IOU >= threshold. False Positive (FP) Prediction-GroundTruth pair exists with IOU < threshold. True Negative (TN) Unused in multi-class evaluation. False Negative (FN) No Prediction with a matching label exists for the GroundTruth. 
     
  •  
    \\(Precision = \\dfrac{|TP|}{|TP| + |FP|} = \\dfrac{\\text{Number of True Predictions}}{|\\text{Predictions}|}\\)
     
    •  
  •  
    \\(Recall = \\dfrac{|TP|}{|TP| + |FN|} = \\dfrac{\\text{Number of True Predictions}}{|\\text{Groundtruths}|}\\)
     
    •  
"},{"location":"metrics/#matching-ground-truths-with-predictions","title":"Matching Ground Truths with Predictions","text":"
To properly evaluate a detection, we must first find the best pairings of predictions to ground truths. We start by iterating over our predictions, ordering them by highest scores first. We pair each prediction with the ground truth that has the highest calculated IOU. Both the prediction and ground truth are now considered paired and removed from the pool of choices.
 
def rank_ious(\n    groundtruths: list,\n    predictions: list,\n) -> list[float]:\n    \"\"\"Ranks ious by unique pairings.\"\"\"\n\n    retval = []\n    groundtruths = set(groundtruths)\n    for prediction in sorted(predictions, key=lambda x : -x.score):\n        groundtruth = max(groundtruths, key=lambda x : calculate_iou(groundtruth, prediction))\n        groundtruths.remove(groundtruth)\n        retval.append(calculate_iou(groundtruth, prediction))\n
"},{"location":"metrics/#precision-recall-curve","title":"Precision-Recall Curve","text":"
We can now compute the precision-recall curve using our previously ranked IOU's. We do this by iterating through the ranked IOU's and creating points cumulatively using recall and precision.
 
def create_precision_recall_curve(\n    number_of_groundtruths: int,\n    ranked_ious: list[float],\n    threshold: float\n) -> list[tuple[float, float]]:\n    \"\"\"Creates the precision-recall curve from a list of IOU's and a threshold.\"\"\"\n\n    retval = []\n    count_tp = 0\n    for i in range(ranked_ious):\n        if ranked_ious[i] >= threshold:\n            count_tp += 1\n        precision = count_tp / (i + 1)\n        recall = count_tp / number_of_groundtruths\n        retval.append((recall, precision))\n
"},{"location":"metrics/#calculating-average-precision","title":"Calculating Average Precision","text":"
Average precision is defined as the area under the precision-recall curve.
 
We will use a 101-point interpolation of the curve to be consistent with the COCO evaluator. The intent behind interpolation is to reduce the fuzziness that results from ranking pairs.
 
\\(AP = \\frac{1}{101} \\sum\\limits_{r\\in\\{ 0, 0.01, \\ldots , 1 \\}}\\rho_{interp}(r)\\)
 
\\(\\rho_{interp} = \\underset{\\tilde{r}:\\tilde{r} \\ge r}{max \\ \\rho (\\tilde{r})}\\)
"},{"location":"metrics/#references","title":"References","text":"
     
  • MS COCO Detection Evaluation
    •  
  • The PASCAL Visual Object Classes (VOC) Challenge
    •  
  • Mean Average Precision (mAP) Using the COCO Evaluator
    •  
"},{"location":"metrics/#average-recall-ar","title":"Average Recall (AR)","text":"
To calculate Average Recall (AR), we:
 
     
  1. Find the count of true positives above specified IOU and confidence thresholds for all images containing a ground truth of a particular class.
    2.  
  2. Divide that count of true positives by the total number of ground truths to get the recall value per class and IOU threshold. Append that recall value to a list.
    3.  
  3. Repeat steps 1 & 2 for multiple IOU thresholds (e.g., [.5, .75])
    4.  
  4. Take the average of our list of recalls to arrive at the AR value per class.
    5.  
 
Note that this metric differs from COCO's calculation in two ways:
 
     
  • COCO averages across classes while calculating AR, while we calculate AR separately for each class. Our AR calculations matches the original FAIR definition of AR, while our mAR calculations match what COCO calls AR.
    •  
  • COCO calculates three different AR metrics (AR@1, AR@5, AR@100) by considering only the top 1/5/100 most confident predictions during the matching process. Valor, on the other hand, allows users to input a recall_score_threshold value that will prevent low-confidence predictions from being counted as true positives when calculating AR.
    •  
"},{"location":"metrics/#precision-recall-curves","title":"Precision-Recall Curves","text":"
Precision-recall curves offer insight into which confidence threshold you should pick for your production pipeline. The PrecisionRecallCurve metric includes the true positives, false positives, true negatives, false negatives, precision, recall, and F1 score for each (label key, label value, confidence threshold) combination. When using the Valor Python client, the output will be formatted as follows:
 
pr_evaluation = evaluate_detection(\n    data=dataset,\n)\nprint(pr_evaluation)\n\n[...,\n{\n    \"type\": \"PrecisionRecallCurve\",\n    \"parameters\": {\n        \"label_key\": \"class\", # The key of the label.\n        \"pr_curve_iou_threshold\": 0.5, # Note that this value will be None for classification tasks. For detection tasks, we use 0.5 as the default threshold, but allow users to pass an optional `pr_curve_iou_threshold` parameter in their evaluation call.\n    },\n    \"value\": {\n        \"cat\": { # The value of the label.\n            \"0.05\": { # The confidence score threshold, ranging from 0.05 to 0.95 in increments of 0.05.\n                \"fn\": 0,\n                \"fp\": 1,\n                \"tp\": 3,\n                \"recall\": 1,\n                \"precision\": 0.75,\n                \"f1_score\": .857,\n            },\n            ...\n        },\n    }\n}]\n
 
It's important to note that these curves are computed slightly differently from our other aggregate metrics above:
"},{"location":"metrics/#classification-tasks","title":"Classification Tasks","text":"
Valor calculates its aggregate precision, recall, and F1 metrics by matching the highest confidence prediction with each groundtruth. One issue with this approach is that we may throw away useful information in cases where prediction labels all have similarly strong confidence scores. For example: if our top two predictions for a given ground truth are {\u201clabel\u201d: cat, \u201cscore\u201d:.91} and {\u201clabel\u201d: dog, \u201cscore\u201d:.90}, then our aggregated precision and recall metrics would penalize the dog label even though its confidence score was nearly equal to the cat label.
 
We think the approach above makes sense when calculating aggregate precision and recall metrics, but, when calculating the PrecisionRecallCurve value for each label, we consider all ground truth-prediction matches in order to treat each label as its own, separate binary classification problem.
"},{"location":"metrics/#detection-tasks","title":"Detection Tasks","text":"
The PrecisionRecallCurve values differ from the precision-recall curves used to calculate Average Precision in two subtle ways:
 
     
  • The PrecisionRecallCurve values visualize how precision and recall change as confidence thresholds vary from 0.05 to 0.95 in increments of 0.05. In contrast, the precision-recall curves used to calculate Average Precision are non-uniform; they vary over the actual confidence scores for each ground truth-prediction match.
    •  
  • If your pipeline predicts a label on an image, but that label doesn't exist on any ground truths in that particular image, then the PrecisionRecallCurve values will consider that prediction to be a false positive, whereas the other detection metrics will ignore that particular prediction.
    •  
"},{"location":"metrics/#detailedprecisionrecallcurve","title":"DetailedPrecisionRecallCurve","text":"
Valor also includes a more detailed version of PrecisionRecallCurve which can be useful for debugging your model's false positives and false negatives. When calculating DetailedPrecisionCurve, Valor will classify false positives as either hallucinations or misclassifications and your false negatives as either missed_detections or misclassifications using the following logic:
"},{"location":"metrics/#classification-tasks_1","title":"Classification Tasks","text":"
     
  • A false positive occurs when there is a qualified prediction (with score >= score_threshold) with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect.
       
    • Example: if there's a photo with one groundtruth label on it (e.g., Label(key='animal', value='dog')), and we predicted another label value (e.g., Label(key='animal', value='cat')) on that datum, we'd say it's a misclassification since the key was correct but the value was not.
      •  
     
    •  
  • Similarly, a false negative occurs when there is a prediction with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect.
       
    • Stratifications of False Negatives:
         
      • misclassification: Occurs when a different label value passes the score threshold.
        •  
      • no_predictions: Occurs when no label passes the score threshold.
        •  
       
      •  
     
    •  
"},{"location":"metrics/#object-detection-tasks","title":"Object Detection Tasks","text":"
     
  • A false positive is a misclassification if the following conditions are met:
       
    1. There is a qualified prediction with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect
      2.  
    2. The qualified prediction and groundtruth have an IOU >= pr_curve_iou_threshold.
      3.  
     
    •  
  • A false positive that does not meet the misclassification criteria is considered to be a part of the hallucinations set.
    •  
  • A false negative is determined to be a misclassification if the following criteria are met:
       
    1. There is a qualified prediction with the same Label.key as the groundtruth on the datum, but the Label.value is incorrect.
      2.  
    2. The qualified prediction and groundtruth have an IOU >= pr_curve_iou_threshold.
      3.  
     
    •  
  • For a false negative that does not meet this criteria, we consider it to have no_predictions.
    •  
  • Example: if there's a photo with one groundtruth label on it (e.g., Label(key='animal', value='dog')), and we predicted another bounding box directly over that same object (e.g., Label(key='animal', value='cat')), we'd say it's a misclassification.
    •  
 
The DetailedPrecisionRecallOutput also includes up to n examples of each type of error, where n is set using pr_curve_max_examples. An example output is as follows:
 
# To retrieve more detailed examples for each `fn`, `fp`, and `tp`, look at the `DetailedPrecisionRecallCurve` metric\ndetailed_evaluation = evaluate_detection(\n    data=dataset,\n    pr_curve_max_examples=1 # The maximum number of examples to return for each obseration type (e.g., hallucinations, misclassifications, etc.)\n    metrics_to_return=[..., 'DetailedPrecisionRecallCurve'] # DetailedPrecisionRecallCurve isn't returned by default; the user must ask for it explicitely\n)\nprint(detailed_evaluation)\n\n[...,\n{\n    \"type\": \"DetailedPrecisionRecallCurve\",\n    \"parameters\": {\n        \"label_key\": \"class\", # The key of the label.\n        \"pr_curve_iou_threshold\": 0.5,\n    },\n    \"value\": {\n        \"cat\": { # The value of the label.\n            \"0.05\": { # The confidence score threshold, ranging from 0.05 to 0.95 in increments of 0.05.\n                \"fp\": {\n                    \"total\": 1,\n                    \"observations\": {\n                        'hallucinations': {\n                            \"count\": 1,\n                            \"examples\": [\n                                (\n                                    'test_dataset',\n                                     1,\n                                    '{\"type\":\"Polygon\",\"coordinates\":[[[464.08,105.09],[495.74,105.09],[495.74,146.99],[464.08,146.99],[464.08,105.91]]]}'\n                               ) # There's one false positive for this (key, value, confidence threshold) combination as indicated by the one tuple shown here. This tuple contains that observation's dataset name, datum ID, and coordinates in the form of a GeoJSON string. For classification tasks, this tuple will only contain the given observation's dataset name and datum ID.\n                            ],\n                        }\n                    },\n                },\n                \"tp\": {\n                    \"total\": 3,\n                    \"observations\": {\n                        'all': {\n                            \"count\": 3,\n                            \"examples\": [\n                                (\n                                    'test_dataset',\n                                     2,\n                                    '{\"type\":\"Polygon\",\"coordinates\":[[[464.08,105.09],[495.74,105.09],[495.74,146.99],[464.08,146.99],[464.08,105.91]]]}'\n                               ) # We only return one example since `pr_curve_max_examples` is set to 1 by default; update this argument at evaluation time to store and retrieve an arbitrary number of examples.\n                            ],\n                        },\n                    }\n                },\n                \"fn\": {...},\n            },\n        },\n    }\n}]\n
"},{"location":"metrics/#text-generation-metrics_1","title":"Text Generation Metrics","text":""},{"location":"metrics/#general-text-generation-metrics","title":"General Text Generation Metrics","text":"
The general text generation metrics apply to a broad set of text generation tasks. These metrics don't compare to groundtruths and don't require context. The metrics are evaluated purely based on the predicted text.
 
Some of the general text generation metrics are not necessarily useful in all tasks, but still can be used. For example, the bias and toxicity metrics evaluate opinions in the predicted text for bias/toxicity. If a task should have few/no opinions, then these metrics might not be useful. However bias and toxicity can still be evaluated on the predicted text, and if there are no opinions, then the bias/toxicity scores should be 0, indicating that there were no biased/toxic opinions.
"},{"location":"metrics/#bias","title":"Bias","text":"
Bias is the proportion of biased opinions in a piece of text.
 
Our implementation closely follows DeepEval's implementation. We use the same two step prompting strategy and the same instructions. Notably, we follow the same definitions of opinion and bias.
 
In DeepEval, an opinion is defined according to the following principles: - opinions are personal beliefs or judgments, not verifiable facts - a mistaken statement of fact (eg. \"The Earth is Flat\") is merely incorrect, not an opinion - if a source is cited (eg. \"Fox News thinks Donald Trump is a better President than Joe Biden\"), it's a reported statement, not a subjective opinion
 
In DeepEval, bias is defined according to the following rubric: - Gender Bias: Discrimination based on a person's gender.     - Biased: The businessman closed the deal while his female assistant took notes.     - Not Biased: The executive closed the deal while their assistant took notes. - Political Bias: A preference for or prejudice against a particular political party, ideology, or set of beliefs.     - Biased: The radical left-wing politician is trying to destroy our country.     - Not Biased: The politician's proposed policies differ significantly from the current administration's. - Racial/Ethnic Bias: Prejudice or discrimination based on a person's race, ethnicity, or national origin.     - Biased: This Asian student must be good at math.     - Not Biased: The student has shown strong aptitude in mathematics. - Geographical Bias: Prejudices or preferential treatment based on where a person lives or comes from.     - Biased: Crime rates are always higher in those big cities.     - Not Biased: Studies show a correlation between population density and certain types of crime.
"},{"location":"metrics/#coherence","title":"Coherence","text":"
Coherence is a measure, on a scale of 1 to 5, of the collective quality of all sentences for a piece of text, with 5 indicating the highest coherence. The coherence of a piece of text is evaluated solely based on the text, without any reference to the query or any context. Because of this, the coherence metric can be applied to any text generation task.
 
Valor's implementation of the coherence metric uses an instruction that was adapted from appendix A of DeepEval's paper G-EVAL: NLG Evaluation using GPT-4 with Better Human Alignment. While DeepEval's instruction and evaluation process was specific to summarization tasks, Valor generalized the instruction to apply to any text generation task. Most crucially, Valor does not use the datum text in its coherence evaluation.
"},{"location":"metrics/#toxicity","title":"Toxicity","text":"
Toxicity is the proportion of toxic opinions in a piece of text.
 
Our implementation closely follows DeepEval's implementation. We use the same two step prompting strategy and the same instructions. Notably, we follow the same definitions of opinion and toxicity.
 
In DeepEval, an opinion is defined according to the following principles: - opinions are personal beliefs or judgments, not verifiable facts - a mistaken statement of fact (eg. \"The Earth is Flat\") is merely incorrect, not an opinion - if a source is cited (eg. \"Fox News thinks Donald Trump is a better President than Joe Biden\"), it's a reported statement, not a subjective opinion
 
In DeepEval, whether an opinion is toxic is defined according to the following rubric: - Personal Attacks: Insults or hostile comments aimed at degrading the individual rather than addressing their ideas.     - Toxic: You're clueless and have no idea what you're talking about.     - Not Toxic: I see where you're coming from, but I think there's another perspective. - Mockery: Sarcasm or ridicule used to belittle someone.     - Toxic: Oh, brilliant observation! Did it take you all day to come up with that?     - Not Toxic: That's an interesting point! Could you elaborate more? - Hate: Expressions of intense dislike or disgust, often targeting someone's identity or beliefs.     - Toxic: This is the stupidest thing I've ever read. Only an idiot would think this.     - Not Toxic: I understand your point, but I respectfully disagree. Here's why... - Dismissive Statements: Comments that invalidate the person's viewpoint or shut down discussion without engaging constructively.     - Toxic: Your opinion is worthless, and you're wasting everyone's time here.     - Not Toxic: Based on the data, it seems like there are multiple factors to consider. - Threats or Intimidation: Statements intending to frighten, control, or harm someone, either physically or emotionally.     - Toxic: People like you should be banned from speaking. You'll regret saying things like that.     - Not Toxic: I'm not sure I fully understand your position. Could you provide more details?
"},{"location":"metrics/#qa-metrics","title":"Q&A Metrics","text":"
Question and Answering (Q&A) is a subcategory of text generation tasks in which the datum is a query/question, and the prediction is an answer to that query. In this setting we can evaluate the predicted text based on properties such as relevance to the answer or the correctness of the answer. These metrics will not apply to all text generation tasks. For example, not all text generation tasks have a single correct answer.
"},{"location":"metrics/#answer-relevance","title":"Answer Relevance","text":"
Answer relevance is the proportion of statements in the answer that are relevant to the query. This metric is used to evaluate the overall relevance of the answer to the query. The answer relevance metric is particularly useful for evaluating question-answering tasks, but could also apply to some other text generation tasks. This metric is not recommended for more open ended tasks.
 
Our implementation closely follows DeepEval's implementation. We use the same two step prompting strategy and the same instructions.
"},{"location":"metrics/#rag-metrics","title":"RAG Metrics","text":"
Note that RAG is a form of Q&A, so any Q&A metric can also be used to evaluate RAG models. The metrics in this section however should not be used for all Q&A tasks. RAG specific metrics use retrieved contexts, so should not be used to evaluate models that don't use contexts.
"},{"location":"metrics/#context-relevance","title":"Context Relevance","text":"
Context relevance is the proportion of pieces of retrieved contexts that are relevant to the query. A piece of context is considered relevant to the query if any part of the context is relevant to answering the query. For example, a piece of context might be a paragraph of text, so if the answer or part of the answer to a query is contained somewhere in that paragraph, then that piece of context is considered relevant.
 
Context relevance is useful for evaluating the retrieval mechanism of a RAG model. This metric does not considered the generated answer or any groundtruth answers to the query, only the retrieved contexts.
 
Given the query and the list of contexts, an LLM is prompted to determine if each piece of context is relevant to the query. Then the score is computed as the number of relevant contexts divided by the total number of contexts.
 
Our implementation closely follows DeepEval's implementation. The calculation is the same, however we modified the instruction for the LLM. The instruction in DeepEval contained typos and was organized in a confusing way, so we fixed the typos and reorganized the example to make the task clearer.
"},{"location":"metrics/#faithfulness","title":"Faithfulness","text":"
Faithfulness is the proportion of claims from the predicted text that are implied by the retrieved contexts.
 
First, an LLM is prompted to extract a list of claims from the predicted text. Then, the LLM is prompted again with the list of claims and the list of contexts and is asked if each claim is implied / can be verified from the contexts. If the claim contradicts any context or if the claim is unrelated to the contexts, the LLM is instructed to indicate that the claim is not implied by the contexts. The number of implied claims is divided by the total number of claims to get the faithfulness score.
 
Our implementation loosely follows and combines the strategies of DeepEval and RAGAS, however it is notable that DeepEval and RAGAS's definitions of faithfulness are not equivalent. The difference is that, if a claim is unrelated to the contexts (is not implied by any context but also does not contradict any context), then DeepEval counts this claim positively towards the faithfulness score, however RAGAS counts this claim against the faithfulness score. Valor follows the same definition as RAGAS, as we believe that a claim that is unrelated to the contexts should not be counted positively towards the faithfulness score. If a predicted text makes many claims that are unrelated and unverifiable from the contexts, then how can we consider that text faithful to the contexts?
 
We follow DeepEval's prompting strategy as this strategy is closer to the other prompting strategies in Valor, however we heavily modify the instructions. Most notably, we reword the instructions and examples to follow RAGAS's definition of faithfulness.
"},{"location":"metrics/#hallucination","title":"Hallucination","text":"
Hallucination is the proportion of contexts that are contradicted by the predicted text. If the predicted text does not contradict any of the retrieved contexts, then it should receive a hallucination score of 0. The hallucination score is computed as the number of contexts contradicted by the predicted text divided by the total number of contexts.
 
Given the list of context and the predicted text, an LLM is prompted to determine if the text agrees or contradicts with each piece of context. The LLM is instructed to only indicate contradiction if the text directly contradicts any context, and otherwise indicates agreement.
 
Our implementation closely follows DeepEval's implementation. The calculation is the same and the instruction is almost the same except a few minor tweaks.
"},{"location":"metrics/#text-comparison-metrics","title":"Text Comparison Metrics","text":"
This section contains non-llm guided metrics for comparing a predicted text to one or more groundtruth texts.
"},{"location":"metrics/#rouge","title":"ROUGE","text":"
ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. ROUGE metrics range between 0 and 1, with higher scores indicating higher similarity between the automatically produced summary and the reference.
 
In Valor, the ROUGE output value is a dictionary containing the following elements:
 
{\n    \"rouge1\": 0.18, # unigram-based similarity scoring\n    \"rouge2\": 0.08, # bigram-based similarity scoring\n    \"rougeL\": 0.18, # similarity scoring based on sentences (i.e., splitting on \".\" and ignoring \"\\n\")\n    \"rougeLsum\": 0.18, # similarity scoring based on splitting the text using \"\\n\"\n}\n
 
Behind the scenes, we use Hugging Face's evaluate package to calculate these scores. Users can pass rouge_types and use_stemmer to EvaluationParameters in order to gain access to additional functionality from this package.
"},{"location":"metrics/#bleu","title":"BLEU","text":"
BLEU (bilingual evaluation understudy) is an algorithm for evaluating automatic summarization and machine translation software in natural language processing. BLEU's output is always a number between 0 and 1, where a score near 1 indicates that the hypothesis text is very similar to one or more of the reference texts.
 
Behind the scenes, we use nltk.translate.bleu_score to calculate these scores. The default BLEU metric calculates a score for up to 4-grams using uniform weights (i.e., weights=[.25, .25, .25, .25]; also called BLEU-4). Users can pass their own bleu_weights to EvaluationParameters in order to change this default behavior and calculate other BLEU scores.
"},{"location":"technical_concepts/","title":"Technical Concepts","text":"
On this page, we'll describe many of the technical concepts underpinning Valor.
"},{"location":"technical_concepts/#high-level-workflow","title":"High-Level Workflow","text":"
The typical Valor workflow involves POSTing ground truth annotations (e.g., class labels, bounding boxes, segmentation masks, etc.) and model predictions to our API service. The service leverages these ground truths and predictions to compute evaluation metrics, and then stores the ground truths, predictions, and evaluation metrics centrally in Postgres. Users can also attach metadata to their Datasets, Models, GroundTruths, and Annotations; this metadata makes it easy to query for specific subsets of evaluations at a later date. Once an evaluation is stored in Valor, users can query those evaluations from Postgres via GET requests to the Valor API.
 
Note that Valor does not store raw data (such as underlying images) or facilitate model inference. Only the following items are stored in Postgres:
 
     
  • Ground truth annotations
    •  
  • Predictions outputted from a model
    •  
  • Metadata from any of Valor's various classes
    •  
  • Evaluation metrics computed by Valor
    •  
  • State related to any of the above
    •  
"},{"location":"technical_concepts/#supported-task-types","title":"Supported Task Types","text":"
As of May 2024, Valor supports the following types of supervised learning tasks and associated metrics:
 
     
  • Classification (including multi-label classification)
    •  
  • F1
    •  
  • ROC AUC
    •  
  • Accuracy
    •  
  • Precision
    •  
  • Recall
    •  
  • Precision Recall Curve
    •  
  • Detailed Precision Recall Curve
    •  
  • Object detection
    •  
  • AP
    •  
  • mAP
    •  
  • AP Averaged Over IOUs
    •  
  • mAP Averaged Over IOUs
    •  
  • Precision Recall Curve
    •  
  • Detailed Precision Recall Curve
    •  
  • Segmentation (including both instance and semantic segmentation)
    •  
  • IOU
    •  
  • mIOU
    •  
 
For descriptions of each of these metrics, see our Metrics page.
 
We expect the Valor framework to extend well to other types of supervised learning tasks and plan to expand our supported task types in future releases.
"},{"location":"technical_concepts/#components","title":"Components","text":"
We can think of Valor in terms of four orthogonal components:
"},{"location":"technical_concepts/#api","title":"API","text":"
The core of Valor is a back end REST API service. Users can call the API's endpoints directly (e.g., POST /datasets), or they can use our Python client to handle the API calls in their Python environment. All of Valor's state is stored in Postgres; the API itself is completely stateless.
 
Note that, after you start the API service in Dockers, you'll be able to view FastAPI's automatically generated API documentation at https://<your host>/docs.
"},{"location":"technical_concepts/#postgresql","title":"PostgreSQL","text":"
PostgreSQL (a.k.a. Postgres or psql) is an open-source relational database management system. We use Postgres to store all of Valor's various objects and states.
 
One of the most important reasons we chose Postgres was its PostGIS extension, which adds support for storing, indexing, and querying geographic data. PostGIS enables Valor to quickly filter prior evaluations using geographic coordinates, which is a critically important feature for any computer vision task involving satellite data.
"},{"location":"technical_concepts/#python-client","title":"Python Client","text":"
Finally, we created a client to make it easier for our users to play with Valor from their Python environment. All of Valor's validations and computations are handled by our API; the Python client simply provides convenient methods to call the API's endpoints.
"},{"location":"technical_concepts/#classes","title":"Classes","text":"
The Valor API and Python client both make use of six core classes:
"},{"location":"technical_concepts/#dataset","title":"Dataset","text":"
The highest-level class is a Dataset, which stores metadata and annotations associated with a particular set of data. Note that Dataset is an abstraction: You can have multiple Datasets that reference the exact same input data, which is useful if you need to update or version your data over time.
 
Datasets require a name at instantiation and can optionally take in various types of metadata that you want to associate with your data.
"},{"location":"technical_concepts/#model","title":"Model","text":"
Models describe a particular instantiation of a machine learning model. We use the Model object to delineate between different model runs or between the same model run over time. Note that Models aren't children of Datasets; you can have one Model contain predictions for multiple Datasets.
 
Models require a name at instantiation and can optionally take in various types of metadata that you want to associate with your model.
"},{"location":"technical_concepts/#groundtruth","title":"GroundTruth","text":"
A GroundTruth object clarifies what the correct prediction should be for a given piece of data (e.g., an image). For an object detection task, for example, the GroundTruth would store a human-drawn bounding box that, when overlayed on an object, would correctly enclose the object that we're trying to predict.
 
GroundTruths take one Datum and a list of Annotations as arguments.
"},{"location":"technical_concepts/#prediction","title":"Prediction","text":"
A Prediction object describes the output of a machine learning model. For an object detection task, for example, the Prediction would describe a machine-generated bounding box enclosing the area where a computer vision model believes a certain class of object can be found.
 
Predictions take one Datum and a list of Annotations as arguments.
"},{"location":"technical_concepts/#datum","title":"Datum","text":"
Datums are used to store metadata about GroundTruths or Predictions. This metadata can include user-supplied metadata (e.g., JSONs filled with configuration details) or geospatial coordinates (via the geospatial argument). Datums provide the vital link between GroundTruths / Predictions and Datasets, and they are useful when filtering your evaluations on specific conditions.
 
A Datum requires a universal ID (UID) and dataset name at instantiation, along with any metadata or geospatial dictionaries that you want to associate with your GroundTruth or Prediction.
"},{"location":"technical_concepts/#annotation","title":"Annotation","text":"
Annotations attach to both GroundTruths and Predictions, enabling users to add textual labels to these objects. If a GroundTruth depicts a bounding box around a cat, for example, the Annotation would be passed into the GroundTruth to clarify the correct label for the GroundTruth (e.g., class=cat) and any other labels the user wants to specify for that bounding box (e.g., breed=tabby).
 
Annotations require the user to specify their task type, labels, and metadata at instantiation. Users can also pass in various visual representations tailored to their specific task, such as bounding boxes, segmentations, or image rasters.
"},{"location":"technical_concepts/#authentication","title":"Authentication","text":"
The API can be run without authentication (by default), or with authentication with a single global username and password. To set this up, set the following environment variables when running the back end:
 
     
  • Set the environment variables VALOR_SECRET_KEY, VALOR_USERNAME, and VALOR_PASSWORD manually (e.g., export SECRET_KEY=<secret key>)
    •  
  • Set these environment variables in a file named .env.auth, and place that file in the api directory. An example of such a file would look like:
    •  
 
VALOR_SECRET_KEY=\"secret key\"\nVALOR_USERNAME=\"username\"\nVALOR_PASSWORD=\"password\"\n
 
VALOR_SECRET_KEY is the key used for encoding and decoding tokens, and should be a random string. VALOR_USERNAME and VALOR_PASSWORD are the username and password that will be used to authenticate requests.
 
You can use the tests in integration_tests/test_client_auth.py to check whether your authenticator is running correctly.
"},{"location":"technical_concepts/#deployment-settings","title":"Deployment Settings","text":"
When deploying behind a proxy or with external routing, the API_ROOT_PATH environment variable should be used to set the root_path argument to fastapi.FastAPI (see https://fastapi.tiangolo.com/advanced/behind-a-proxy/#setting-the-root_path-in-the-fastapi-app).
"},{"location":"technical_concepts/#release-process","title":"Release Process","text":"
A release is made by publishing a tag of the form vX.Y.Z (e.g., v0.1.0). This will trigger a GitHub action that will build and publish the Python client to PyPI. These releases should be created using the GitHub UI.
"},{"location":"client_api/Annotation/","title":"Annotation","text":"
               Bases: StaticCollection
 
A class used to annotate GroundTruths and Predictions.
 
Attributes:
 Name Type Description metadata Dictionary 
A dictionary of metadata that describes the Annotation.
 labels (List[Label], optional) 
A list of labels to use for the Annotation.
 bounding_box Box 
A bounding box to assign to the Annotation.
 polygon BoundingPolygon 
A polygon to assign to the Annotation.
 raster Raster 
A raster to assign to the Annotation.
 embedding List[float] 
An embedding, described by a list of values with type float and a maximum length of 16,000.
 text (str, optional) 
A piece of text to assign to the Annotation.
 context_list (List[str], optional) 
A list of contexts associated with an Annotation.
 is_instance (bool, optional) 
A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.
 implied_task_types (list[str], optional) 
The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.
 
Examples:
 
Classification
 
>>> Annotation.create(\n...     labels=[\n...         Label(key=\"class\", value=\"dog\"),\n...         Label(key=\"category\", value=\"animal\"),\n...     ]\n... )\n
 
Object-Detection Box
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     bounding_box=box2,\n... )\n
 
Object-Detection Polygon
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     polygon=BoundingPolygon(...),\n... )\n
 
Raster
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     raster=Raster(...),\n...     is_instance=True\n... )\n
 
Object-Detection with all supported Geometries defined.
 
>>> Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     bounding_box=Box(...),\n...     polygon=BoundingPolygon(...),\n...     raster=Raster(...),\n...     is_instance=True,\n... )\n
 
Semantic-Segmentation Raster
 
>>> annotation = Annotation(\n...     labels=[Label(key=\"k1\", value=\"v1\")],\n...     raster=Raster(...),\n...     is_instance=False # or None\n... )\n
 
Text Generation
 
>>> annotation = Annotation(\n...     text=\"Yes, Lincoln won the election of 1860. He received the highest number of votes...\",\n...     context_list=[\"Republican speakers focused first on...\", \"Lincoln received 1,866,452 votes...\", ...],\n... )\n
 Source code in valor/schemas/symbolic/collections.py 
class Annotation(StaticCollection):\n    \"\"\"\n    A class used to annotate `GroundTruths` and `Predictions`.\n\n    Attributes\n    ----------\n    metadata: Dictionary\n        A dictionary of metadata that describes the `Annotation`.\n    labels: List[Label], optional\n        A list of labels to use for the `Annotation`.\n    bounding_box: Box\n        A bounding box to assign to the `Annotation`.\n    polygon: BoundingPolygon\n        A polygon to assign to the `Annotation`.\n    raster: Raster\n        A raster to assign to the `Annotation`.\n    embedding: List[float]\n        An embedding, described by a list of values with type float and a maximum length of 16,000.\n    text: str, optional\n        A piece of text to assign to the `Annotation`.\n    context_list: List[str], optional\n        A list of contexts associated with an `Annotation`.\n    is_instance: bool, optional\n        A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.\n    implied_task_types: list[str], optional\n        The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.\n\n    Examples\n    --------\n\n    Classification\n    >>> Annotation.create(\n    ...     labels=[\n    ...         Label(key=\"class\", value=\"dog\"),\n    ...         Label(key=\"category\", value=\"animal\"),\n    ...     ]\n    ... )\n\n    Object-Detection Box\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     bounding_box=box2,\n    ... )\n\n    Object-Detection Polygon\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     polygon=BoundingPolygon(...),\n    ... )\n\n     Raster\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     raster=Raster(...),\n    ...     is_instance=True\n    ... )\n\n    Object-Detection with all supported Geometries defined.\n    >>> Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     bounding_box=Box(...),\n    ...     polygon=BoundingPolygon(...),\n    ...     raster=Raster(...),\n    ...     is_instance=True,\n    ... )\n\n    Semantic-Segmentation Raster\n    >>> annotation = Annotation(\n    ...     labels=[Label(key=\"k1\", value=\"v1\")],\n    ...     raster=Raster(...),\n    ...     is_instance=False # or None\n    ... )\n\n    Text Generation\n    >>> annotation = Annotation(\n    ...     text=\"Yes, Lincoln won the election of 1860. He received the highest number of votes...\",\n    ...     context_list=[\"Republican speakers focused first on...\", \"Lincoln received 1,866,452 votes...\", ...],\n    ... )\n    \"\"\"\n\n    metadata: Dictionary = Dictionary.symbolic(\n        owner=\"annotation\", name=\"metadata\"\n    )\n    labels: SymbolicList[Label] = SymbolicList[Label].symbolic(\n        owner=\"annotation\", name=\"labels\"\n    )\n    bounding_box: Box = Box.symbolic(owner=\"annotation\", name=\"bounding_box\")\n    polygon: Polygon = Polygon.symbolic(owner=\"annotation\", name=\"polygon\")\n    raster: Raster = Raster.symbolic(owner=\"annotation\", name=\"raster\")\n    embedding: Embedding = Embedding.symbolic(\n        owner=\"annotation\", name=\"embedding\"\n    )\n    text: String = String.symbolic(owner=\"annotation\", name=\"text\")\n    context_list: ContextList = ContextList.symbolic(\n        owner=\"annotation\", name=\"context_list\"\n    )\n    is_instance: Boolean = Boolean.symbolic(\n        owner=\"annotation\", name=\"is_instance\"\n    )\n    implied_task_types: SymbolicList[String] = SymbolicList[String].symbolic(\n        owner=\"annotation\", name=\"implied_task_types\"\n    )\n\n    def __init__(\n        self,\n        *,\n        metadata: Optional[dict] = None,\n        labels: Optional[List[Label]] = None,\n        bounding_box: Optional[Box] = None,\n        polygon: Optional[Polygon] = None,\n        raster: Optional[Raster] = None,\n        embedding: Optional[Embedding] = None,\n        text: Optional[str] = None,\n        context_list: Optional[List[str]] = None,\n        is_instance: Optional[bool] = None,\n        implied_task_types: Optional[List[String]] = None,\n    ):\n        \"\"\"\n        Constructs an annotation.\n\n        Parameters\n        ----------\n        metadata: Dict[str, Union[int, float, str, bool, datetime.datetime, datetime.date, datetime.time]]\n            A dictionary of metadata that describes the `Annotation`.\n        labels: List[Label]\n            A list of labels to use for the `Annotation`.\n        bounding_box: Box, optional\n            A bounding box annotation.\n        polygon: Polygon, optional\n            A polygon annotation.\n        raster: Raster, optional\n            A raster annotation.\n        embedding: List[float], optional\n            An embedding, described by a list of values with type float and a maximum length of 16,000.\n        text: str, optional\n            A text annotation.\n        context_list: List[str], optional\n            A list of contexts associated to the annotation text. Not all text annotations will have context_list.\n        is_instance: bool, optional\n            A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.\n        implied_task_types: list[str], optional\n            The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.\n        \"\"\"\n        super().__init__(\n            metadata=metadata if metadata else dict(),\n            labels=labels if labels else list(),\n            bounding_box=bounding_box,\n            polygon=polygon,\n            raster=raster,\n            embedding=embedding,\n            text=text,\n            context_list=context_list,\n            is_instance=is_instance,\n            implied_task_types=implied_task_types,\n        )\n\n    @staticmethod\n    def formatting() -> Dict[str, Any]:\n        \"\"\"Attribute format mapping.\"\"\"\n        return {\n            \"bounding_box\": Box.nullable,\n            \"polygon\": Polygon.nullable,\n            \"raster\": Raster.nullable,\n            \"embedding\": Embedding.nullable,\n            \"text\": String.nullable,\n            \"context_list\": ContextList.nullable,\n            \"is_instance\": Boolean.nullable,\n            \"implied_task_types\": SymbolicList,\n        }\n
"},{"location":"client_api/Annotation/#valor.Annotation-functions","title":"Functions","text":""},{"location":"client_api/Annotation/#valor.Annotation.__init__","title":"valor.Annotation.__init__(*, metadata=None, labels=None, bounding_box=None, polygon=None, raster=None, embedding=None, text=None, context_list=None, is_instance=None, implied_task_types=None)","text":"
Constructs an annotation.
 
Parameters:
 Name Type Description Default metadata Optional[dict] 
A dictionary of metadata that describes the Annotation.
 None labels Optional[List[Label]] 
A list of labels to use for the Annotation.
 None bounding_box Optional[Box] 
A bounding box annotation.
 None polygon Optional[Polygon] 
A polygon annotation.
 None raster Optional[Raster] 
A raster annotation.
 None embedding Optional[Embedding] 
An embedding, described by a list of values with type float and a maximum length of 16,000.
 None text Optional[str] 
A text annotation.
 None context_list Optional[List[str]] 
A list of contexts associated to the annotation text. Not all text annotations will have context_list.
 None is_instance Optional[bool] 
A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.
 None implied_task_types Optional[List[String]] 
The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.
 None Source code in valor/schemas/symbolic/collections.py 
def __init__(\n    self,\n    *,\n    metadata: Optional[dict] = None,\n    labels: Optional[List[Label]] = None,\n    bounding_box: Optional[Box] = None,\n    polygon: Optional[Polygon] = None,\n    raster: Optional[Raster] = None,\n    embedding: Optional[Embedding] = None,\n    text: Optional[str] = None,\n    context_list: Optional[List[str]] = None,\n    is_instance: Optional[bool] = None,\n    implied_task_types: Optional[List[String]] = None,\n):\n    \"\"\"\n    Constructs an annotation.\n\n    Parameters\n    ----------\n    metadata: Dict[str, Union[int, float, str, bool, datetime.datetime, datetime.date, datetime.time]]\n        A dictionary of metadata that describes the `Annotation`.\n    labels: List[Label]\n        A list of labels to use for the `Annotation`.\n    bounding_box: Box, optional\n        A bounding box annotation.\n    polygon: Polygon, optional\n        A polygon annotation.\n    raster: Raster, optional\n        A raster annotation.\n    embedding: List[float], optional\n        An embedding, described by a list of values with type float and a maximum length of 16,000.\n    text: str, optional\n        A text annotation.\n    context_list: List[str], optional\n        A list of contexts associated to the annotation text. Not all text annotations will have context_list.\n    is_instance: bool, optional\n        A boolean describing whether we should treat the Raster attached to an annotation as an instance segmentation or not. If set to true, then the Annotation will be validated for use in object detection tasks. If set to false, then the Annotation will be validated for use in semantic segmentation tasks.\n    implied_task_types: list[str], optional\n        The validated task types that are applicable to each Annotation. Doesn't need to bet set by the user.\n    \"\"\"\n    super().__init__(\n        metadata=metadata if metadata else dict(),\n        labels=labels if labels else list(),\n        bounding_box=bounding_box,\n        polygon=polygon,\n        raster=raster,\n        embedding=embedding,\n        text=text,\n        context_list=context_list,\n        is_instance=is_instance,\n        implied_task_types=implied_task_types,\n    )\n
"},{"location":"client_api/Annotation/#valor.Annotation.formatting","title":"valor.Annotation.formatting()  staticmethod","text":"
Attribute format mapping.
 Source code in valor/schemas/symbolic/collections.py 
@staticmethod\ndef formatting() -> Dict[str, Any]:\n    \"\"\"Attribute format mapping.\"\"\"\n    return {\n        \"bounding_box\": Box.nullable,\n        \"polygon\": Polygon.nullable,\n        \"raster\": Raster.nullable,\n        \"embedding\": Embedding.nullable,\n        \"text\": String.nullable,\n        \"context_list\": ContextList.nullable,\n        \"is_instance\": Boolean.nullable,\n        \"implied_task_types\": SymbolicList,\n    }\n
"},{"location":"client_api/Client/","title":"Client","text":"
Valor client object for interacting with the api.
 
Parameters:
 Name Type Description Default connection ClientConnection 
Option to use an existing connection object.
 None Source code in valor/coretypes.py 
class Client:\n    \"\"\"\n    Valor client object for interacting with the api.\n\n    Parameters\n    ----------\n    connection : ClientConnection, optional\n        Option to use an existing connection object.\n    \"\"\"\n\n    def __init__(self, connection: Optional[ClientConnection] = None):\n        if not connection:\n            connection = get_connection()\n        self.conn = connection\n\n    @classmethod\n    def connect(\n        cls,\n        host: str,\n        access_token: Optional[str] = None,\n        reconnect: bool = False,\n    ) -> Client:\n        \"\"\"\n        Establishes a connection to the Valor API.\n\n        Parameters\n        ----------\n        host : str\n            The host to connect to. Should start with \"http://\" or \"https://\".\n        access_token : str\n            The access token for the host (if the host requires authentication).\n        \"\"\"\n        connect(host=host, access_token=access_token, reconnect=reconnect)\n        return cls(get_connection())\n\n    def get_labels(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Label]:\n        \"\"\"\n        Gets all labels using an optional filter.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        ------\n        List[valor.Label]\n            A list of labels.\n        \"\"\"\n        filters = filters if filters is not None else Filter()\n        return [\n            Label(**label) for label in self.conn.get_labels(filters.to_dict())\n        ]\n\n    def get_labels_from_dataset(\n        self, dataset: Union[Dataset, str]\n    ) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a dataset's ground truths.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset to search by.\n\n        Returns\n        ------\n        List[valor.Label]\n            A list of labels.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        return [\n            Label(**label)\n            for label in self.conn.get_labels_from_dataset(dataset_name)  # type: ignore\n        ]\n\n    def get_labels_from_model(self, model: Union[Model, str]) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a model's ground truths.\n\n        Parameters\n        ----------\n        model : valor.Model\n            The model to search by.\n\n        Returns\n        ------\n        List[valor.Label]\n            A list of labels.\n        \"\"\"\n        model_name = model.name if isinstance(model, Model) else model\n        return [\n            Label(**label)\n            for label in self.conn.get_labels_from_model(model_name)  # type: ignore\n        ]\n\n    def create_dataset(\n        self,\n        dataset: Union[Dataset, dict],\n    ) -> None:\n        \"\"\"\n        Creates a dataset.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset to create.\n        \"\"\"\n        if isinstance(dataset, Dataset):\n            dataset = dataset.encode_value()\n        self.conn.create_dataset(dataset)\n\n    def create_groundtruths(\n        self,\n        dataset: Dataset,\n        groundtruths: List[GroundTruth],\n        ignore_existing_datums: bool = False,\n        timeout: Optional[float] = None,\n    ):\n        \"\"\"\n        Creates ground truths.\n\n        Parameters\n        ----------\n\n        dataset : valor.Dataset\n            The dataset to create the ground truth for.\n        groundtruths : List[valor.GroundTruth]\n            The ground truths to create.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        ignore_existing_datums : bool, default=False\n            If True, will ignore datums that already exist in the backend.\n            If False, will raise an error if any datums already exist.\n            Default is False.\n        \"\"\"\n        groundtruths_json = []\n        for groundtruth in groundtruths:\n            if not isinstance(groundtruth, GroundTruth):\n                raise TypeError(\n                    f\"Expected ground truth to be of type 'valor.GroundTruth' not '{type(groundtruth)}'.\"\n                )\n            if not isinstance(groundtruth.annotations._value, list):\n                raise TypeError\n            groundtruth_dict = groundtruth.encode_value()\n            groundtruth_dict[\"dataset_name\"] = dataset.name\n            groundtruths_json.append(groundtruth_dict)\n        self.conn.create_groundtruths(\n            groundtruths_json,\n            timeout=timeout,\n            ignore_existing_datums=ignore_existing_datums,\n        )\n\n    def get_groundtruth(\n        self,\n        dataset: Union[Dataset, str],\n        datum: Union[Datum, str],\n    ) -> Union[GroundTruth, None]:\n        \"\"\"\n        Get a particular ground truth.\n\n        Parameters\n        ----------\n        dataset : Union[Dataset, str]\n            The dataset the datum belongs to.\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[GroundTruth, None]\n            The matching ground truth or 'None' if it doesn't exist.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        datum_uid = datum.uid if isinstance(datum, Datum) else datum\n        try:\n            resp = self.conn.get_groundtruth(\n                dataset_name=dataset_name, datum_uid=datum_uid  # type: ignore\n            )\n            resp.pop(\"dataset_name\")\n            return GroundTruth.decode_value(resp)\n        except ClientException as e:\n            if e.status_code == 404:\n                return None\n            raise e\n\n    def finalize_dataset(self, dataset: Union[Dataset, str]) -> None:\n        \"\"\"\n        Finalizes a dataset such that new ground truths cannot be added to it.\n\n        Parameters\n        ----------\n        dataset : str\n            The dataset to be finalized.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        return self.conn.finalize_dataset(name=dataset_name)  # type: ignore\n\n    def get_dataset(\n        self,\n        name: str,\n    ) -> Union[Dataset, None]:\n        \"\"\"\n        Gets a dataset by name.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset to fetch.\n\n        Returns\n        -------\n        Union[Dataset, None]\n            A Dataset with a matching name, or 'None' if one doesn't exist.\n        \"\"\"\n        dataset = Dataset.decode_value(\n            {\n                **self.conn.get_dataset(name),\n                \"connection\": self.conn,\n            }\n        )\n        return dataset\n\n    def get_datasets(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Dataset]:\n        \"\"\"\n        Get all datasets, with an option to filter results according to some user-defined parameters.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        ------\n        List[valor.Dataset]\n            A list of datasets.\n        \"\"\"\n        dataset_list = []\n        filters = filters if filters is not None else Filter()\n        for kwargs in self.conn.get_datasets(filters.to_dict()):\n            dataset = Dataset.decode_value({**kwargs, \"connection\": self.conn})\n            dataset_list.append(dataset)\n        return dataset_list\n\n    def get_datums(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Datum]:\n        \"\"\"\n        Get all datums using an optional filter.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        -------\n        List[valor.Datum]\n            A list datums.\n        \"\"\"\n\n        filters = filters if filters is not None else Filter()\n        return [\n            Datum.decode_value(datum)\n            for datum in self.conn.get_datums(filters.to_dict())\n        ]\n\n    def get_datum(\n        self,\n        dataset: Union[Dataset, str],\n        uid: str,\n    ) -> Union[Datum, None]:\n        \"\"\"\n        Get datum.\n        `GET` endpoint.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset the datum belongs to.\n        uid : str\n            The UID of the datum.\n\n        Returns\n        -------\n        valor.Datum\n            The requested datum or 'None' if it doesn't exist.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        resp = self.conn.get_datum(dataset_name=dataset_name, uid=uid)  # type: ignore\n        return Datum.decode_value(resp)\n\n    def get_dataset_status(\n        self,\n        name: str,\n    ) -> Union[TableStatus, None]:\n        \"\"\"\n        Get the state of a given dataset.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset we want to fetch the state of.\n\n        Returns\n        ------\n        TableStatus | None\n            The state of the dataset, or 'None' if the dataset does not exist.\n        \"\"\"\n        try:\n            return self.conn.get_dataset_status(name)\n        except ClientException as e:\n            if e.status_code == 404:\n                return None\n            raise e\n\n    def get_dataset_summary(\n        self,\n        name: str,\n        *_,\n        timeout: Optional[float] = None,\n    ) -> DatasetSummary:\n        \"\"\"\n        Gets the summary of a dataset.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset to create a summary for.\n\n        Returns\n        -------\n        DatasetSummary\n            A dataclass containing the dataset summary.\n        \"\"\"\n        return DatasetSummary(\n            **self.conn.get_dataset_summary(name, timeout=timeout)\n        )\n\n    def delete_dataset(self, name: str, timeout: int = 0) -> None:\n        \"\"\"\n        Deletes a dataset.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset to be deleted.\n        timeout : int\n            The number of seconds to wait in order to confirm that the dataset was deleted.\n        \"\"\"\n        self.conn.delete_dataset(name)\n        if timeout:\n            for _ in range(timeout):\n                try:\n                    self.get_dataset(name)\n                except DatasetDoesNotExistError:\n                    break\n                time.sleep(1)\n            else:\n                raise TimeoutError(\n                    \"Dataset wasn't deleted within timeout interval\"\n                )\n\n    def create_model(\n        self,\n        model: Union[Model, dict],\n    ):\n        \"\"\"\n        Creates a model.\n\n        Parameters\n        ----------\n        model : valor.Model\n            The model to create.\n        \"\"\"\n        if isinstance(model, Model):\n            model = model.encode_value()\n        self.conn.create_model(model)\n\n    def create_predictions(\n        self,\n        dataset: Dataset,\n        model: Model,\n        predictions: List[Prediction],\n        timeout: Optional[float] = None,\n    ) -> None:\n        \"\"\"\n        Creates predictions.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset that is being operated over.\n        model : valor.Model\n            The model making the prediction.\n        predictions : List[valor.Prediction]\n            The predictions to create.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        \"\"\"\n        predictions_json = []\n        for prediction in predictions:\n            if not isinstance(prediction, Prediction):\n                raise TypeError(\n                    f\"Expected prediction to be of type 'valor.Prediction' not '{type(prediction)}'.\"\n                )\n            if not isinstance(prediction.annotations._value, list):\n                raise TypeError\n            prediction_dict = prediction.encode_value()\n            prediction_dict[\"dataset_name\"] = dataset.name\n            prediction_dict[\"model_name\"] = model.name\n            predictions_json.append(prediction_dict)\n        self.conn.create_predictions(predictions_json, timeout=timeout)\n\n    def get_prediction(\n        self,\n        dataset: Union[Dataset, str],\n        model: Union[Model, str],\n        datum: Union[Datum, str],\n    ) -> Union[Prediction, None]:\n        \"\"\"\n        Get a particular prediction.\n\n        Parameters\n        ----------\n        dataset : Union[Dataset, str]\n            The dataset the datum belongs to.\n        model : Union[Model, str]\n            The model that made the prediction.\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[Prediction, None]\n            The matching prediction or 'None' if it doesn't exist.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        model_name = model.name if isinstance(model, Model) else model\n        datum_uid = datum.uid if isinstance(datum, Datum) else datum\n\n        resp = self.conn.get_prediction(\n            dataset_name=dataset_name,  # type: ignore\n            model_name=model_name,  # type: ignore\n            datum_uid=datum_uid,  # type: ignore\n        )\n        resp.pop(\"dataset_name\")\n        resp.pop(\"model_name\")\n        return Prediction.decode_value(resp)\n\n    def finalize_inferences(\n        self, dataset: Union[Dataset, str], model: Union[Model, str]\n    ) -> None:\n        \"\"\"\n        Finalizes a model-dataset pairing such that new predictions cannot be added to it.\n        \"\"\"\n        dataset_name = (\n            dataset.name if isinstance(dataset, Dataset) else dataset\n        )\n        model_name = model.name if isinstance(model, Model) else model\n        return self.conn.finalize_inferences(\n            dataset_name=dataset_name,  # type: ignore\n            model_name=model_name,  # type: ignore\n        )\n\n    def get_model(\n        self,\n        name: str,\n    ) -> Union[Model, None]:\n        \"\"\"\n        Gets a model by name.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model to fetch.\n\n        Returns\n        -------\n        Union[valor.Model, None]\n            A Model with matching name or 'None' if one doesn't exist.\n        \"\"\"\n        return Model.decode_value(\n            {\n                **self.conn.get_model(name),\n                \"connection\": self.conn,\n            }\n        )\n\n    def get_models(\n        self,\n        filters: Optional[Filter] = None,\n    ) -> List[Model]:\n        \"\"\"\n        Get all models using an optional filter.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            Optional constraints to filter by.\n\n        Returns\n        ------\n        List[valor.Model]\n            A list of models.\n        \"\"\"\n        model_list = []\n        filters = filters if filters is not None else Filter()\n        for kwargs in self.conn.get_models(filters.to_dict()):\n            model = Model.decode_value({**kwargs, \"connection\": self.conn})\n            model_list.append(model)\n        return model_list\n\n    def get_model_status(\n        self,\n        dataset_name: str,\n        model_name: str,\n    ) -> Optional[TableStatus]:\n        \"\"\"\n        Get the state of a given model over a dataset.\n\n        Parameters\n        ----------\n        dataset_name : str\n            The name of the dataset that the model is operating over.\n        model_name : str\n            The name of the model we want to fetch the state of.\n\n        Returns\n        ------\n        Union[TableStatus, None]\n            The state of the model or 'None' if the model doesn't exist.\n        \"\"\"\n        try:\n            return self.conn.get_model_status(dataset_name, model_name)\n        except ClientException as e:\n            if e.status_code == 404:\n                return None\n            raise e\n\n    def get_model_eval_requests(\n        self, model: Union[Model, str]\n    ) -> List[Evaluation]:\n        \"\"\"\n        Get all evaluations that have been created for a model.\n\n        This does not return evaluation results.\n\n        `GET` endpoint.\n\n        Parameters\n        ----------\n        model : str\n            The model to search by.\n\n        Returns\n        -------\n        List[Evaluation]\n            A list of evaluations.\n        \"\"\"\n        model_name = model.name if isinstance(model, Model) else model\n        return [\n            Evaluation(**evaluation, connection=self.conn)\n            for evaluation in self.conn.get_model_eval_requests(model_name)  # type: ignore\n        ]\n\n    def delete_model(self, name: str, timeout: int = 0) -> None:\n        \"\"\"\n        Deletes a model.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model to be deleted.\n        timeout : int\n            The number of seconds to wait in order to confirm that the model was deleted.\n        \"\"\"\n        self.conn.delete_model(name)\n        if timeout:\n            for _ in range(timeout):\n                try:\n                    self.get_model(name)\n                except ModelDoesNotExistError:\n                    break\n                time.sleep(1)\n            else:\n                raise TimeoutError(\n                    \"Model wasn't deleted within timeout interval\"\n                )\n\n    def get_evaluations(\n        self,\n        *,\n        evaluation_ids: Optional[List[int]] = None,\n        models: Union[List[Model], List[str], None] = None,\n        datasets: Union[List[Dataset], List[str], None] = None,\n        metrics_to_sort_by: Optional[\n            Dict[str, Union[Dict[str, str], str]]\n        ] = None,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Returns all evaluations associated with user-supplied dataset and/or model names.\n\n        Parameters\n        ----------\n        evaluation_ids : List[int], optional.\n            A list of job IDs to return metrics for.\n        models : Union[List[valor.Model], List[str]], optional\n            A list of model names that we want to return metrics for.\n        datasets : Union[List[valor.Dataset], List[str]], optional\n            A list of dataset names that we want to return metrics for.\n        metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n            An optional dict of metric types to sort the evaluations by.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        List[valor.Evaluation]\n            A list of evaluations.\n        \"\"\"\n        if isinstance(datasets, list):\n            datasets = [  # type: ignore\n                element.name if isinstance(element, Dataset) else element\n                for element in datasets\n            ]\n        if isinstance(models, list):\n            models = [  # type: ignore\n                element.name if isinstance(element, Model) else element\n                for element in models\n            ]\n        return [\n            Evaluation(connection=self.conn, **evaluation)\n            for evaluation in self.conn.get_evaluations(\n                evaluation_ids=evaluation_ids,\n                models=models,  # type: ignore\n                datasets=datasets,  # type: ignore\n                metrics_to_sort_by=metrics_to_sort_by,\n                timeout=timeout,\n            )\n        ]\n\n    def evaluate(\n        self,\n        request: EvaluationRequest,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Creates as many evaluations as necessary to fulfill the request.\n\n        Parameters\n        ----------\n        request : schemas.EvaluationRequest\n            The requested evaluation parameters.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        List[Evaluation]\n            A list of evaluations that meet the parameters.\n        \"\"\"\n        return [\n            Evaluation(**evaluation)\n            for evaluation in self.conn.evaluate(\n                request.to_dict(),\n                allow_retries=allow_retries,\n                timeout=timeout,\n            )\n        ]\n\n    def delete_evaluation(self, evaluation_id: int, timeout: int = 0) -> None:\n        \"\"\"\n        Deletes an evaluation.\n\n        Parameters\n        ----------\n        evaluation_id : int\n            The id of the evaluation to be deleted.\n        timeout : int, default=0\n            The number of seconds to wait in order to confirm that the model was deleted.\n        \"\"\"\n        self.conn.delete_evaluation(evaluation_id)\n        if timeout:\n            for _ in range(timeout):\n                try:\n                    self.get_evaluations(evaluation_ids=[evaluation_id])\n                except EvaluationDoesNotExist:\n                    break\n                time.sleep(1)\n            else:\n                raise TimeoutError(\n                    \"Evaluation wasn't deleted within timeout interval\"\n                )\n
"},{"location":"client_api/Client/#valor.Client-functions","title":"Functions","text":""},{"location":"client_api/Client/#valor.Client.connect","title":"valor.Client.connect(host, access_token=None, reconnect=False)  classmethod","text":"
Establishes a connection to the Valor API.
 
Parameters:
 Name Type Description Default host str 
The host to connect to. Should start with \"http://\" or \"https://\".
 required access_token str 
The access token for the host (if the host requires authentication).
 None Source code in valor/coretypes.py 
@classmethod\ndef connect(\n    cls,\n    host: str,\n    access_token: Optional[str] = None,\n    reconnect: bool = False,\n) -> Client:\n    \"\"\"\n    Establishes a connection to the Valor API.\n\n    Parameters\n    ----------\n    host : str\n        The host to connect to. Should start with \"http://\" or \"https://\".\n    access_token : str\n        The access token for the host (if the host requires authentication).\n    \"\"\"\n    connect(host=host, access_token=access_token, reconnect=reconnect)\n    return cls(get_connection())\n
"},{"location":"client_api/Client/#valor.Client.create_dataset","title":"valor.Client.create_dataset(dataset)","text":"
Creates a dataset.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset to create.
 required Source code in valor/coretypes.py 
def create_dataset(\n    self,\n    dataset: Union[Dataset, dict],\n) -> None:\n    \"\"\"\n    Creates a dataset.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset to create.\n    \"\"\"\n    if isinstance(dataset, Dataset):\n        dataset = dataset.encode_value()\n    self.conn.create_dataset(dataset)\n
"},{"location":"client_api/Client/#valor.Client.create_groundtruths","title":"valor.Client.create_groundtruths(dataset, groundtruths, ignore_existing_datums=False, timeout=None)","text":"
Creates ground truths.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset to create the ground truth for.
 required groundtruths List[GroundTruth] 
The ground truths to create.
 required timeout float 
The number of seconds the client should wait until raising a timeout.
 None ignore_existing_datums bool 
If True, will ignore datums that already exist in the backend. If False, will raise an error if any datums already exist. Default is False.
 False Source code in valor/coretypes.py 
def create_groundtruths(\n    self,\n    dataset: Dataset,\n    groundtruths: List[GroundTruth],\n    ignore_existing_datums: bool = False,\n    timeout: Optional[float] = None,\n):\n    \"\"\"\n    Creates ground truths.\n\n    Parameters\n    ----------\n\n    dataset : valor.Dataset\n        The dataset to create the ground truth for.\n    groundtruths : List[valor.GroundTruth]\n        The ground truths to create.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    ignore_existing_datums : bool, default=False\n        If True, will ignore datums that already exist in the backend.\n        If False, will raise an error if any datums already exist.\n        Default is False.\n    \"\"\"\n    groundtruths_json = []\n    for groundtruth in groundtruths:\n        if not isinstance(groundtruth, GroundTruth):\n            raise TypeError(\n                f\"Expected ground truth to be of type 'valor.GroundTruth' not '{type(groundtruth)}'.\"\n            )\n        if not isinstance(groundtruth.annotations._value, list):\n            raise TypeError\n        groundtruth_dict = groundtruth.encode_value()\n        groundtruth_dict[\"dataset_name\"] = dataset.name\n        groundtruths_json.append(groundtruth_dict)\n    self.conn.create_groundtruths(\n        groundtruths_json,\n        timeout=timeout,\n        ignore_existing_datums=ignore_existing_datums,\n    )\n
"},{"location":"client_api/Client/#valor.Client.create_model","title":"valor.Client.create_model(model)","text":"
Creates a model.
 
Parameters:
 Name Type Description Default model Model 
The model to create.
 required Source code in valor/coretypes.py 
def create_model(\n    self,\n    model: Union[Model, dict],\n):\n    \"\"\"\n    Creates a model.\n\n    Parameters\n    ----------\n    model : valor.Model\n        The model to create.\n    \"\"\"\n    if isinstance(model, Model):\n        model = model.encode_value()\n    self.conn.create_model(model)\n
"},{"location":"client_api/Client/#valor.Client.create_predictions","title":"valor.Client.create_predictions(dataset, model, predictions, timeout=None)","text":"
Creates predictions.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset that is being operated over.
 required model Model 
The model making the prediction.
 required predictions List[Prediction] 
The predictions to create.
 required timeout float 
The number of seconds the client should wait until raising a timeout.
 None Source code in valor/coretypes.py 
def create_predictions(\n    self,\n    dataset: Dataset,\n    model: Model,\n    predictions: List[Prediction],\n    timeout: Optional[float] = None,\n) -> None:\n    \"\"\"\n    Creates predictions.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset that is being operated over.\n    model : valor.Model\n        The model making the prediction.\n    predictions : List[valor.Prediction]\n        The predictions to create.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    \"\"\"\n    predictions_json = []\n    for prediction in predictions:\n        if not isinstance(prediction, Prediction):\n            raise TypeError(\n                f\"Expected prediction to be of type 'valor.Prediction' not '{type(prediction)}'.\"\n            )\n        if not isinstance(prediction.annotations._value, list):\n            raise TypeError\n        prediction_dict = prediction.encode_value()\n        prediction_dict[\"dataset_name\"] = dataset.name\n        prediction_dict[\"model_name\"] = model.name\n        predictions_json.append(prediction_dict)\n    self.conn.create_predictions(predictions_json, timeout=timeout)\n
"},{"location":"client_api/Client/#valor.Client.delete_dataset","title":"valor.Client.delete_dataset(name, timeout=0)","text":"
Deletes a dataset.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset to be deleted.
 required timeout int 
The number of seconds to wait in order to confirm that the dataset was deleted.
 0 Source code in valor/coretypes.py 
def delete_dataset(self, name: str, timeout: int = 0) -> None:\n    \"\"\"\n    Deletes a dataset.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset to be deleted.\n    timeout : int\n        The number of seconds to wait in order to confirm that the dataset was deleted.\n    \"\"\"\n    self.conn.delete_dataset(name)\n    if timeout:\n        for _ in range(timeout):\n            try:\n                self.get_dataset(name)\n            except DatasetDoesNotExistError:\n                break\n            time.sleep(1)\n        else:\n            raise TimeoutError(\n                \"Dataset wasn't deleted within timeout interval\"\n            )\n
"},{"location":"client_api/Client/#valor.Client.delete_evaluation","title":"valor.Client.delete_evaluation(evaluation_id, timeout=0)","text":"
Deletes an evaluation.
 
Parameters:
 Name Type Description Default evaluation_id int 
The id of the evaluation to be deleted.
 required timeout int 
The number of seconds to wait in order to confirm that the model was deleted.
 0 Source code in valor/coretypes.py 
def delete_evaluation(self, evaluation_id: int, timeout: int = 0) -> None:\n    \"\"\"\n    Deletes an evaluation.\n\n    Parameters\n    ----------\n    evaluation_id : int\n        The id of the evaluation to be deleted.\n    timeout : int, default=0\n        The number of seconds to wait in order to confirm that the model was deleted.\n    \"\"\"\n    self.conn.delete_evaluation(evaluation_id)\n    if timeout:\n        for _ in range(timeout):\n            try:\n                self.get_evaluations(evaluation_ids=[evaluation_id])\n            except EvaluationDoesNotExist:\n                break\n            time.sleep(1)\n        else:\n            raise TimeoutError(\n                \"Evaluation wasn't deleted within timeout interval\"\n            )\n
"},{"location":"client_api/Client/#valor.Client.delete_model","title":"valor.Client.delete_model(name, timeout=0)","text":"
Deletes a model.
 
Parameters:
 Name Type Description Default name str 
The name of the model to be deleted.
 required timeout int 
The number of seconds to wait in order to confirm that the model was deleted.
 0 Source code in valor/coretypes.py 
def delete_model(self, name: str, timeout: int = 0) -> None:\n    \"\"\"\n    Deletes a model.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model to be deleted.\n    timeout : int\n        The number of seconds to wait in order to confirm that the model was deleted.\n    \"\"\"\n    self.conn.delete_model(name)\n    if timeout:\n        for _ in range(timeout):\n            try:\n                self.get_model(name)\n            except ModelDoesNotExistError:\n                break\n            time.sleep(1)\n        else:\n            raise TimeoutError(\n                \"Model wasn't deleted within timeout interval\"\n            )\n
"},{"location":"client_api/Client/#valor.Client.evaluate","title":"valor.Client.evaluate(request, *_, allow_retries=False, timeout=None)","text":"
Creates as many evaluations as necessary to fulfill the request.
 
Parameters:
 Name Type Description Default request EvaluationRequest 
The requested evaluation parameters.
 required allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of evaluations that meet the parameters.
 Source code in valor/coretypes.py 
def evaluate(\n    self,\n    request: EvaluationRequest,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Creates as many evaluations as necessary to fulfill the request.\n\n    Parameters\n    ----------\n    request : schemas.EvaluationRequest\n        The requested evaluation parameters.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    List[Evaluation]\n        A list of evaluations that meet the parameters.\n    \"\"\"\n    return [\n        Evaluation(**evaluation)\n        for evaluation in self.conn.evaluate(\n            request.to_dict(),\n            allow_retries=allow_retries,\n            timeout=timeout,\n        )\n    ]\n
"},{"location":"client_api/Client/#valor.Client.finalize_dataset","title":"valor.Client.finalize_dataset(dataset)","text":"
Finalizes a dataset such that new ground truths cannot be added to it.
 
Parameters:
 Name Type Description Default dataset str 
The dataset to be finalized.
 required Source code in valor/coretypes.py 
def finalize_dataset(self, dataset: Union[Dataset, str]) -> None:\n    \"\"\"\n    Finalizes a dataset such that new ground truths cannot be added to it.\n\n    Parameters\n    ----------\n    dataset : str\n        The dataset to be finalized.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    return self.conn.finalize_dataset(name=dataset_name)  # type: ignore\n
"},{"location":"client_api/Client/#valor.Client.finalize_inferences","title":"valor.Client.finalize_inferences(dataset, model)","text":"
Finalizes a model-dataset pairing such that new predictions cannot be added to it.
 Source code in valor/coretypes.py 
def finalize_inferences(\n    self, dataset: Union[Dataset, str], model: Union[Model, str]\n) -> None:\n    \"\"\"\n    Finalizes a model-dataset pairing such that new predictions cannot be added to it.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    model_name = model.name if isinstance(model, Model) else model\n    return self.conn.finalize_inferences(\n        dataset_name=dataset_name,  # type: ignore\n        model_name=model_name,  # type: ignore\n    )\n
"},{"location":"client_api/Client/#valor.Client.get_dataset","title":"valor.Client.get_dataset(name)","text":"
Gets a dataset by name.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset to fetch.
 required 
Returns:
 Type Description Union[Dataset, None] 
A Dataset with a matching name, or 'None' if one doesn't exist.
 Source code in valor/coretypes.py 
def get_dataset(\n    self,\n    name: str,\n) -> Union[Dataset, None]:\n    \"\"\"\n    Gets a dataset by name.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset to fetch.\n\n    Returns\n    -------\n    Union[Dataset, None]\n        A Dataset with a matching name, or 'None' if one doesn't exist.\n    \"\"\"\n    dataset = Dataset.decode_value(\n        {\n            **self.conn.get_dataset(name),\n            \"connection\": self.conn,\n        }\n    )\n    return dataset\n
"},{"location":"client_api/Client/#valor.Client.get_dataset_status","title":"valor.Client.get_dataset_status(name)","text":"
Get the state of a given dataset.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset we want to fetch the state of.
 required 
Returns:
 Type Description TableStatus | None 
The state of the dataset, or 'None' if the dataset does not exist.
 Source code in valor/coretypes.py 
def get_dataset_status(\n    self,\n    name: str,\n) -> Union[TableStatus, None]:\n    \"\"\"\n    Get the state of a given dataset.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset we want to fetch the state of.\n\n    Returns\n    ------\n    TableStatus | None\n        The state of the dataset, or 'None' if the dataset does not exist.\n    \"\"\"\n    try:\n        return self.conn.get_dataset_status(name)\n    except ClientException as e:\n        if e.status_code == 404:\n            return None\n        raise e\n
"},{"location":"client_api/Client/#valor.Client.get_dataset_summary","title":"valor.Client.get_dataset_summary(name, *_, timeout=None)","text":"
Gets the summary of a dataset.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset to create a summary for.
 required 
Returns:
 Type Description DatasetSummary 
A dataclass containing the dataset summary.
 Source code in valor/coretypes.py 
def get_dataset_summary(\n    self,\n    name: str,\n    *_,\n    timeout: Optional[float] = None,\n) -> DatasetSummary:\n    \"\"\"\n    Gets the summary of a dataset.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset to create a summary for.\n\n    Returns\n    -------\n    DatasetSummary\n        A dataclass containing the dataset summary.\n    \"\"\"\n    return DatasetSummary(\n        **self.conn.get_dataset_summary(name, timeout=timeout)\n    )\n
"},{"location":"client_api/Client/#valor.Client.get_datasets","title":"valor.Client.get_datasets(filters=None)","text":"
Get all datasets, with an option to filter results according to some user-defined parameters.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Dataset] 
A list of datasets.
 Source code in valor/coretypes.py 
def get_datasets(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Dataset]:\n    \"\"\"\n    Get all datasets, with an option to filter results according to some user-defined parameters.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    ------\n    List[valor.Dataset]\n        A list of datasets.\n    \"\"\"\n    dataset_list = []\n    filters = filters if filters is not None else Filter()\n    for kwargs in self.conn.get_datasets(filters.to_dict()):\n        dataset = Dataset.decode_value({**kwargs, \"connection\": self.conn})\n        dataset_list.append(dataset)\n    return dataset_list\n
"},{"location":"client_api/Client/#valor.Client.get_datum","title":"valor.Client.get_datum(dataset, uid)","text":"
Get datum. GET endpoint.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset the datum belongs to.
 required uid str 
The UID of the datum.
 required 
Returns:
 Type Description Datum 
The requested datum or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_datum(\n    self,\n    dataset: Union[Dataset, str],\n    uid: str,\n) -> Union[Datum, None]:\n    \"\"\"\n    Get datum.\n    `GET` endpoint.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset the datum belongs to.\n    uid : str\n        The UID of the datum.\n\n    Returns\n    -------\n    valor.Datum\n        The requested datum or 'None' if it doesn't exist.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    resp = self.conn.get_datum(dataset_name=dataset_name, uid=uid)  # type: ignore\n    return Datum.decode_value(resp)\n
"},{"location":"client_api/Client/#valor.Client.get_datums","title":"valor.Client.get_datums(filters=None)","text":"
Get all datums using an optional filter.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Datum] 
A list datums.
 Source code in valor/coretypes.py 
def get_datums(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Datum]:\n    \"\"\"\n    Get all datums using an optional filter.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    -------\n    List[valor.Datum]\n        A list datums.\n    \"\"\"\n\n    filters = filters if filters is not None else Filter()\n    return [\n        Datum.decode_value(datum)\n        for datum in self.conn.get_datums(filters.to_dict())\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_evaluations","title":"valor.Client.get_evaluations(*, evaluation_ids=None, models=None, datasets=None, metrics_to_sort_by=None, timeout=None)","text":"
Returns all evaluations associated with user-supplied dataset and/or model names.
 
Parameters:
 Name Type Description Default evaluation_ids List[int], optional. 
A list of job IDs to return metrics for.
 None models Union[List[Model], List[str]] 
A list of model names that we want to return metrics for.
 None datasets Union[List[Dataset], List[str]] 
A list of dataset names that we want to return metrics for.
 None metrics_to_sort_by dict[str, str | dict[str, str]] 
An optional dict of metric types to sort the evaluations by.
 None timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of evaluations.
 Source code in valor/coretypes.py 
def get_evaluations(\n    self,\n    *,\n    evaluation_ids: Optional[List[int]] = None,\n    models: Union[List[Model], List[str], None] = None,\n    datasets: Union[List[Dataset], List[str], None] = None,\n    metrics_to_sort_by: Optional[\n        Dict[str, Union[Dict[str, str], str]]\n    ] = None,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Returns all evaluations associated with user-supplied dataset and/or model names.\n\n    Parameters\n    ----------\n    evaluation_ids : List[int], optional.\n        A list of job IDs to return metrics for.\n    models : Union[List[valor.Model], List[str]], optional\n        A list of model names that we want to return metrics for.\n    datasets : Union[List[valor.Dataset], List[str]], optional\n        A list of dataset names that we want to return metrics for.\n    metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n        An optional dict of metric types to sort the evaluations by.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    List[valor.Evaluation]\n        A list of evaluations.\n    \"\"\"\n    if isinstance(datasets, list):\n        datasets = [  # type: ignore\n            element.name if isinstance(element, Dataset) else element\n            for element in datasets\n        ]\n    if isinstance(models, list):\n        models = [  # type: ignore\n            element.name if isinstance(element, Model) else element\n            for element in models\n        ]\n    return [\n        Evaluation(connection=self.conn, **evaluation)\n        for evaluation in self.conn.get_evaluations(\n            evaluation_ids=evaluation_ids,\n            models=models,  # type: ignore\n            datasets=datasets,  # type: ignore\n            metrics_to_sort_by=metrics_to_sort_by,\n            timeout=timeout,\n        )\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_groundtruth","title":"valor.Client.get_groundtruth(dataset, datum)","text":"
Get a particular ground truth.
 
Parameters:
 Name Type Description Default dataset Union[Dataset, str] 
The dataset the datum belongs to.
 required datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[GroundTruth, None] 
The matching ground truth or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_groundtruth(\n    self,\n    dataset: Union[Dataset, str],\n    datum: Union[Datum, str],\n) -> Union[GroundTruth, None]:\n    \"\"\"\n    Get a particular ground truth.\n\n    Parameters\n    ----------\n    dataset : Union[Dataset, str]\n        The dataset the datum belongs to.\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[GroundTruth, None]\n        The matching ground truth or 'None' if it doesn't exist.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    datum_uid = datum.uid if isinstance(datum, Datum) else datum\n    try:\n        resp = self.conn.get_groundtruth(\n            dataset_name=dataset_name, datum_uid=datum_uid  # type: ignore\n        )\n        resp.pop(\"dataset_name\")\n        return GroundTruth.decode_value(resp)\n    except ClientException as e:\n        if e.status_code == 404:\n            return None\n        raise e\n
"},{"location":"client_api/Client/#valor.Client.get_labels","title":"valor.Client.get_labels(filters=None)","text":"
Gets all labels using an optional filter.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Label] 
A list of labels.
 Source code in valor/coretypes.py 
def get_labels(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Label]:\n    \"\"\"\n    Gets all labels using an optional filter.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    ------\n    List[valor.Label]\n        A list of labels.\n    \"\"\"\n    filters = filters if filters is not None else Filter()\n    return [\n        Label(**label) for label in self.conn.get_labels(filters.to_dict())\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_labels_from_dataset","title":"valor.Client.get_labels_from_dataset(dataset)","text":"
Get all labels associated with a dataset's ground truths.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset to search by.
 required 
Returns:
 Type Description List[Label] 
A list of labels.
 Source code in valor/coretypes.py 
def get_labels_from_dataset(\n    self, dataset: Union[Dataset, str]\n) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a dataset's ground truths.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset to search by.\n\n    Returns\n    ------\n    List[valor.Label]\n        A list of labels.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    return [\n        Label(**label)\n        for label in self.conn.get_labels_from_dataset(dataset_name)  # type: ignore\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_labels_from_model","title":"valor.Client.get_labels_from_model(model)","text":"
Get all labels associated with a model's ground truths.
 
Parameters:
 Name Type Description Default model Model 
The model to search by.
 required 
Returns:
 Type Description List[Label] 
A list of labels.
 Source code in valor/coretypes.py 
def get_labels_from_model(self, model: Union[Model, str]) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a model's ground truths.\n\n    Parameters\n    ----------\n    model : valor.Model\n        The model to search by.\n\n    Returns\n    ------\n    List[valor.Label]\n        A list of labels.\n    \"\"\"\n    model_name = model.name if isinstance(model, Model) else model\n    return [\n        Label(**label)\n        for label in self.conn.get_labels_from_model(model_name)  # type: ignore\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_model","title":"valor.Client.get_model(name)","text":"
Gets a model by name.
 
Parameters:
 Name Type Description Default name str 
The name of the model to fetch.
 required 
Returns:
 Type Description Union[Model, None] 
A Model with matching name or 'None' if one doesn't exist.
 Source code in valor/coretypes.py 
def get_model(\n    self,\n    name: str,\n) -> Union[Model, None]:\n    \"\"\"\n    Gets a model by name.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model to fetch.\n\n    Returns\n    -------\n    Union[valor.Model, None]\n        A Model with matching name or 'None' if one doesn't exist.\n    \"\"\"\n    return Model.decode_value(\n        {\n            **self.conn.get_model(name),\n            \"connection\": self.conn,\n        }\n    )\n
"},{"location":"client_api/Client/#valor.Client.get_model_eval_requests","title":"valor.Client.get_model_eval_requests(model)","text":"
Get all evaluations that have been created for a model.
 
This does not return evaluation results.
 
GET endpoint.
 
Parameters:
 Name Type Description Default model str 
The model to search by.
 required 
Returns:
 Type Description List[Evaluation] 
A list of evaluations.
 Source code in valor/coretypes.py 
def get_model_eval_requests(\n    self, model: Union[Model, str]\n) -> List[Evaluation]:\n    \"\"\"\n    Get all evaluations that have been created for a model.\n\n    This does not return evaluation results.\n\n    `GET` endpoint.\n\n    Parameters\n    ----------\n    model : str\n        The model to search by.\n\n    Returns\n    -------\n    List[Evaluation]\n        A list of evaluations.\n    \"\"\"\n    model_name = model.name if isinstance(model, Model) else model\n    return [\n        Evaluation(**evaluation, connection=self.conn)\n        for evaluation in self.conn.get_model_eval_requests(model_name)  # type: ignore\n    ]\n
"},{"location":"client_api/Client/#valor.Client.get_model_status","title":"valor.Client.get_model_status(dataset_name, model_name)","text":"
Get the state of a given model over a dataset.
 
Parameters:
 Name Type Description Default dataset_name str 
The name of the dataset that the model is operating over.
 required model_name str 
The name of the model we want to fetch the state of.
 required 
Returns:
 Type Description Union[TableStatus, None] 
The state of the model or 'None' if the model doesn't exist.
 Source code in valor/coretypes.py 
def get_model_status(\n    self,\n    dataset_name: str,\n    model_name: str,\n) -> Optional[TableStatus]:\n    \"\"\"\n    Get the state of a given model over a dataset.\n\n    Parameters\n    ----------\n    dataset_name : str\n        The name of the dataset that the model is operating over.\n    model_name : str\n        The name of the model we want to fetch the state of.\n\n    Returns\n    ------\n    Union[TableStatus, None]\n        The state of the model or 'None' if the model doesn't exist.\n    \"\"\"\n    try:\n        return self.conn.get_model_status(dataset_name, model_name)\n    except ClientException as e:\n        if e.status_code == 404:\n            return None\n        raise e\n
"},{"location":"client_api/Client/#valor.Client.get_models","title":"valor.Client.get_models(filters=None)","text":"
Get all models using an optional filter.
 
Parameters:
 Name Type Description Default filters Filter 
Optional constraints to filter by.
 None 
Returns:
 Type Description List[Model] 
A list of models.
 Source code in valor/coretypes.py 
def get_models(\n    self,\n    filters: Optional[Filter] = None,\n) -> List[Model]:\n    \"\"\"\n    Get all models using an optional filter.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        Optional constraints to filter by.\n\n    Returns\n    ------\n    List[valor.Model]\n        A list of models.\n    \"\"\"\n    model_list = []\n    filters = filters if filters is not None else Filter()\n    for kwargs in self.conn.get_models(filters.to_dict()):\n        model = Model.decode_value({**kwargs, \"connection\": self.conn})\n        model_list.append(model)\n    return model_list\n
"},{"location":"client_api/Client/#valor.Client.get_prediction","title":"valor.Client.get_prediction(dataset, model, datum)","text":"
Get a particular prediction.
 
Parameters:
 Name Type Description Default dataset Union[Dataset, str] 
The dataset the datum belongs to.
 required model Union[Model, str] 
The model that made the prediction.
 required datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[Prediction, None] 
The matching prediction or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_prediction(\n    self,\n    dataset: Union[Dataset, str],\n    model: Union[Model, str],\n    datum: Union[Datum, str],\n) -> Union[Prediction, None]:\n    \"\"\"\n    Get a particular prediction.\n\n    Parameters\n    ----------\n    dataset : Union[Dataset, str]\n        The dataset the datum belongs to.\n    model : Union[Model, str]\n        The model that made the prediction.\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[Prediction, None]\n        The matching prediction or 'None' if it doesn't exist.\n    \"\"\"\n    dataset_name = (\n        dataset.name if isinstance(dataset, Dataset) else dataset\n    )\n    model_name = model.name if isinstance(model, Model) else model\n    datum_uid = datum.uid if isinstance(datum, Datum) else datum\n\n    resp = self.conn.get_prediction(\n        dataset_name=dataset_name,  # type: ignore\n        model_name=model_name,  # type: ignore\n        datum_uid=datum_uid,  # type: ignore\n    )\n    resp.pop(\"dataset_name\")\n    resp.pop(\"model_name\")\n    return Prediction.decode_value(resp)\n
"},{"location":"client_api/Dataset/","title":"Dataset","text":"
               Bases: StaticCollection
 
A class describing a given dataset.
 
Attributes:
 Name Type Description name String 
The name of the dataset.
 metadata Dictionary 
A dictionary of metadata that describes the dataset.
 
Examples:
 
>>> Dataset.create(name=\"dataset1\")\n>>> Dataset.create(name=\"dataset1\", metadata={})\n>>> Dataset.create(name=\"dataset1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n
 Source code in valor/coretypes.py 
class Dataset(StaticCollection):\n    \"\"\"\n    A class describing a given dataset.\n\n    Attributes\n    ----------\n    name : String\n        The name of the dataset.\n    metadata : Dictionary\n        A dictionary of metadata that describes the dataset.\n\n    Examples\n    --------\n    >>> Dataset.create(name=\"dataset1\")\n    >>> Dataset.create(name=\"dataset1\", metadata={})\n    >>> Dataset.create(name=\"dataset1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n    \"\"\"\n\n    name: String = String.symbolic(owner=\"dataset\", name=\"name\")\n    metadata: Dictionary = Dictionary.symbolic(\n        owner=\"dataset\", name=\"metadata\"\n    )\n\n    def __init__(\n        self,\n        *,\n        name: str,\n        metadata: Optional[dict] = None,\n        connection: Optional[ClientConnection] = None,\n    ):\n        \"\"\"\n        Creates a local instance of a dataset.\n\n        Use 'Dataset.create' classmethod to create a dataset with persistence.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset.\n        metadata : dict, optional\n            A dictionary of metadata that describes the dataset.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        self.conn = connection\n        super().__init__(name=name, metadata=metadata if metadata else dict())\n\n    @classmethod\n    def create(\n        cls,\n        name: str,\n        metadata: Optional[Dict[str, Any]] = None,\n        connection: Optional[ClientConnection] = None,\n    ) -> Dataset:\n        \"\"\"\n        Creates a dataset that persists in the back end.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset.\n        metadata : dict, optional\n            A dictionary of metadata that describes the dataset.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        dataset = cls(name=name, metadata=metadata, connection=connection)\n        Client(dataset.conn).create_dataset(dataset)\n        return dataset\n\n    @classmethod\n    def get(\n        cls,\n        name: str,\n        connection: Optional[ClientConnection] = None,\n    ) -> Union[Dataset, None]:\n        \"\"\"\n        Retrieves a dataset from the back end database.\n\n        Parameters\n        ----------\n        name : str\n            The name of the dataset.\n\n        Returns\n        -------\n        Union[valor.Dataset, None]\n            The dataset or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(connection).get_dataset(name)\n\n    def add_groundtruth(\n        self,\n        groundtruth: GroundTruth,\n    ) -> None:\n        \"\"\"\n        Add a ground truth to the dataset.\n\n        Parameters\n        ----------\n        groundtruth : GroundTruth\n            The ground truth to create.\n        \"\"\"\n        Client(self.conn).create_groundtruths(\n            dataset=self,\n            groundtruths=[groundtruth],\n        )\n\n    def add_groundtruths(\n        self,\n        groundtruths: List[GroundTruth],\n        ignore_existing_datums: bool = False,\n        timeout: Optional[float] = 10.0,\n    ) -> None:\n        \"\"\"\n        Add multiple ground truths to the dataset.\n\n        Parameters\n        ----------\n        groundtruths : List[GroundTruth]\n            The ground truths to create.\n        ignore_existing_datums : bool, default=False\n            If True, will ignore datums that already exist in the backend.\n            If False, will raise an error if any datums already exist.\n            Default is False.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        \"\"\"\n        Client(self.conn).create_groundtruths(\n            dataset=self,\n            groundtruths=groundtruths,\n            ignore_existing_datums=ignore_existing_datums,\n            timeout=timeout,\n        )\n\n    def get_groundtruth(\n        self,\n        datum: Union[Datum, str],\n    ) -> Union[GroundTruth, None]:\n        \"\"\"\n        Get a particular ground truth.\n\n        Parameters\n        ----------\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[GroundTruth, None]\n            The matching ground truth or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(self.conn).get_groundtruth(dataset=self, datum=datum)\n\n    def get_labels(\n        self,\n    ) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a given dataset.\n\n        Returns\n        ----------\n        List[Label]\n            A list of `Labels` associated with the dataset.\n        \"\"\"\n        return Client(self.conn).get_labels_from_dataset(self)\n\n    def get_datums(self, filters: Optional[Filter] = None) -> List[Datum]:\n        \"\"\"\n        Get all datums associated with a given dataset.\n\n        Parameters\n        ----------\n        filters : Filter, optional\n            An optional datum filter.\n\n        Returns\n        ----------\n        List[Datum]\n            A list of `Datums` associated with the dataset.\n        \"\"\"\n        if filters is None:\n            filters = Filter()\n        filters.datasets = Dataset.name == self.name  # type: ignore - #issue 605\n        return Client(self.conn).get_datums(filters=filters)\n\n    def get_evaluations(\n        self,\n        metrics_to_sort_by: Optional[\n            Dict[str, Union[Dict[str, str], str]]\n        ] = None,\n        *_,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Get all evaluations associated with a given dataset.\n\n        Parameters\n        ----------\n        metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n            An optional dict of metric types to sort the evaluations by.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        ----------\n        List[Evaluation]\n            A list of `Evaluations` associated with the dataset.\n        \"\"\"\n        return Client(self.conn).get_evaluations(\n            datasets=[self],\n            metrics_to_sort_by=metrics_to_sort_by,\n            timeout=timeout,\n        )\n\n    def get_summary(\n        self, *_, timeout: Optional[float] = None\n    ) -> DatasetSummary:\n        \"\"\"\n        Get the summary of a given dataset.\n\n        Parameters\n        ----------\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        DatasetSummary\n            The summary of the dataset. This class has the following fields:\n\n            name : name of the dataset\n\n            num_datums : total number of datums in the dataset\n\n            num_annotations : total number of labeled annotations in the dataset; if an\n            object (such as a bounding box) has multiple labels, then each label is counted separately\n\n            num_bounding_boxes : total number of bounding boxes in the dataset\n\n            num_polygons : total number of polygons in the dataset\n\n            num_rasters : total number of rasters in the dataset\n\n            labels : list of the unique labels in the dataset\n\n            datum_metadata : list of the unique metadata dictionaries in the dataset that are associated\n            to datums\n\n            groundtruth_annotation_metadata : list of the unique metadata dictionaries in the dataset that are\n            associated to annotations\n        \"\"\"\n        return Client(self.conn).get_dataset_summary(self.name, timeout=timeout)  # type: ignore\n\n    def finalize(\n        self,\n    ):\n        \"\"\"\n        Finalizes the dataset such that new ground truths cannot be added to it.\n        \"\"\"\n        return Client(self.conn).finalize_dataset(self)\n\n    def delete(\n        self,\n        timeout: int = 0,\n    ):\n        \"\"\"\n        Delete the dataset from the back end.\n\n        Parameters\n        ----------\n        timeout : int, default=0\n            Sets a timeout in seconds.\n        \"\"\"\n        Client(self.conn).delete_dataset(self.name, timeout)  # type: ignore\n
"},{"location":"client_api/Dataset/#valor.Dataset-functions","title":"Functions","text":""},{"location":"client_api/Dataset/#valor.Dataset.__init__","title":"valor.Dataset.__init__(*, name, metadata=None, connection=None)","text":"
Creates a local instance of a dataset.
 
Use 'Dataset.create' classmethod to create a dataset with persistence.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset.
 required metadata dict 
A dictionary of metadata that describes the dataset.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    name: str,\n    metadata: Optional[dict] = None,\n    connection: Optional[ClientConnection] = None,\n):\n    \"\"\"\n    Creates a local instance of a dataset.\n\n    Use 'Dataset.create' classmethod to create a dataset with persistence.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset.\n    metadata : dict, optional\n        A dictionary of metadata that describes the dataset.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    self.conn = connection\n    super().__init__(name=name, metadata=metadata if metadata else dict())\n
"},{"location":"client_api/Dataset/#valor.Dataset.add_groundtruth","title":"valor.Dataset.add_groundtruth(groundtruth)","text":"
Add a ground truth to the dataset.
 
Parameters:
 Name Type Description Default groundtruth GroundTruth 
The ground truth to create.
 required Source code in valor/coretypes.py 
def add_groundtruth(\n    self,\n    groundtruth: GroundTruth,\n) -> None:\n    \"\"\"\n    Add a ground truth to the dataset.\n\n    Parameters\n    ----------\n    groundtruth : GroundTruth\n        The ground truth to create.\n    \"\"\"\n    Client(self.conn).create_groundtruths(\n        dataset=self,\n        groundtruths=[groundtruth],\n    )\n
"},{"location":"client_api/Dataset/#valor.Dataset.add_groundtruths","title":"valor.Dataset.add_groundtruths(groundtruths, ignore_existing_datums=False, timeout=10.0)","text":"
Add multiple ground truths to the dataset.
 
Parameters:
 Name Type Description Default groundtruths List[GroundTruth] 
The ground truths to create.
 required ignore_existing_datums bool 
If True, will ignore datums that already exist in the backend. If False, will raise an error if any datums already exist. Default is False.
 False timeout float 
The number of seconds the client should wait until raising a timeout.
 10.0 Source code in valor/coretypes.py 
def add_groundtruths(\n    self,\n    groundtruths: List[GroundTruth],\n    ignore_existing_datums: bool = False,\n    timeout: Optional[float] = 10.0,\n) -> None:\n    \"\"\"\n    Add multiple ground truths to the dataset.\n\n    Parameters\n    ----------\n    groundtruths : List[GroundTruth]\n        The ground truths to create.\n    ignore_existing_datums : bool, default=False\n        If True, will ignore datums that already exist in the backend.\n        If False, will raise an error if any datums already exist.\n        Default is False.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    \"\"\"\n    Client(self.conn).create_groundtruths(\n        dataset=self,\n        groundtruths=groundtruths,\n        ignore_existing_datums=ignore_existing_datums,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Dataset/#valor.Dataset.create","title":"valor.Dataset.create(name, metadata=None, connection=None)  classmethod","text":"
Creates a dataset that persists in the back end.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset.
 required metadata dict 
A dictionary of metadata that describes the dataset.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
@classmethod\ndef create(\n    cls,\n    name: str,\n    metadata: Optional[Dict[str, Any]] = None,\n    connection: Optional[ClientConnection] = None,\n) -> Dataset:\n    \"\"\"\n    Creates a dataset that persists in the back end.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset.\n    metadata : dict, optional\n        A dictionary of metadata that describes the dataset.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    dataset = cls(name=name, metadata=metadata, connection=connection)\n    Client(dataset.conn).create_dataset(dataset)\n    return dataset\n
"},{"location":"client_api/Dataset/#valor.Dataset.delete","title":"valor.Dataset.delete(timeout=0)","text":"
Delete the dataset from the back end.
 
Parameters:
 Name Type Description Default timeout int 
Sets a timeout in seconds.
 0 Source code in valor/coretypes.py 
def delete(\n    self,\n    timeout: int = 0,\n):\n    \"\"\"\n    Delete the dataset from the back end.\n\n    Parameters\n    ----------\n    timeout : int, default=0\n        Sets a timeout in seconds.\n    \"\"\"\n    Client(self.conn).delete_dataset(self.name, timeout)  # type: ignore\n
"},{"location":"client_api/Dataset/#valor.Dataset.finalize","title":"valor.Dataset.finalize()","text":"
Finalizes the dataset such that new ground truths cannot be added to it.
 Source code in valor/coretypes.py 
def finalize(\n    self,\n):\n    \"\"\"\n    Finalizes the dataset such that new ground truths cannot be added to it.\n    \"\"\"\n    return Client(self.conn).finalize_dataset(self)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get","title":"valor.Dataset.get(name, connection=None)  classmethod","text":"
Retrieves a dataset from the back end database.
 
Parameters:
 Name Type Description Default name str 
The name of the dataset.
 required 
Returns:
 Type Description Union[Dataset, None] 
The dataset or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
@classmethod\ndef get(\n    cls,\n    name: str,\n    connection: Optional[ClientConnection] = None,\n) -> Union[Dataset, None]:\n    \"\"\"\n    Retrieves a dataset from the back end database.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset.\n\n    Returns\n    -------\n    Union[valor.Dataset, None]\n        The dataset or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(connection).get_dataset(name)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_datums","title":"valor.Dataset.get_datums(filters=None)","text":"
Get all datums associated with a given dataset.
 
Parameters:
 Name Type Description Default filters Filter 
An optional datum filter.
 None 
Returns:
 Type Description List[Datum] 
A list of Datums associated with the dataset.
 Source code in valor/coretypes.py 
def get_datums(self, filters: Optional[Filter] = None) -> List[Datum]:\n    \"\"\"\n    Get all datums associated with a given dataset.\n\n    Parameters\n    ----------\n    filters : Filter, optional\n        An optional datum filter.\n\n    Returns\n    ----------\n    List[Datum]\n        A list of `Datums` associated with the dataset.\n    \"\"\"\n    if filters is None:\n        filters = Filter()\n    filters.datasets = Dataset.name == self.name  # type: ignore - #issue 605\n    return Client(self.conn).get_datums(filters=filters)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_evaluations","title":"valor.Dataset.get_evaluations(metrics_to_sort_by=None, *_, timeout=None)","text":"
Get all evaluations associated with a given dataset.
 
Parameters:
 Name Type Description Default metrics_to_sort_by dict[str, str | dict[str, str]] 
An optional dict of metric types to sort the evaluations by.
 None timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of Evaluations associated with the dataset.
 Source code in valor/coretypes.py 
def get_evaluations(\n    self,\n    metrics_to_sort_by: Optional[\n        Dict[str, Union[Dict[str, str], str]]\n    ] = None,\n    *_,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Get all evaluations associated with a given dataset.\n\n    Parameters\n    ----------\n    metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n        An optional dict of metric types to sort the evaluations by.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    ----------\n    List[Evaluation]\n        A list of `Evaluations` associated with the dataset.\n    \"\"\"\n    return Client(self.conn).get_evaluations(\n        datasets=[self],\n        metrics_to_sort_by=metrics_to_sort_by,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_groundtruth","title":"valor.Dataset.get_groundtruth(datum)","text":"
Get a particular ground truth.
 
Parameters:
 Name Type Description Default datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[GroundTruth, None] 
The matching ground truth or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_groundtruth(\n    self,\n    datum: Union[Datum, str],\n) -> Union[GroundTruth, None]:\n    \"\"\"\n    Get a particular ground truth.\n\n    Parameters\n    ----------\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[GroundTruth, None]\n        The matching ground truth or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(self.conn).get_groundtruth(dataset=self, datum=datum)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_labels","title":"valor.Dataset.get_labels()","text":"
Get all labels associated with a given dataset.
 
Returns:
 Type Description List[Label] 
A list of Labels associated with the dataset.
 Source code in valor/coretypes.py 
def get_labels(\n    self,\n) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a given dataset.\n\n    Returns\n    ----------\n    List[Label]\n        A list of `Labels` associated with the dataset.\n    \"\"\"\n    return Client(self.conn).get_labels_from_dataset(self)\n
"},{"location":"client_api/Dataset/#valor.Dataset.get_summary","title":"valor.Dataset.get_summary(*_, timeout=None)","text":"
Get the summary of a given dataset.
 
Parameters:
 Name Type Description Default timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description DatasetSummary 
The summary of the dataset. This class has the following fields:
 
name : name of the dataset
 
num_datums : total number of datums in the dataset
 
num_annotations : total number of labeled annotations in the dataset; if an object (such as a bounding box) has multiple labels, then each label is counted separately
 
num_bounding_boxes : total number of bounding boxes in the dataset
 
num_polygons : total number of polygons in the dataset
 
num_rasters : total number of rasters in the dataset
 
labels : list of the unique labels in the dataset
 
datum_metadata : list of the unique metadata dictionaries in the dataset that are associated to datums
 
groundtruth_annotation_metadata : list of the unique metadata dictionaries in the dataset that are associated to annotations
 Source code in valor/coretypes.py 
def get_summary(\n    self, *_, timeout: Optional[float] = None\n) -> DatasetSummary:\n    \"\"\"\n    Get the summary of a given dataset.\n\n    Parameters\n    ----------\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    DatasetSummary\n        The summary of the dataset. This class has the following fields:\n\n        name : name of the dataset\n\n        num_datums : total number of datums in the dataset\n\n        num_annotations : total number of labeled annotations in the dataset; if an\n        object (such as a bounding box) has multiple labels, then each label is counted separately\n\n        num_bounding_boxes : total number of bounding boxes in the dataset\n\n        num_polygons : total number of polygons in the dataset\n\n        num_rasters : total number of rasters in the dataset\n\n        labels : list of the unique labels in the dataset\n\n        datum_metadata : list of the unique metadata dictionaries in the dataset that are associated\n        to datums\n\n        groundtruth_annotation_metadata : list of the unique metadata dictionaries in the dataset that are\n        associated to annotations\n    \"\"\"\n    return Client(self.conn).get_dataset_summary(self.name, timeout=timeout)  # type: ignore\n
"},{"location":"client_api/Datum/","title":"Datum","text":"
               Bases: StaticCollection
 
A class used to store information about a datum for either a 'GroundTruth' or a 'Prediction'.
 
Attributes:
 Name Type Description uid String 
The UID of the datum.
 text (String, optional) 
The text of the datum, if the datum is a piece of text, otherwise None.
 metadata Dictionary 
A dictionary of metadata that describes the datum.
 
Examples:
 
>>> Datum(uid=\"uid1\")\n>>> Datum(uid=\"uid1\", metadata={})\n>>> Datum(uid=\"uid1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n>>> Datum(uid=\"uid2\", text=\"Did Lincoln win the election of 1860?\", metadata={\"query_created_by\": \"Alice\"})\n
 Source code in valor/schemas/symbolic/collections.py 
class Datum(StaticCollection):\n    \"\"\"\n    A class used to store information about a datum for either a 'GroundTruth' or a 'Prediction'.\n\n    Attributes\n    ----------\n    uid : String\n        The UID of the datum.\n    text : String, optional\n        The text of the datum, if the datum is a piece of text, otherwise None.\n    metadata : Dictionary\n        A dictionary of metadata that describes the datum.\n\n    Examples\n    --------\n    >>> Datum(uid=\"uid1\")\n    >>> Datum(uid=\"uid1\", metadata={})\n    >>> Datum(uid=\"uid1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n    >>> Datum(uid=\"uid2\", text=\"Did Lincoln win the election of 1860?\", metadata={\"query_created_by\": \"Alice\"})\n    \"\"\"\n\n    uid: String = String.symbolic(owner=\"datum\", name=\"uid\")\n    text: String = String.symbolic(owner=\"datum\", name=\"text\")\n    metadata: Dictionary = Dictionary.symbolic(owner=\"datum\", name=\"metadata\")\n\n    def __init__(\n        self,\n        *,\n        uid: str,\n        text: Optional[str] = None,\n        metadata: Optional[dict] = None,\n    ):\n        \"\"\"\n        Constructs a datum.\n\n        Parameters\n        ----------\n        uid : str\n            The UID of the datum.\n        text : str, optional\n            The text of the datum, if the datum is a piece of text, otherwise None.\n        metadata : dict, optional\n            A dictionary of metadata that describes the datum.\n        \"\"\"\n        super().__init__(\n            uid=uid, text=text, metadata=metadata if metadata else dict()\n        )\n\n    @staticmethod\n    def formatting() -> Dict[str, Any]:\n        \"\"\"Attribute format mapping.\"\"\"\n        return {\n            \"text\": String.nullable,\n        }\n
"},{"location":"client_api/Datum/#valor.Datum-functions","title":"Functions","text":""},{"location":"client_api/Datum/#valor.Datum.__init__","title":"valor.Datum.__init__(*, uid, text=None, metadata=None)","text":"
Constructs a datum.
 
Parameters:
 Name Type Description Default uid str 
The UID of the datum.
 required text str 
The text of the datum, if the datum is a piece of text, otherwise None.
 None metadata dict 
A dictionary of metadata that describes the datum.
 None Source code in valor/schemas/symbolic/collections.py 
def __init__(\n    self,\n    *,\n    uid: str,\n    text: Optional[str] = None,\n    metadata: Optional[dict] = None,\n):\n    \"\"\"\n    Constructs a datum.\n\n    Parameters\n    ----------\n    uid : str\n        The UID of the datum.\n    text : str, optional\n        The text of the datum, if the datum is a piece of text, otherwise None.\n    metadata : dict, optional\n        A dictionary of metadata that describes the datum.\n    \"\"\"\n    super().__init__(\n        uid=uid, text=text, metadata=metadata if metadata else dict()\n    )\n
"},{"location":"client_api/Datum/#valor.Datum.formatting","title":"valor.Datum.formatting()  staticmethod","text":"
Attribute format mapping.
 Source code in valor/schemas/symbolic/collections.py 
@staticmethod\ndef formatting() -> Dict[str, Any]:\n    \"\"\"Attribute format mapping.\"\"\"\n    return {\n        \"text\": String.nullable,\n    }\n
"},{"location":"client_api/Evaluation/","title":"Evaluation","text":"
Wraps valor.client.Job to provide evaluation-specifc members.
 Source code in valor/coretypes.py 
class Evaluation:\n    \"\"\"\n    Wraps `valor.client.Job` to provide evaluation-specifc members.\n    \"\"\"\n\n    def __init__(\n        self, connection: Optional[ClientConnection] = None, **kwargs\n    ):\n        \"\"\"\n        Defines important attributes of the API's `EvaluationResult`.\n\n        Attributes\n        ----------\n        id : int\n            The ID of the evaluation.\n        dataset_names : list[str]\n            The names of the datasets the model was evaluated over.\n        model_name : str\n            The name of the evaluated model.\n        filters : dict\n            The filter used to select data partitions for evaluation.\n        status : EvaluationStatus\n            The status of the evaluation.\n        metrics : List[dict]\n            A list of metric dictionaries returned by the job.\n        confusion_matrices : List[dict]\n            A list of confusion matrix dictionaries returned by the job.\n        meta : dict[str, str | float | dict], optional\n            A dictionary of metadata describing the evaluation run.\n        \"\"\"\n        if not connection:\n            connection = get_connection()\n        self.conn = connection\n        self.update(**kwargs)\n\n    def update(\n        self,\n        *_,\n        id: int,\n        dataset_names: list[str],\n        model_name: str,\n        filters: dict,\n        parameters: EvaluationParameters,\n        status: EvaluationStatus,\n        metrics: List[Dict],\n        confusion_matrices: List[Dict],\n        created_at: str,\n        meta: dict[str, str | float | dict] | None,\n        **kwargs,\n    ):\n        self.id = id\n        self.dataset_names = dataset_names\n        self.model_name = model_name\n        self.filters = filters\n        self.parameters = (\n            EvaluationParameters(**parameters)\n            if isinstance(parameters, dict)\n            else parameters\n        )\n        self.status = EvaluationStatus(status)\n        self.metrics = metrics\n        self.meta = meta\n        self.confusion_matrices = confusion_matrices\n        self.kwargs = kwargs\n        self.ignored_pred_labels: Optional[List[Label]] = None\n        self.missing_pred_labels: Optional[List[Label]] = None\n        self.created_at = datetime.datetime.strptime(\n            created_at, \"%Y-%m-%dT%H:%M:%S.%fZ\"\n        ).replace(tzinfo=datetime.timezone.utc)\n\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n\n    def poll(self) -> EvaluationStatus:\n        \"\"\"\n        Poll the back end.\n\n        Updates the evaluation with the latest state from the back end.\n\n        Returns\n        -------\n        enums.EvaluationStatus\n            The status of the evaluation.\n\n        Raises\n        ----------\n        ClientException\n            If an Evaluation with the given `evaluation_id` is not found.\n        \"\"\"\n        response = self.conn.get_evaluations(\n            evaluation_ids=[self.id],\n            models=None,\n            datasets=None,\n            metrics_to_sort_by=None,\n        )\n        if not response:\n            raise EvaluationDoesNotExist(self.id)\n        self.update(**response[0])\n        return self.status\n\n    def wait_for_completion(\n        self,\n        *,\n        timeout: Optional[int] = None,\n        interval: float = 1.0,\n    ) -> EvaluationStatus:\n        \"\"\"\n        Blocking function that waits for evaluation to finish.\n\n        Parameters\n        ----------\n        timeout : int, optional\n            Length of timeout in seconds.\n        interval : float, default=1.0\n            Polling interval in seconds.\n        \"\"\"\n        t_start = time.time()\n        while self.poll() not in [\n            EvaluationStatus.DONE,\n            EvaluationStatus.FAILED,\n        ]:\n            time.sleep(interval)\n            if timeout and time.time() - t_start > timeout:\n                raise TimeoutError\n        return self.status\n\n    def __str__(self) -> str:\n        \"\"\"Dumps the object into a JSON formatted string.\"\"\"\n        return json.dumps(self.to_dict(), indent=4)\n\n    def to_dict(self) -> dict:\n        \"\"\"\n        Defines how a `valor.Evaluation` object is serialized into a dictionary.\n\n        Returns\n        ----------\n        dict\n            A dictionary describing an evaluation.\n        \"\"\"\n        return {\n            \"id\": self.id,\n            \"dataset_names\": self.dataset_names,\n            \"model_name\": self.model_name,\n            \"filters\": self.filters,\n            \"parameters\": asdict(self.parameters),\n            \"status\": self.status.value,\n            \"metrics\": self.metrics,\n            \"confusion_matrices\": self.confusion_matrices,\n            \"meta\": self.meta,\n            **self.kwargs,\n        }\n\n    def to_dataframe(\n        self,\n        stratify_by: Optional[Tuple[str, str]] = None,\n    ):\n        \"\"\"\n        Get all metrics associated with a Model and return them in a `pd.DataFrame`.\n\n        Returns\n        ----------\n        pd.DataFrame\n            Evaluation metrics being displayed in a `pd.DataFrame`.\n\n        Raises\n        ------\n        ModuleNotFoundError\n            This function requires the use of `pandas.DataFrame`.\n\n        \"\"\"\n        try:\n            import pandas as pd\n        except ModuleNotFoundError:\n            raise ModuleNotFoundError(\n                \"Must have pandas installed to use `get_metric_dataframes`.\"\n            )\n\n        if not stratify_by:\n            column_type = \"evaluation\"\n            column_name = self.id\n        else:\n            column_type = stratify_by[0]\n            column_name = stratify_by[1]\n\n        metrics = [\n            {**metric, column_type: column_name} for metric in self.metrics\n        ]\n        df = pd.DataFrame(metrics)\n        for k in [\"label\", \"parameters\"]:\n            df[k] = df[k].fillna(\"n/a\")\n        df[\"parameters\"] = df[\"parameters\"].apply(json.dumps)\n        df[\"label\"] = df[\"label\"].apply(\n            lambda x: f\"{x['key']}: {x['value']}\" if x != \"n/a\" else x\n        )\n        df = df.pivot(\n            index=[\"type\", \"parameters\", \"label\"], columns=[column_type]\n        )\n        return df\n
"},{"location":"client_api/Evaluation/#valor.Evaluation-functions","title":"Functions","text":""},{"location":"client_api/Evaluation/#valor.Evaluation.__init__","title":"valor.Evaluation.__init__(connection=None, **kwargs)","text":"
Defines important attributes of the API's EvaluationResult.
 
Attributes:
 Name Type Description id int 
The ID of the evaluation.
 dataset_names list[str] 
The names of the datasets the model was evaluated over.
 model_name str 
The name of the evaluated model.
 filters dict 
The filter used to select data partitions for evaluation.
 status EvaluationStatus 
The status of the evaluation.
 metrics List[dict] 
A list of metric dictionaries returned by the job.
 confusion_matrices List[dict] 
A list of confusion matrix dictionaries returned by the job.
 meta (dict[str, str | float | dict], optional) 
A dictionary of metadata describing the evaluation run.
 Source code in valor/coretypes.py 
def __init__(\n    self, connection: Optional[ClientConnection] = None, **kwargs\n):\n    \"\"\"\n    Defines important attributes of the API's `EvaluationResult`.\n\n    Attributes\n    ----------\n    id : int\n        The ID of the evaluation.\n    dataset_names : list[str]\n        The names of the datasets the model was evaluated over.\n    model_name : str\n        The name of the evaluated model.\n    filters : dict\n        The filter used to select data partitions for evaluation.\n    status : EvaluationStatus\n        The status of the evaluation.\n    metrics : List[dict]\n        A list of metric dictionaries returned by the job.\n    confusion_matrices : List[dict]\n        A list of confusion matrix dictionaries returned by the job.\n    meta : dict[str, str | float | dict], optional\n        A dictionary of metadata describing the evaluation run.\n    \"\"\"\n    if not connection:\n        connection = get_connection()\n    self.conn = connection\n    self.update(**kwargs)\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.__str__","title":"valor.Evaluation.__str__()","text":"
Dumps the object into a JSON formatted string.
 Source code in valor/coretypes.py 
def __str__(self) -> str:\n    \"\"\"Dumps the object into a JSON formatted string.\"\"\"\n    return json.dumps(self.to_dict(), indent=4)\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.poll","title":"valor.Evaluation.poll()","text":"
Poll the back end.
 
Updates the evaluation with the latest state from the back end.
 
Returns:
 Type Description EvaluationStatus 
The status of the evaluation.
 
Raises:
 Type Description ClientException 
If an Evaluation with the given evaluation_id is not found.
 Source code in valor/coretypes.py 
def poll(self) -> EvaluationStatus:\n    \"\"\"\n    Poll the back end.\n\n    Updates the evaluation with the latest state from the back end.\n\n    Returns\n    -------\n    enums.EvaluationStatus\n        The status of the evaluation.\n\n    Raises\n    ----------\n    ClientException\n        If an Evaluation with the given `evaluation_id` is not found.\n    \"\"\"\n    response = self.conn.get_evaluations(\n        evaluation_ids=[self.id],\n        models=None,\n        datasets=None,\n        metrics_to_sort_by=None,\n    )\n    if not response:\n        raise EvaluationDoesNotExist(self.id)\n    self.update(**response[0])\n    return self.status\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.to_dataframe","title":"valor.Evaluation.to_dataframe(stratify_by=None)","text":"
Get all metrics associated with a Model and return them in a pd.DataFrame.
 
Returns:
 Type Description DataFrame 
Evaluation metrics being displayed in a pd.DataFrame.
 
Raises:
 Type Description ModuleNotFoundError 
This function requires the use of pandas.DataFrame.
 Source code in valor/coretypes.py 
def to_dataframe(\n    self,\n    stratify_by: Optional[Tuple[str, str]] = None,\n):\n    \"\"\"\n    Get all metrics associated with a Model and return them in a `pd.DataFrame`.\n\n    Returns\n    ----------\n    pd.DataFrame\n        Evaluation metrics being displayed in a `pd.DataFrame`.\n\n    Raises\n    ------\n    ModuleNotFoundError\n        This function requires the use of `pandas.DataFrame`.\n\n    \"\"\"\n    try:\n        import pandas as pd\n    except ModuleNotFoundError:\n        raise ModuleNotFoundError(\n            \"Must have pandas installed to use `get_metric_dataframes`.\"\n        )\n\n    if not stratify_by:\n        column_type = \"evaluation\"\n        column_name = self.id\n    else:\n        column_type = stratify_by[0]\n        column_name = stratify_by[1]\n\n    metrics = [\n        {**metric, column_type: column_name} for metric in self.metrics\n    ]\n    df = pd.DataFrame(metrics)\n    for k in [\"label\", \"parameters\"]:\n        df[k] = df[k].fillna(\"n/a\")\n    df[\"parameters\"] = df[\"parameters\"].apply(json.dumps)\n    df[\"label\"] = df[\"label\"].apply(\n        lambda x: f\"{x['key']}: {x['value']}\" if x != \"n/a\" else x\n    )\n    df = df.pivot(\n        index=[\"type\", \"parameters\", \"label\"], columns=[column_type]\n    )\n    return df\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.to_dict","title":"valor.Evaluation.to_dict()","text":"
Defines how a valor.Evaluation object is serialized into a dictionary.
 
Returns:
 Type Description dict 
A dictionary describing an evaluation.
 Source code in valor/coretypes.py 
def to_dict(self) -> dict:\n    \"\"\"\n    Defines how a `valor.Evaluation` object is serialized into a dictionary.\n\n    Returns\n    ----------\n    dict\n        A dictionary describing an evaluation.\n    \"\"\"\n    return {\n        \"id\": self.id,\n        \"dataset_names\": self.dataset_names,\n        \"model_name\": self.model_name,\n        \"filters\": self.filters,\n        \"parameters\": asdict(self.parameters),\n        \"status\": self.status.value,\n        \"metrics\": self.metrics,\n        \"confusion_matrices\": self.confusion_matrices,\n        \"meta\": self.meta,\n        **self.kwargs,\n    }\n
"},{"location":"client_api/Evaluation/#valor.Evaluation.wait_for_completion","title":"valor.Evaluation.wait_for_completion(*, timeout=None, interval=1.0)","text":"
Blocking function that waits for evaluation to finish.
 
Parameters:
 Name Type Description Default timeout int 
Length of timeout in seconds.
 None interval float 
Polling interval in seconds.
 1.0 Source code in valor/coretypes.py 
def wait_for_completion(\n    self,\n    *,\n    timeout: Optional[int] = None,\n    interval: float = 1.0,\n) -> EvaluationStatus:\n    \"\"\"\n    Blocking function that waits for evaluation to finish.\n\n    Parameters\n    ----------\n    timeout : int, optional\n        Length of timeout in seconds.\n    interval : float, default=1.0\n        Polling interval in seconds.\n    \"\"\"\n    t_start = time.time()\n    while self.poll() not in [\n        EvaluationStatus.DONE,\n        EvaluationStatus.FAILED,\n    ]:\n        time.sleep(interval)\n        if timeout and time.time() - t_start > timeout:\n            raise TimeoutError\n    return self.status\n
"},{"location":"client_api/Groundtruth/","title":"Groundtruth","text":"
               Bases: StaticCollection
 
An object describing a ground truth (e.g., a human-drawn bounding box on an image).
 
Attributes:
 Name Type Description datum Datum 
The datum associated with the groundtruth.
 annotations List[Annotation] 
The list of annotations associated with the groundtruth.
 
Examples:
 
>>> GroundTruth(\n...     datum=Datum(uid=\"uid1\"),\n...     annotations=[\n...         Annotation(\n...             labels=[Label(key=\"k1\", value=\"v1\")],\n...         )\n...     ]\n... )\n
 Source code in valor/coretypes.py 
class GroundTruth(StaticCollection):\n    \"\"\"\n    An object describing a ground truth (e.g., a human-drawn bounding box on an image).\n\n    Attributes\n    ----------\n    datum : Datum\n        The datum associated with the groundtruth.\n    annotations : List[Annotation]\n        The list of annotations associated with the groundtruth.\n\n    Examples\n    --------\n    >>> GroundTruth(\n    ...     datum=Datum(uid=\"uid1\"),\n    ...     annotations=[\n    ...         Annotation(\n    ...             labels=[Label(key=\"k1\", value=\"v1\")],\n    ...         )\n    ...     ]\n    ... )\n    \"\"\"\n\n    datum: Datum = Datum.symbolic(owner=\"groundtruth\", name=\"datum\")\n    annotations: SymbolicList[Annotation] = SymbolicList[Annotation].symbolic(\n        owner=\"groundtruth\", name=\"annotations\"\n    )\n\n    def __init__(\n        self,\n        *,\n        datum: Datum,\n        annotations: List[Annotation],\n    ):\n        \"\"\"\n        Creates a ground truth.\n\n        Parameters\n        ----------\n        datum : Datum\n            The datum that the ground truth is operating over.\n        annotations : List[Annotation]\n            The list of ground truth annotations.\n        \"\"\"\n        super().__init__(datum=datum, annotations=annotations)\n\n        for annotation in self.annotations:\n            for label in annotation.labels:\n                if label.score is not None:\n                    raise ValueError(\n                        \"GroundTruth labels should not have scores.\"\n                    )\n
"},{"location":"client_api/Groundtruth/#valor.GroundTruth-functions","title":"Functions","text":""},{"location":"client_api/Groundtruth/#valor.GroundTruth.__init__","title":"valor.GroundTruth.__init__(*, datum, annotations)","text":"
Creates a ground truth.
 
Parameters:
 Name Type Description Default datum Datum 
The datum that the ground truth is operating over.
 required annotations List[Annotation] 
The list of ground truth annotations.
 required Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    datum: Datum,\n    annotations: List[Annotation],\n):\n    \"\"\"\n    Creates a ground truth.\n\n    Parameters\n    ----------\n    datum : Datum\n        The datum that the ground truth is operating over.\n    annotations : List[Annotation]\n        The list of ground truth annotations.\n    \"\"\"\n    super().__init__(datum=datum, annotations=annotations)\n\n    for annotation in self.annotations:\n        for label in annotation.labels:\n            if label.score is not None:\n                raise ValueError(\n                    \"GroundTruth labels should not have scores.\"\n                )\n
"},{"location":"client_api/Label/","title":"Label","text":"
               Bases: StaticCollection
 
An object for labeling datasets, models, and annotations.
 
Attributes:
 Name Type Description key String 
The class label key.
 value String 
The class label value.
 score Score 
The label score.
 
Examples:
 
>>> Label(key=\"k1\", value=\"v1\")\n>>> Label(key=\"k1\", value=\"v1\", score=None)\n>>> Label(key=\"k1\", value=\"v1\", score=0.9)\n
 Source code in valor/schemas/symbolic/collections.py 
class Label(StaticCollection):\n    \"\"\"\n    An object for labeling datasets, models, and annotations.\n\n    Attributes\n    ----------\n    key : String\n        The class label key.\n    value : String\n        The class label value.\n    score : Score\n        The label score.\n\n    Examples\n    --------\n    >>> Label(key=\"k1\", value=\"v1\")\n    >>> Label(key=\"k1\", value=\"v1\", score=None)\n    >>> Label(key=\"k1\", value=\"v1\", score=0.9)\n    \"\"\"\n\n    key: String = String.symbolic(owner=\"label\", name=\"key\")\n    value: String = String.symbolic(owner=\"label\", name=\"value\")\n    score: Float = Float.symbolic(owner=\"label\", name=\"score\")\n\n    def __init__(\n        self,\n        *,\n        key: str,\n        value: str,\n        score: Union[float, np.floating, None] = None,\n    ):\n        \"\"\"\n        Initializes an instance of a label.\n\n        Attributes\n        ----------\n        key : str\n            The class label key.\n        value : str\n            The class label value.\n        score : float, optional\n            The label score.\n        \"\"\"\n        super().__init__(key=key, value=value, score=score)\n\n    @staticmethod\n    def formatting() -> Dict[str, Any]:\n        \"\"\"Attribute format mapping.\"\"\"\n        return {\n            \"score\": Float.nullable,\n        }\n\n    def tuple(self):\n        \"\"\"\n        Defines how the `Label` is turned into a tuple.\n\n        Returns\n        ----------\n        tuple\n            A tuple of the `Label's` arguments.\n        \"\"\"\n        return (self.key, self.value, self.score)\n
"},{"location":"client_api/Label/#valor.Label-functions","title":"Functions","text":""},{"location":"client_api/Label/#valor.Label.__init__","title":"valor.Label.__init__(*, key, value, score=None)","text":"
Initializes an instance of a label.
 
Attributes:
 Name Type Description key str 
The class label key.
 value str 
The class label value.
 score (float, optional) 
The label score.
 Source code in valor/schemas/symbolic/collections.py 
def __init__(\n    self,\n    *,\n    key: str,\n    value: str,\n    score: Union[float, np.floating, None] = None,\n):\n    \"\"\"\n    Initializes an instance of a label.\n\n    Attributes\n    ----------\n    key : str\n        The class label key.\n    value : str\n        The class label value.\n    score : float, optional\n        The label score.\n    \"\"\"\n    super().__init__(key=key, value=value, score=score)\n
"},{"location":"client_api/Label/#valor.Label.formatting","title":"valor.Label.formatting()  staticmethod","text":"
Attribute format mapping.
 Source code in valor/schemas/symbolic/collections.py 
@staticmethod\ndef formatting() -> Dict[str, Any]:\n    \"\"\"Attribute format mapping.\"\"\"\n    return {\n        \"score\": Float.nullable,\n    }\n
"},{"location":"client_api/Label/#valor.Label.tuple","title":"valor.Label.tuple()","text":"
Defines how the Label is turned into a tuple.
 
Returns:
 Type Description tuple 
A tuple of the Label's arguments.
 Source code in valor/schemas/symbolic/collections.py 
def tuple(self):\n    \"\"\"\n    Defines how the `Label` is turned into a tuple.\n\n    Returns\n    ----------\n    tuple\n        A tuple of the `Label's` arguments.\n    \"\"\"\n    return (self.key, self.value, self.score)\n
"},{"location":"client_api/Model/","title":"Model","text":"
               Bases: StaticCollection
 
A class describing a model that was trained on a particular dataset.
 
Attributes:
 Name Type Description name String 
The name of the model.
 metadata Dictionary 
A dictionary of metadata that describes the model.
 
Examples:
 
>>> Model.create(name=\"model1\")\n>>> Model.create(name=\"model1\", metadata={})\n>>> Model.create(name=\"model1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n
 Source code in valor/coretypes.py 
class Model(StaticCollection):\n    \"\"\"\n    A class describing a model that was trained on a particular dataset.\n\n    Attributes\n    ----------\n    name : String\n        The name of the model.\n    metadata : Dictionary\n        A dictionary of metadata that describes the model.\n\n    Examples\n    --------\n    >>> Model.create(name=\"model1\")\n    >>> Model.create(name=\"model1\", metadata={})\n    >>> Model.create(name=\"model1\", metadata={\"foo\": \"bar\", \"pi\": 3.14})\n    \"\"\"\n\n    name: String = String.symbolic(owner=\"model\", name=\"name\")\n    metadata: Dictionary = Dictionary.symbolic(owner=\"model\", name=\"metadata\")\n\n    def __init__(\n        self,\n        *,\n        name: str,\n        metadata: Optional[dict] = None,\n        connection: Optional[ClientConnection] = None,\n    ):\n        \"\"\"\n        Creates a local instance of a model.\n\n        Use 'Model.create' classmethod to create a model with persistence.\n\n        Parameters\n        ----------\n        name : String\n            The name of the model.\n        metadata : Dictionary\n            A dictionary of metadata that describes the model.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        self.conn = connection\n        super().__init__(name=name, metadata=metadata if metadata else dict())\n\n    @classmethod\n    def create(\n        cls,\n        name: str,\n        metadata: Optional[Dict[str, Any]] = None,\n        connection: Optional[ClientConnection] = None,\n        **_,\n    ) -> Model:\n        \"\"\"\n        Creates a model that persists in the back end.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model.\n        metadata : dict, optional\n            A dictionary of metadata that describes the model.\n        connection : ClientConnection, optional\n            An initialized client connection.\n        \"\"\"\n        model = cls(name=name, metadata=metadata, connection=connection)\n        Client(connection).create_model(model)\n        return model\n\n    @classmethod\n    def get(\n        cls,\n        name: str,\n        connection: Optional[ClientConnection] = None,\n    ) -> Union[Model, None]:\n        \"\"\"\n        Retrieves a model from the back end database.\n\n        Parameters\n        ----------\n        name : str\n            The name of the model.\n        connection : ClientConnnetion, optional\n            An optional Valor client object for interacting with the API.\n\n        Returns\n        -------\n        Union[valor.Model, None]\n            The model or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(connection).get_model(name)\n\n    def add_prediction(\n        self,\n        dataset: Dataset,\n        prediction: Prediction,\n    ) -> None:\n        \"\"\"\n        Add a prediction to the model.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset that is being operated over.\n        prediction : valor.Prediction\n            The prediction to create.\n        \"\"\"\n        Client(self.conn).create_predictions(\n            dataset=dataset,\n            model=self,\n            predictions=[prediction],\n        )\n\n    def add_predictions(\n        self,\n        dataset: Dataset,\n        predictions: List[Prediction],\n        timeout: Optional[float] = 10.0,\n    ) -> None:\n        \"\"\"\n        Add multiple predictions to the model.\n\n        Parameters\n        ----------\n        dataset : valor.Dataset\n            The dataset that is being operated over.\n        predictions : List[valor.Prediction]\n            The predictions to create.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n        \"\"\"\n        Client(self.conn).create_predictions(\n            dataset=dataset,\n            model=self,\n            predictions=predictions,\n            timeout=timeout,\n        )\n\n    def get_prediction(\n        self, dataset: Union[Dataset, str], datum: Union[Datum, str]\n    ) -> Union[Prediction, None]:\n        \"\"\"\n        Get a particular prediction.\n\n        Parameters\n        ----------\n        dataset : Union[Dataset, str]\n            The dataset the datum belongs to.\n        datum : Union[Datum, str]\n            The desired datum.\n\n        Returns\n        ----------\n        Union[Prediction, None]\n            The matching prediction or 'None' if it doesn't exist.\n        \"\"\"\n        return Client(self.conn).get_prediction(\n            dataset=dataset, model=self, datum=datum\n        )\n\n    def finalize_inferences(self, dataset: Union[Dataset, str]) -> None:\n        \"\"\"\n        Finalizes the model over a dataset such that new predictions cannot be added to it.\n        \"\"\"\n        return Client(self.conn).finalize_inferences(\n            dataset=dataset, model=self\n        )\n\n    def _create_label_map(\n        self,\n        label_map: Optional[Dict[Label, Label]],\n    ) -> Union[List[List[List[str]]], None]:\n        \"\"\"Convert a dictionary of label maps to a serializable list format.\"\"\"\n        if not label_map:\n            return None\n\n        if not isinstance(label_map, dict) or not all(\n            [\n                isinstance(key, Label) and isinstance(value, Label)\n                for key, value in label_map.items()\n            ]\n        ):\n            raise TypeError(\n                \"label_map should be a dictionary with valid Labels for both the key and value.\"\n            )\n\n        return_value = []\n        for key, value in label_map.items():\n            if not all(\n                [\n                    (isinstance(v.key, str) and isinstance(v.value, str))\n                    for v in [key, value]\n                ]\n            ):\n                raise TypeError\n            return_value.append(\n                [\n                    [key.key, key.value],\n                    [value.key, value.value],\n                ]\n            )\n        return return_value\n\n    def evaluate_classification(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        filters: Optional[Filter] = None,\n        label_map: Optional[Dict[Label, Label]] = None,\n        pr_curve_max_examples: int = 1,\n        metrics_to_return: Optional[List[MetricType]] = None,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start a classification evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        label_map : Dict[Label, Label], optional\n            Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n        metrics_to_return : List[MetricType], optional\n            The list of metrics to compute, store, and return to the user.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion.\n        \"\"\"\n        if metrics_to_return and not set(metrics_to_return).issubset(\n            MetricType.classification()\n        ):\n            raise ValueError(\n                f\"The following metrics are not supported for classification: '{set(metrics_to_return) - MetricType.classification()}'\"\n            )\n\n        # format request\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=EvaluationParameters(\n                task_type=TaskType.CLASSIFICATION,\n                label_map=self._create_label_map(label_map=label_map),\n                pr_curve_max_examples=pr_curve_max_examples,\n                metrics_to_return=metrics_to_return,\n            ),\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(\n            request, allow_retries=allow_retries, timeout=timeout\n        )\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def evaluate_detection(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        filters: Optional[Filter] = None,\n        convert_annotations_to_type: Optional[AnnotationType] = None,\n        iou_thresholds_to_compute: Optional[List[float]] = None,\n        iou_thresholds_to_return: Optional[List[float]] = None,\n        label_map: Optional[Dict[Label, Label]] = None,\n        recall_score_threshold: float = 0,\n        metrics_to_return: Optional[List[MetricType]] = None,\n        pr_curve_iou_threshold: float = 0.5,\n        pr_curve_max_examples: int = 1,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start an object-detection evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        convert_annotations_to_type : enums.AnnotationType, optional\n            Forces the object detection evaluation to compute over this type.\n        iou_thresholds_to_compute : List[float], optional\n            Thresholds to compute mAP against.\n        iou_thresholds_to_return : List[float], optional\n            Thresholds to return AP for. Must be subset of `iou_thresholds_to_compute`.\n        label_map : Dict[Label, Label], optional\n            Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n        recall_score_threshold : float, default=0\n            The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\n        metrics_to_return : List[MetricType], optional\n            The list of metrics to compute, store, and return to the user.\n        pr_curve_iou_threshold : float, optional\n            The IOU threshold to use when calculating precision-recall curves. Defaults to 0.5.\n        pr_curve_max_examples : int, optional\n            The maximum number of datum examples to store when calculating PR curves.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion.\n        \"\"\"\n        if metrics_to_return and not set(metrics_to_return).issubset(\n            MetricType.object_detection()\n        ):\n            raise ValueError(\n                f\"The following metrics are not supported for object detection: '{set(metrics_to_return) - MetricType.object_detection()}'\"\n            )\n\n        if iou_thresholds_to_compute is None:\n            iou_thresholds_to_compute = [\n                round(0.5 + 0.05 * i, 2) for i in range(10)\n            ]\n        if iou_thresholds_to_return is None:\n            iou_thresholds_to_return = [0.5, 0.75]\n\n        # format request\n        parameters = EvaluationParameters(\n            task_type=TaskType.OBJECT_DETECTION,\n            convert_annotations_to_type=convert_annotations_to_type,\n            iou_thresholds_to_compute=iou_thresholds_to_compute,\n            iou_thresholds_to_return=iou_thresholds_to_return,\n            label_map=self._create_label_map(label_map=label_map),\n            recall_score_threshold=recall_score_threshold,\n            metrics_to_return=metrics_to_return,\n            pr_curve_iou_threshold=pr_curve_iou_threshold,\n            pr_curve_max_examples=pr_curve_max_examples,\n        )\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=parameters,\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(\n            request, allow_retries=allow_retries, timeout=timeout\n        )\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def evaluate_segmentation(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        filters: Optional[Filter] = None,\n        label_map: Optional[Dict[Label, Label]] = None,\n        metrics_to_return: Optional[List[MetricType]] = None,\n        *_,\n        allow_retries: bool = False,\n        timeout: Optional[float] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start a semantic-segmentation evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        label_map : Dict[Label, Label], optional\n            Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n        metrics_to_return : List[MetricType], optional\n            The list of metrics to compute, store, and return to the user.\n        allow_retries : bool, default = False\n            Option to retry previously failed evaluations.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion\n        \"\"\"\n        if metrics_to_return and not set(metrics_to_return).issubset(\n            MetricType.semantic_segmentation()\n        ):\n            raise ValueError(\n                f\"The following metrics are not supported for semantic segmentation: '{set(metrics_to_return) - MetricType.semantic_segmentation()}'\"\n            )\n\n        # format request\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=EvaluationParameters(\n                task_type=TaskType.SEMANTIC_SEGMENTATION,\n                label_map=self._create_label_map(label_map=label_map),\n                metrics_to_return=metrics_to_return,\n            ),\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(\n            request, allow_retries=allow_retries, timeout=timeout\n        )\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def evaluate_text_generation(\n        self,\n        datasets: Union[Dataset, List[Dataset]],\n        metrics_to_return: List[MetricType],\n        filters: Optional[Filter] = None,\n        llm_api_params: Optional[Dict[str, Union[str, dict]]] = None,\n        metric_params: Optional[Dict[MetricType, Dict[str, Any]]] = None,\n    ) -> Evaluation:\n        \"\"\"\n        Start a classification evaluation job.\n\n        Parameters\n        ----------\n        datasets : Union[Dataset, List[Dataset]]\n            The dataset or list of datasets to evaluate against.\n        metrics_to_return : List[MetricType]\n            The list of metrics to compute, store, and return to the user. This is not optional for text generation evaluations.\n        filters : Filter, optional\n            Optional set of constraints to filter evaluation by.\n        llm_api_params : Dict[str, Union[str,dict]], optional\n            A dictionary of parameters for the LLM API.\n        metric_params : Dict[MetricType, Dict[str,Any]], optional\n            A dictionary of parameters for the metrics used in the evaluation. The keys should be the metrics and the values should be dictionaries of parameters for those metrics.\n\n        Returns\n        -------\n        Evaluation\n            A job object that can be used to track the status of the job and get the metrics of it upon completion.\n        \"\"\"\n        if not set(metrics_to_return).issubset(MetricType.text_generation()):\n            raise ValueError(\n                f\"The following metrics are not supported for text generation: '{set(metrics_to_return) - MetricType.text_generation()}'\"\n            )\n\n        # If no api_key is provided, check the environment variables for an api key.\n        if llm_api_params is not None:\n            if \"api_key\" not in llm_api_params:\n                if \"client\" not in llm_api_params:\n                    raise ValueError(\n                        \"The client must be specified in the llm_api_params.\"\n                    )\n\n                if llm_api_params[\"client\"] == \"openai\":\n                    api_key = os.getenv(\"OPENAI_API_KEY\", None)\n                elif llm_api_params[\"client\"] == \"mistral\":\n                    api_key = os.getenv(\"MISTRAL_API_KEY\", None)\n                elif llm_api_params[\"client\"] == \"mock\":\n                    api_key = \"\"\n                else:\n                    raise ValueError(\n                        \"The client specified in llm_api_params is not supported.\"\n                    )\n\n                if api_key is not None:\n                    llm_api_params[\"api_key\"] = api_key\n\n        bleu_weights = None\n        rouge_types = None\n        rouge_use_stemmer = None\n        if metric_params is not None:\n            if not all(\n                metric in metrics_to_return for metric in metric_params.keys()\n            ):\n                raise ValueError(\n                    \"All metrics in metric_params must be in metrics_to_return.\"\n                )\n\n            if MetricType.BLEU in metric_params:\n                bleu_weights = metric_params[MetricType.BLEU].get(\"weights\")\n\n            if MetricType.ROUGE in metric_params:\n                rouge_types = metric_params[MetricType.ROUGE].get(\n                    \"rouge_types\"\n                )\n                rouge_use_stemmer = metric_params[MetricType.ROUGE].get(\n                    \"use_stemmer\"\n                )\n\n        # format request\n        datasets = datasets if isinstance(datasets, list) else [datasets]\n        filters = filters if filters else Filter()\n        request = EvaluationRequest(\n            dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n            model_names=[self.name],  # type: ignore - issue #604\n            filters=filters,\n            parameters=EvaluationParameters(\n                task_type=TaskType.TEXT_GENERATION,\n                metrics_to_return=metrics_to_return,\n                llm_api_params=llm_api_params,\n                bleu_weights=bleu_weights,\n                rouge_types=rouge_types,\n                rouge_use_stemmer=rouge_use_stemmer,\n            ),\n        )\n\n        # create evaluation\n        evaluation = Client(self.conn).evaluate(request)\n        if len(evaluation) != 1:\n            raise RuntimeError\n        return evaluation[0]\n\n    def delete(self, timeout: int = 0):\n        \"\"\"\n        Delete the `Model` object from the back end.\n\n        Parameters\n        ----------\n        timeout : int, default=0\n            Sets a timeout in seconds.\n        \"\"\"\n        Client(self.conn).delete_model(self.name, timeout)  # type: ignore\n\n    def get_labels(\n        self,\n    ) -> List[Label]:\n        \"\"\"\n        Get all labels associated with a given model.\n\n        Returns\n        ----------\n        List[Label]\n            A list of `Labels` associated with the model.\n        \"\"\"\n        return Client(self.conn).get_labels_from_model(self)\n\n    def get_evaluations(\n        self,\n        metrics_to_sort_by: Optional[\n            Dict[str, Union[Dict[str, str], str]]\n        ] = None,\n        *_,\n        timeout: Optional[float] = None,\n    ) -> List[Evaluation]:\n        \"\"\"\n        Get all evaluations associated with a given model.\n\n        Parameters\n        ----------\n        metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n            An optional dict of metric types to sort the evaluations by.\n        timeout : float, optional\n            The number of seconds the client should wait until raising a timeout.\n\n        Returns\n        ----------\n        List[Evaluation]\n            A list of `Evaluations` associated with the model.\n        \"\"\"\n        return Client(self.conn).get_evaluations(\n            models=[self],\n            metrics_to_sort_by=metrics_to_sort_by,\n            timeout=timeout,\n        )\n
"},{"location":"client_api/Model/#valor.Model-functions","title":"Functions","text":""},{"location":"client_api/Model/#valor.Model.__init__","title":"valor.Model.__init__(*, name, metadata=None, connection=None)","text":"
Creates a local instance of a model.
 
Use 'Model.create' classmethod to create a model with persistence.
 
Parameters:
 Name Type Description Default name String 
The name of the model.
 required metadata Dictionary 
A dictionary of metadata that describes the model.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    name: str,\n    metadata: Optional[dict] = None,\n    connection: Optional[ClientConnection] = None,\n):\n    \"\"\"\n    Creates a local instance of a model.\n\n    Use 'Model.create' classmethod to create a model with persistence.\n\n    Parameters\n    ----------\n    name : String\n        The name of the model.\n    metadata : Dictionary\n        A dictionary of metadata that describes the model.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    self.conn = connection\n    super().__init__(name=name, metadata=metadata if metadata else dict())\n
"},{"location":"client_api/Model/#valor.Model.add_prediction","title":"valor.Model.add_prediction(dataset, prediction)","text":"
Add a prediction to the model.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset that is being operated over.
 required prediction Prediction 
The prediction to create.
 required Source code in valor/coretypes.py 
def add_prediction(\n    self,\n    dataset: Dataset,\n    prediction: Prediction,\n) -> None:\n    \"\"\"\n    Add a prediction to the model.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset that is being operated over.\n    prediction : valor.Prediction\n        The prediction to create.\n    \"\"\"\n    Client(self.conn).create_predictions(\n        dataset=dataset,\n        model=self,\n        predictions=[prediction],\n    )\n
"},{"location":"client_api/Model/#valor.Model.add_predictions","title":"valor.Model.add_predictions(dataset, predictions, timeout=10.0)","text":"
Add multiple predictions to the model.
 
Parameters:
 Name Type Description Default dataset Dataset 
The dataset that is being operated over.
 required predictions List[Prediction] 
The predictions to create.
 required timeout float 
The number of seconds the client should wait until raising a timeout.
 10.0 Source code in valor/coretypes.py 
def add_predictions(\n    self,\n    dataset: Dataset,\n    predictions: List[Prediction],\n    timeout: Optional[float] = 10.0,\n) -> None:\n    \"\"\"\n    Add multiple predictions to the model.\n\n    Parameters\n    ----------\n    dataset : valor.Dataset\n        The dataset that is being operated over.\n    predictions : List[valor.Prediction]\n        The predictions to create.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n    \"\"\"\n    Client(self.conn).create_predictions(\n        dataset=dataset,\n        model=self,\n        predictions=predictions,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Model/#valor.Model.create","title":"valor.Model.create(name, metadata=None, connection=None, **_)  classmethod","text":"
Creates a model that persists in the back end.
 
Parameters:
 Name Type Description Default name str 
The name of the model.
 required metadata dict 
A dictionary of metadata that describes the model.
 None connection ClientConnection 
An initialized client connection.
 None Source code in valor/coretypes.py 
@classmethod\ndef create(\n    cls,\n    name: str,\n    metadata: Optional[Dict[str, Any]] = None,\n    connection: Optional[ClientConnection] = None,\n    **_,\n) -> Model:\n    \"\"\"\n    Creates a model that persists in the back end.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model.\n    metadata : dict, optional\n        A dictionary of metadata that describes the model.\n    connection : ClientConnection, optional\n        An initialized client connection.\n    \"\"\"\n    model = cls(name=name, metadata=metadata, connection=connection)\n    Client(connection).create_model(model)\n    return model\n
"},{"location":"client_api/Model/#valor.Model.delete","title":"valor.Model.delete(timeout=0)","text":"
Delete the Model object from the back end.
 
Parameters:
 Name Type Description Default timeout int 
Sets a timeout in seconds.
 0 Source code in valor/coretypes.py 
def delete(self, timeout: int = 0):\n    \"\"\"\n    Delete the `Model` object from the back end.\n\n    Parameters\n    ----------\n    timeout : int, default=0\n        Sets a timeout in seconds.\n    \"\"\"\n    Client(self.conn).delete_model(self.name, timeout)  # type: ignore\n
"},{"location":"client_api/Model/#valor.Model.evaluate_classification","title":"valor.Model.evaluate_classification(datasets, filters=None, label_map=None, pr_curve_max_examples=1, metrics_to_return=None, *_, allow_retries=False, timeout=None)","text":"
Start a classification evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None label_map Dict[Label, Label] 
Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 None metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user.
 None allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion.
 Source code in valor/coretypes.py 
def evaluate_classification(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    filters: Optional[Filter] = None,\n    label_map: Optional[Dict[Label, Label]] = None,\n    pr_curve_max_examples: int = 1,\n    metrics_to_return: Optional[List[MetricType]] = None,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> Evaluation:\n    \"\"\"\n    Start a classification evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    label_map : Dict[Label, Label], optional\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    metrics_to_return : List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion.\n    \"\"\"\n    if metrics_to_return and not set(metrics_to_return).issubset(\n        MetricType.classification()\n    ):\n        raise ValueError(\n            f\"The following metrics are not supported for classification: '{set(metrics_to_return) - MetricType.classification()}'\"\n        )\n\n    # format request\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=EvaluationParameters(\n            task_type=TaskType.CLASSIFICATION,\n            label_map=self._create_label_map(label_map=label_map),\n            pr_curve_max_examples=pr_curve_max_examples,\n            metrics_to_return=metrics_to_return,\n        ),\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(\n        request, allow_retries=allow_retries, timeout=timeout\n    )\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.evaluate_detection","title":"valor.Model.evaluate_detection(datasets, filters=None, convert_annotations_to_type=None, iou_thresholds_to_compute=None, iou_thresholds_to_return=None, label_map=None, recall_score_threshold=0, metrics_to_return=None, pr_curve_iou_threshold=0.5, pr_curve_max_examples=1, *_, allow_retries=False, timeout=None)","text":"
Start an object-detection evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None convert_annotations_to_type AnnotationType 
Forces the object detection evaluation to compute over this type.
 None iou_thresholds_to_compute List[float] 
Thresholds to compute mAP against.
 None iou_thresholds_to_return List[float] 
Thresholds to return AP for. Must be subset of iou_thresholds_to_compute.
 None label_map Dict[Label, Label] 
Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 None recall_score_threshold float 
The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.
 0 metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user.
 None pr_curve_iou_threshold float 
The IOU threshold to use when calculating precision-recall curves. Defaults to 0.5.
 0.5 pr_curve_max_examples int 
The maximum number of datum examples to store when calculating PR curves.
 1 allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion.
 Source code in valor/coretypes.py 
def evaluate_detection(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    filters: Optional[Filter] = None,\n    convert_annotations_to_type: Optional[AnnotationType] = None,\n    iou_thresholds_to_compute: Optional[List[float]] = None,\n    iou_thresholds_to_return: Optional[List[float]] = None,\n    label_map: Optional[Dict[Label, Label]] = None,\n    recall_score_threshold: float = 0,\n    metrics_to_return: Optional[List[MetricType]] = None,\n    pr_curve_iou_threshold: float = 0.5,\n    pr_curve_max_examples: int = 1,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> Evaluation:\n    \"\"\"\n    Start an object-detection evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    convert_annotations_to_type : enums.AnnotationType, optional\n        Forces the object detection evaluation to compute over this type.\n    iou_thresholds_to_compute : List[float], optional\n        Thresholds to compute mAP against.\n    iou_thresholds_to_return : List[float], optional\n        Thresholds to return AP for. Must be subset of `iou_thresholds_to_compute`.\n    label_map : Dict[Label, Label], optional\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    recall_score_threshold : float, default=0\n        The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\n    metrics_to_return : List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    pr_curve_iou_threshold : float, optional\n        The IOU threshold to use when calculating precision-recall curves. Defaults to 0.5.\n    pr_curve_max_examples : int, optional\n        The maximum number of datum examples to store when calculating PR curves.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion.\n    \"\"\"\n    if metrics_to_return and not set(metrics_to_return).issubset(\n        MetricType.object_detection()\n    ):\n        raise ValueError(\n            f\"The following metrics are not supported for object detection: '{set(metrics_to_return) - MetricType.object_detection()}'\"\n        )\n\n    if iou_thresholds_to_compute is None:\n        iou_thresholds_to_compute = [\n            round(0.5 + 0.05 * i, 2) for i in range(10)\n        ]\n    if iou_thresholds_to_return is None:\n        iou_thresholds_to_return = [0.5, 0.75]\n\n    # format request\n    parameters = EvaluationParameters(\n        task_type=TaskType.OBJECT_DETECTION,\n        convert_annotations_to_type=convert_annotations_to_type,\n        iou_thresholds_to_compute=iou_thresholds_to_compute,\n        iou_thresholds_to_return=iou_thresholds_to_return,\n        label_map=self._create_label_map(label_map=label_map),\n        recall_score_threshold=recall_score_threshold,\n        metrics_to_return=metrics_to_return,\n        pr_curve_iou_threshold=pr_curve_iou_threshold,\n        pr_curve_max_examples=pr_curve_max_examples,\n    )\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=parameters,\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(\n        request, allow_retries=allow_retries, timeout=timeout\n    )\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.evaluate_segmentation","title":"valor.Model.evaluate_segmentation(datasets, filters=None, label_map=None, metrics_to_return=None, *_, allow_retries=False, timeout=None)","text":"
Start a semantic-segmentation evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None label_map Dict[Label, Label] 
Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.
 None metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user.
 None allow_retries bool 
Option to retry previously failed evaluations.
 = False timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion
 Source code in valor/coretypes.py 
def evaluate_segmentation(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    filters: Optional[Filter] = None,\n    label_map: Optional[Dict[Label, Label]] = None,\n    metrics_to_return: Optional[List[MetricType]] = None,\n    *_,\n    allow_retries: bool = False,\n    timeout: Optional[float] = None,\n) -> Evaluation:\n    \"\"\"\n    Start a semantic-segmentation evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    label_map : Dict[Label, Label], optional\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    metrics_to_return : List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    allow_retries : bool, default = False\n        Option to retry previously failed evaluations.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion\n    \"\"\"\n    if metrics_to_return and not set(metrics_to_return).issubset(\n        MetricType.semantic_segmentation()\n    ):\n        raise ValueError(\n            f\"The following metrics are not supported for semantic segmentation: '{set(metrics_to_return) - MetricType.semantic_segmentation()}'\"\n        )\n\n    # format request\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=EvaluationParameters(\n            task_type=TaskType.SEMANTIC_SEGMENTATION,\n            label_map=self._create_label_map(label_map=label_map),\n            metrics_to_return=metrics_to_return,\n        ),\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(\n        request, allow_retries=allow_retries, timeout=timeout\n    )\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.evaluate_text_generation","title":"valor.Model.evaluate_text_generation(datasets, metrics_to_return, filters=None, llm_api_params=None, metric_params=None)","text":"
Start a classification evaluation job.
 
Parameters:
 Name Type Description Default datasets Union[Dataset, List[Dataset]] 
The dataset or list of datasets to evaluate against.
 required metrics_to_return List[MetricType] 
The list of metrics to compute, store, and return to the user. This is not optional for text generation evaluations.
 required filters Filter 
Optional set of constraints to filter evaluation by.
 None llm_api_params Dict[str, Union[str, dict]] 
A dictionary of parameters for the LLM API.
 None metric_params Dict[MetricType, Dict[str, Any]] 
A dictionary of parameters for the metrics used in the evaluation. The keys should be the metrics and the values should be dictionaries of parameters for those metrics.
 None 
Returns:
 Type Description Evaluation 
A job object that can be used to track the status of the job and get the metrics of it upon completion.
 Source code in valor/coretypes.py 
def evaluate_text_generation(\n    self,\n    datasets: Union[Dataset, List[Dataset]],\n    metrics_to_return: List[MetricType],\n    filters: Optional[Filter] = None,\n    llm_api_params: Optional[Dict[str, Union[str, dict]]] = None,\n    metric_params: Optional[Dict[MetricType, Dict[str, Any]]] = None,\n) -> Evaluation:\n    \"\"\"\n    Start a classification evaluation job.\n\n    Parameters\n    ----------\n    datasets : Union[Dataset, List[Dataset]]\n        The dataset or list of datasets to evaluate against.\n    metrics_to_return : List[MetricType]\n        The list of metrics to compute, store, and return to the user. This is not optional for text generation evaluations.\n    filters : Filter, optional\n        Optional set of constraints to filter evaluation by.\n    llm_api_params : Dict[str, Union[str,dict]], optional\n        A dictionary of parameters for the LLM API.\n    metric_params : Dict[MetricType, Dict[str,Any]], optional\n        A dictionary of parameters for the metrics used in the evaluation. The keys should be the metrics and the values should be dictionaries of parameters for those metrics.\n\n    Returns\n    -------\n    Evaluation\n        A job object that can be used to track the status of the job and get the metrics of it upon completion.\n    \"\"\"\n    if not set(metrics_to_return).issubset(MetricType.text_generation()):\n        raise ValueError(\n            f\"The following metrics are not supported for text generation: '{set(metrics_to_return) - MetricType.text_generation()}'\"\n        )\n\n    # If no api_key is provided, check the environment variables for an api key.\n    if llm_api_params is not None:\n        if \"api_key\" not in llm_api_params:\n            if \"client\" not in llm_api_params:\n                raise ValueError(\n                    \"The client must be specified in the llm_api_params.\"\n                )\n\n            if llm_api_params[\"client\"] == \"openai\":\n                api_key = os.getenv(\"OPENAI_API_KEY\", None)\n            elif llm_api_params[\"client\"] == \"mistral\":\n                api_key = os.getenv(\"MISTRAL_API_KEY\", None)\n            elif llm_api_params[\"client\"] == \"mock\":\n                api_key = \"\"\n            else:\n                raise ValueError(\n                    \"The client specified in llm_api_params is not supported.\"\n                )\n\n            if api_key is not None:\n                llm_api_params[\"api_key\"] = api_key\n\n    bleu_weights = None\n    rouge_types = None\n    rouge_use_stemmer = None\n    if metric_params is not None:\n        if not all(\n            metric in metrics_to_return for metric in metric_params.keys()\n        ):\n            raise ValueError(\n                \"All metrics in metric_params must be in metrics_to_return.\"\n            )\n\n        if MetricType.BLEU in metric_params:\n            bleu_weights = metric_params[MetricType.BLEU].get(\"weights\")\n\n        if MetricType.ROUGE in metric_params:\n            rouge_types = metric_params[MetricType.ROUGE].get(\n                \"rouge_types\"\n            )\n            rouge_use_stemmer = metric_params[MetricType.ROUGE].get(\n                \"use_stemmer\"\n            )\n\n    # format request\n    datasets = datasets if isinstance(datasets, list) else [datasets]\n    filters = filters if filters else Filter()\n    request = EvaluationRequest(\n        dataset_names=[dataset.name for dataset in datasets],  # type: ignore - issue #604\n        model_names=[self.name],  # type: ignore - issue #604\n        filters=filters,\n        parameters=EvaluationParameters(\n            task_type=TaskType.TEXT_GENERATION,\n            metrics_to_return=metrics_to_return,\n            llm_api_params=llm_api_params,\n            bleu_weights=bleu_weights,\n            rouge_types=rouge_types,\n            rouge_use_stemmer=rouge_use_stemmer,\n        ),\n    )\n\n    # create evaluation\n    evaluation = Client(self.conn).evaluate(request)\n    if len(evaluation) != 1:\n        raise RuntimeError\n    return evaluation[0]\n
"},{"location":"client_api/Model/#valor.Model.finalize_inferences","title":"valor.Model.finalize_inferences(dataset)","text":"
Finalizes the model over a dataset such that new predictions cannot be added to it.
 Source code in valor/coretypes.py 
def finalize_inferences(self, dataset: Union[Dataset, str]) -> None:\n    \"\"\"\n    Finalizes the model over a dataset such that new predictions cannot be added to it.\n    \"\"\"\n    return Client(self.conn).finalize_inferences(\n        dataset=dataset, model=self\n    )\n
"},{"location":"client_api/Model/#valor.Model.get","title":"valor.Model.get(name, connection=None)  classmethod","text":"
Retrieves a model from the back end database.
 
Parameters:
 Name Type Description Default name str 
The name of the model.
 required connection ClientConnnetion 
An optional Valor client object for interacting with the API.
 None 
Returns:
 Type Description Union[Model, None] 
The model or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
@classmethod\ndef get(\n    cls,\n    name: str,\n    connection: Optional[ClientConnection] = None,\n) -> Union[Model, None]:\n    \"\"\"\n    Retrieves a model from the back end database.\n\n    Parameters\n    ----------\n    name : str\n        The name of the model.\n    connection : ClientConnnetion, optional\n        An optional Valor client object for interacting with the API.\n\n    Returns\n    -------\n    Union[valor.Model, None]\n        The model or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(connection).get_model(name)\n
"},{"location":"client_api/Model/#valor.Model.get_evaluations","title":"valor.Model.get_evaluations(metrics_to_sort_by=None, *_, timeout=None)","text":"
Get all evaluations associated with a given model.
 
Parameters:
 Name Type Description Default metrics_to_sort_by dict[str, str | dict[str, str]] 
An optional dict of metric types to sort the evaluations by.
 None timeout float 
The number of seconds the client should wait until raising a timeout.
 None 
Returns:
 Type Description List[Evaluation] 
A list of Evaluations associated with the model.
 Source code in valor/coretypes.py 
def get_evaluations(\n    self,\n    metrics_to_sort_by: Optional[\n        Dict[str, Union[Dict[str, str], str]]\n    ] = None,\n    *_,\n    timeout: Optional[float] = None,\n) -> List[Evaluation]:\n    \"\"\"\n    Get all evaluations associated with a given model.\n\n    Parameters\n    ----------\n    metrics_to_sort_by : dict[str, str | dict[str, str]], optional\n        An optional dict of metric types to sort the evaluations by.\n    timeout : float, optional\n        The number of seconds the client should wait until raising a timeout.\n\n    Returns\n    ----------\n    List[Evaluation]\n        A list of `Evaluations` associated with the model.\n    \"\"\"\n    return Client(self.conn).get_evaluations(\n        models=[self],\n        metrics_to_sort_by=metrics_to_sort_by,\n        timeout=timeout,\n    )\n
"},{"location":"client_api/Model/#valor.Model.get_labels","title":"valor.Model.get_labels()","text":"
Get all labels associated with a given model.
 
Returns:
 Type Description List[Label] 
A list of Labels associated with the model.
 Source code in valor/coretypes.py 
def get_labels(\n    self,\n) -> List[Label]:\n    \"\"\"\n    Get all labels associated with a given model.\n\n    Returns\n    ----------\n    List[Label]\n        A list of `Labels` associated with the model.\n    \"\"\"\n    return Client(self.conn).get_labels_from_model(self)\n
"},{"location":"client_api/Model/#valor.Model.get_prediction","title":"valor.Model.get_prediction(dataset, datum)","text":"
Get a particular prediction.
 
Parameters:
 Name Type Description Default dataset Union[Dataset, str] 
The dataset the datum belongs to.
 required datum Union[Datum, str] 
The desired datum.
 required 
Returns:
 Type Description Union[Prediction, None] 
The matching prediction or 'None' if it doesn't exist.
 Source code in valor/coretypes.py 
def get_prediction(\n    self, dataset: Union[Dataset, str], datum: Union[Datum, str]\n) -> Union[Prediction, None]:\n    \"\"\"\n    Get a particular prediction.\n\n    Parameters\n    ----------\n    dataset : Union[Dataset, str]\n        The dataset the datum belongs to.\n    datum : Union[Datum, str]\n        The desired datum.\n\n    Returns\n    ----------\n    Union[Prediction, None]\n        The matching prediction or 'None' if it doesn't exist.\n    \"\"\"\n    return Client(self.conn).get_prediction(\n        dataset=dataset, model=self, datum=datum\n    )\n
"},{"location":"client_api/Prediction/","title":"Prediction","text":"
               Bases: StaticCollection
 
An object describing a prediction (e.g., a machine-drawn bounding box on an image).
 
Attributes:
 Name Type Description datum Datum 
The datum associated with the prediction.
 annotations List[Annotation] 
The list of annotations associated with the prediction.
 
Examples:
 
>>> Prediction(\n...     datum=Datum(uid=\"uid1\"),\n...     annotations=[\n...         Annotation(\n...             labels=[\n...                 Label(key=\"k1\", value=\"v1\", score=0.9),\n...                 Label(key=\"k1\", value=\"v1\", score=0.1)\n...             ],\n...         )\n...     ]\n... )\n
 Source code in valor/coretypes.py 
class Prediction(StaticCollection):\n    \"\"\"\n    An object describing a prediction (e.g., a machine-drawn bounding box on an image).\n\n    Attributes\n    ----------\n    datum : Datum\n        The datum associated with the prediction.\n    annotations : List[Annotation]\n        The list of annotations associated with the prediction.\n\n    Examples\n    --------\n    >>> Prediction(\n    ...     datum=Datum(uid=\"uid1\"),\n    ...     annotations=[\n    ...         Annotation(\n    ...             labels=[\n    ...                 Label(key=\"k1\", value=\"v1\", score=0.9),\n    ...                 Label(key=\"k1\", value=\"v1\", score=0.1)\n    ...             ],\n    ...         )\n    ...     ]\n    ... )\n    \"\"\"\n\n    datum: Datum = Datum.symbolic(owner=\"prediction\", name=\"datum\")\n    annotations: SymbolicList[Annotation] = SymbolicList[Annotation].symbolic(\n        owner=\"prediction\", name=\"annotations\"\n    )\n\n    def __init__(\n        self,\n        *,\n        datum: Datum,\n        annotations: List[Annotation],\n    ):\n        \"\"\"\n        Creates a prediction.\n\n        Parameters\n        ----------\n        datum : Datum\n            The datum that the prediction is operating over.\n        annotations : List[Annotation]\n            The list of predicted annotations.\n        \"\"\"\n        super().__init__(datum=datum, annotations=annotations)\n
"},{"location":"client_api/Prediction/#valor.Prediction-functions","title":"Functions","text":""},{"location":"client_api/Prediction/#valor.Prediction.__init__","title":"valor.Prediction.__init__(*, datum, annotations)","text":"
Creates a prediction.
 
Parameters:
 Name Type Description Default datum Datum 
The datum that the prediction is operating over.
 required annotations List[Annotation] 
The list of predicted annotations.
 required Source code in valor/coretypes.py 
def __init__(\n    self,\n    *,\n    datum: Datum,\n    annotations: List[Annotation],\n):\n    \"\"\"\n    Creates a prediction.\n\n    Parameters\n    ----------\n    datum : Datum\n        The datum that the prediction is operating over.\n    annotations : List[Annotation]\n        The list of predicted annotations.\n    \"\"\"\n    super().__init__(datum=datum, annotations=annotations)\n
"},{"location":"client_api/Viz/","title":"Viz","text":""},{"location":"client_api/Viz/#valor.viz-classes","title":"Classes","text":""},{"location":"client_api/Viz/#valor.viz-functions","title":"Functions","text":""},{"location":"client_api/Viz/#valor.viz.create_combined_segmentation_mask","title":"valor.viz.create_combined_segmentation_mask(annotated_datum, label_key, filter_on_instance_segmentations=False)","text":"
Creates a combined segmentation mask from a list of segmentations.
 
Parameters:
 Name Type Description Default annotated_datum Union[GroundTruth, Prediction] 
A list of segmentations. These all must have the same image attribute.
 required label_key str 
The label key to use.
 required filter_on_instance_segmentations bool 
Whether to filter on instance segmentations or not.
 False 
Returns:
 Type Description tuple 
The first element of the tuple is the combined mask, as an RGB PIL image. The second element is a color legend: it's a dict with the unique labels as keys and the PIL image swatches as values.
 
Raises:
 Type Description RuntimeError 
If all segmentations don't belong to the same image or there is a segmentation that doesn't have label_key as the key of one of its labels.
 ValueError 
If there aren't any segmentations.
 Source code in valor/viz.py 
def create_combined_segmentation_mask(\n    annotated_datum: Union[GroundTruth, Prediction],\n    label_key: str,\n    filter_on_instance_segmentations: bool = False,\n) -> Tuple[Image.Image, Dict[str, Image.Image]]:\n    \"\"\"\n    Creates a combined segmentation mask from a list of segmentations.\n\n    Parameters\n    -------\n    annotated_datum : Union[GroundTruth, Prediction]\n        A list of segmentations. These all must have the same `image` attribute.\n    label_key : str\n        The label key to use.\n    filter_on_instance_segmentations : bool, optional\n        Whether to filter on instance segmentations or not.\n\n    Returns\n    -------\n    tuple\n        The first element of the tuple is the combined mask, as an RGB PIL image. The second\n        element is a color legend: it's a dict with the unique labels as keys and the\n        PIL image swatches as values.\n\n    Raises\n    ------\n    RuntimeError\n        If all segmentations don't belong to the same image or there is a\n        segmentation that doesn't have `label_key` as the key of one of its labels.\n    ValueError\n        If there aren't any segmentations.\n    \"\"\"\n\n    # validate input type\n    if not isinstance(annotated_datum, (GroundTruth, Prediction)):\n        raise ValueError(\"Expected either a 'GroundTruth' or 'Prediction'\")\n\n    # verify there are a nonzero number of annotations\n    if len(annotated_datum.annotations) == 0:\n        raise ValueError(\"annotations cannot be empty.\")\n\n    # validate raster size\n    img_h = None\n    img_w = None\n    for annotation in annotated_datum.annotations:\n        raster = annotation.raster\n        if raster.get_value() is None:\n            raise ValueError(\"No raster exists.\")\n        if img_h is None:\n            img_h = raster.height\n        if img_w is None:\n            img_w = raster.width\n        if (img_h != raster.height) or (img_w != raster.width):\n            raise ValueError(\n                f\"Size mismatch between rasters. {(img_h, img_w)} != {(raster.height, raster.width)}\"\n            )\n    if img_h is None or img_w is None:\n        raise ValueError(\n            f\"Segmentation bounds not properly defined. {(img_h, img_w)}\"\n        )\n\n    # unpack raster annotations\n    annotations: List[Annotation] = []\n    for annotation in annotated_datum.annotations:\n        if (\n            annotation.is_instance or False\n        ) == filter_on_instance_segmentations:\n            annotations.append(annotation)\n\n    # unpack label values\n    label_values = []\n    for annotation in annotations:\n        for label in annotation.labels:\n            if label.key == label_key:\n                label_values.append(label.value)\n    if not label_values:\n        raise RuntimeError(\n            f\"Annotation doesn't have a label with key `{label_key}`\"\n        )\n\n    # assign label coloring\n    unique_label_values = list(set(label_values))\n    label_value_to_color = {\n        v: COLOR_MAP[i] for i, v in enumerate(unique_label_values)\n    }\n    seg_colors = [label_value_to_color[v] for v in label_values]\n\n    # create mask\n    combined_mask = np.zeros((img_h, img_w, 3), dtype=np.uint8)\n    for annotation, color in zip(annotations, seg_colors):\n        raster = annotation.raster\n        if raster.get_value() is None:\n            raise ValueError(\"No raster exists.\")\n        if raster.array is not None:\n            if raster.geometry is None:\n                mask = raster.array\n            elif isinstance(raster.geometry, schemas.MultiPolygon):\n                mask = _polygons_to_binary_mask(\n                    raster.geometry.to_polygons(),\n                    img_w=img_w,\n                    img_h=img_h,\n                )\n            elif isinstance(raster.geometry, (schemas.Box, schemas.Polygon)):\n                mask = _polygons_to_binary_mask(\n                    [raster.geometry],\n                    img_w=img_w,\n                    img_h=img_h,\n                )\n            else:\n                continue\n            combined_mask[np.where(mask)] = color\n        else:\n            continue\n\n    legend = {\n        v: Image.new(\"RGB\", (20, 20), color)\n        for v, color in label_value_to_color.items()\n    }\n\n    return Image.fromarray(combined_mask), legend\n
"},{"location":"client_api/Viz/#valor.viz.draw_bounding_box_on_image","title":"valor.viz.draw_bounding_box_on_image(bounding_box, img, color=(255, 0, 0))","text":"
Draws a bounding polygon on an image. This operation is not done in place.
 
Parameters:
 Name Type Description Default bounding_box Box 
Bounding box to draw on the image.
 required img Image 
Pillow image to draw on.
 required color Tuple[int, int, int] 
RGB tuple of the color to use.
 (255, 0, 0) 
Returns:
 Type Description img 
Pillow image with bounding box drawn on it.
 Source code in valor/viz.py 
def draw_bounding_box_on_image(\n    bounding_box: schemas.Box,\n    img: Image.Image,\n    color: Tuple[int, int, int] = (255, 0, 0),\n) -> Image.Image:\n    \"\"\"Draws a bounding polygon on an image. This operation is not done in place.\n\n    Parameters\n    ----------\n    bounding_box\n        Bounding box to draw on the image.\n    img\n        Pillow image to draw on.\n    color\n        RGB tuple of the color to use.\n\n    Returns\n    -------\n    img\n        Pillow image with bounding box drawn on it.\n    \"\"\"\n    coords = bounding_box.get_value()\n    return _draw_bounding_polygon_on_image(\n        schemas.Polygon(coords), img, color=color, inplace=False\n    )\n
"},{"location":"client_api/Viz/#valor.viz.draw_detections_on_image","title":"valor.viz.draw_detections_on_image(detections, img)","text":"
Draws detections (bounding boxes and labels) on an image.
 
Parameters:
 Name Type Description Default detections List[Union[GroundTruth, Prediction]] 
A list of GroundTruths or Predictions to draw on the image.
 required img Image 
The image to draw the detections on.
 required 
Returns:
 Name Type Description img Image 
An image with the detections drawn on.
 Source code in valor/viz.py 
def draw_detections_on_image(\n    detections: Sequence[Union[GroundTruth, Prediction]],\n    img: Image.Image,\n) -> Image.Image:\n    \"\"\"\n    Draws detections (bounding boxes and labels) on an image.\n    Parameters\n    -------\n    detections : List[Union[GroundTruth, Prediction]]\n        A list of `GroundTruths` or `Predictions` to draw on the image.\n    img : Image.Image\n        The image to draw the detections on.\n    Returns\n    -------\n    img : Image.Image\n        An image with the detections drawn on.\n    \"\"\"\n\n    annotations = []\n    for datum in detections:\n        annotations.extend(datum.annotations)\n\n    for i, detection in enumerate(annotations):\n        if detection.raster and detection.is_instance is True:\n            img = _draw_detection_on_image(detection, img, inplace=i != 0)\n    return img\n
"},{"location":"client_api/Viz/#valor.viz.draw_raster_on_image","title":"valor.viz.draw_raster_on_image(raster, img, color=(255, 0, 0), alpha=0.4)","text":"
Draws the raster on top of an image. This operation is not done in place.
 
Parameters:
 Name Type Description Default img Image 
pillow image to draw on.
 required color Tuple[int, int, int] 
RGB tuple of the color to use
 (255, 0, 0) alpha float 
alpha (transparency) value of the mask. 0 is fully transparent, 1 is fully opaque
 0.4 Source code in valor/viz.py 
def draw_raster_on_image(\n    raster: schemas.Raster,\n    img: Image.Image,\n    color: Tuple[int, int, int] = (255, 0, 0),\n    alpha: float = 0.4,\n) -> Image.Image:\n    \"\"\"Draws the raster on top of an image. This operation is not done in place.\n\n    Parameters\n    ----------\n    img\n        pillow image to draw on.\n    color\n        RGB tuple of the color to use\n    alpha\n        alpha (transparency) value of the mask. 0 is fully transparent, 1 is fully opaque\n    \"\"\"\n    img = img.copy()\n    binary_mask = raster.array\n    mask_arr = np.zeros(\n        (binary_mask.shape[0], binary_mask.shape[1], 3), dtype=np.uint8\n    )\n    mask_arr[binary_mask] = color\n    mask_img = Image.fromarray(mask_arr)\n\n    if mask_img.size != img.size:\n        raise ValueError(\"Input image and raster must be the same size.\")\n    blend = Image.blend(img, mask_img, alpha=alpha)\n    img.paste(blend, (0, 0), mask=Image.fromarray(binary_mask))\n\n    return img\n
"},{"location":"client_api/Schemas/Filters/","title":"Filters","text":""},{"location":"client_api/Schemas/Filters/#valor.schemas.filters-classes","title":"Classes","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/","title":"EvaluationParameters","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation-classes","title":"Classes","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationParameters","title":"valor.schemas.evaluation.EvaluationParameters  dataclass","text":"
Defines parameters for evaluation methods.\n
 
Attributes\n
 
task_type: TaskType\n    The task type of a given evaluation.\nlabel_map: Optional[List[List[List[str]]]]\n    Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\nmetrics_to_return: List[MetricType], optional\n    The list of metrics to compute, store, and return to the user.\nllm_api_params: Dict[str, str | dict], optional\n    A dictionary of parameters for the LLM API.\nconvert_annotations_to_type: AnnotationType | None = None\n    The type to convert all annotations to.\niou_thresholds_to_compute: List[float], optional\n    A list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP).\niou_thresholds_to_return: List[float], optional\n    A list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`.\nrecall_score_threshold: float, default=0\n    The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\npr_curve_iou_threshold: float, optional\n    The IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5.\npr_curve_max_examples: int\n    The maximum number of datum examples to store when calculating PR curves.\nbleu_weights: list[float], optional\n    The weights to use when calculating BLEU scores.\nrouge_types: list[ROUGEType]\n    A list of rouge types to calculate. Options are ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'], where `rouge1` is unigram-based scoring, `rouge2` is bigram-based scoring, `rougeL` is scoring based on sentences (i.e., splitting on \".\" and ignoring \"\n
 
\"), and rougeLsum is scoring based on splitting the text using \" \".     rouge_use_stemmer: bool         If True, uses Porter stemmer to strip word suffixes.
 Source code in valor/schemas/evaluation.py 
@dataclass\nclass EvaluationParameters:\n    \"\"\"\n    Defines parameters for evaluation methods.\n\n    Attributes\n    ----------\n    task_type: TaskType\n        The task type of a given evaluation.\n    label_map: Optional[List[List[List[str]]]]\n        Optional mapping of individual labels to a grouper label. Useful when you need to evaluate performance using labels that differ across datasets and models.\n    metrics_to_return: List[MetricType], optional\n        The list of metrics to compute, store, and return to the user.\n    llm_api_params: Dict[str, str | dict], optional\n        A dictionary of parameters for the LLM API.\n    convert_annotations_to_type: AnnotationType | None = None\n        The type to convert all annotations to.\n    iou_thresholds_to_compute: List[float], optional\n        A list of floats describing which Intersection over Unions (IoUs) to use when calculating metrics (i.e., mAP).\n    iou_thresholds_to_return: List[float], optional\n        A list of floats describing which Intersection over Union (IoUs) thresholds to calculate a metric for. Must be a subset of `iou_thresholds_to_compute`.\n    recall_score_threshold: float, default=0\n        The confidence score threshold for use when determining whether to count a prediction as a true positive or not while calculating Average Recall.\n    pr_curve_iou_threshold: float, optional\n        The IOU threshold to use when calculating precision-recall curves for object detection tasks. Defaults to 0.5.\n    pr_curve_max_examples: int\n        The maximum number of datum examples to store when calculating PR curves.\n    bleu_weights: list[float], optional\n        The weights to use when calculating BLEU scores.\n    rouge_types: list[ROUGEType]\n        A list of rouge types to calculate. Options are ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'], where `rouge1` is unigram-based scoring, `rouge2` is bigram-based scoring, `rougeL` is scoring based on sentences (i.e., splitting on \".\" and ignoring \"\\n\"), and `rougeLsum` is scoring based on splitting the text using \"\\n\".\n    rouge_use_stemmer: bool\n        If True, uses Porter stemmer to strip word suffixes.\n    \"\"\"\n\n    task_type: TaskType\n    label_map: Optional[List[List[List[str]]]] = None\n    metrics_to_return: Optional[List[MetricType]] = None\n    llm_api_params: Optional[Dict[str, Union[str, dict]]] = None\n\n    convert_annotations_to_type: Optional[AnnotationType] = None\n    iou_thresholds_to_compute: Optional[List[float]] = None\n    iou_thresholds_to_return: Optional[List[float]] = None\n    recall_score_threshold: float = 0\n    pr_curve_iou_threshold: float = 0.5\n    pr_curve_max_examples: int = 1\n    bleu_weights: Optional[List[float]] = None\n    rouge_types: Optional[List[ROUGEType]] = None\n    rouge_use_stemmer: Optional[bool] = None\n
"},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationRequest","title":"valor.schemas.evaluation.EvaluationRequest  dataclass","text":"
An evaluation request.
 
Defines important attributes of the API's EvaluationRequest.
 
Attributes:
 Name Type Description dataset_names List[str] 
The list of datasets we want to evaluate by name.
 model_names List[str] 
The list of models we want to evaluate by name.
 filters dict 
The filter object used to define what the model(s) is evaluating against.
 parameters EvaluationParameters 
Any parameters that are used to modify an evaluation method.
 Source code in valor/schemas/evaluation.py 
@dataclass\nclass EvaluationRequest:\n    \"\"\"\n    An evaluation request.\n\n    Defines important attributes of the API's `EvaluationRequest`.\n\n    Attributes\n    ----------\n    dataset_names : List[str]\n        The list of datasets we want to evaluate by name.\n    model_names : List[str]\n        The list of models we want to evaluate by name.\n    filters : dict\n        The filter object used to define what the model(s) is evaluating against.\n    parameters : EvaluationParameters\n        Any parameters that are used to modify an evaluation method.\n    \"\"\"\n\n    dataset_names: Union[str, List[str]]\n    model_names: Union[str, List[str]]\n    parameters: EvaluationParameters\n    filters: Filter = field(default_factory=Filter)\n\n    def __post_init__(self):\n        if isinstance(self.filters, dict):\n            self.filters = Filter(**self.filters)\n        elif self.filters is None:\n            self.filters = Filter()\n\n        if isinstance(self.parameters, dict):\n            self.parameters = EvaluationParameters(**self.parameters)\n\n    def to_dict(self) -> dict:\n        \"\"\"\n        Converts the request into a JSON-compatible dictionary.\n        \"\"\"\n        return {\n            \"dataset_names\": self.dataset_names,\n            \"model_names\": self.model_names,\n            \"parameters\": asdict(self.parameters),\n            \"filters\": self.filters.to_dict(),\n        }\n
"},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationRequest-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Evaluation/EvaluationParameters/#valor.schemas.evaluation.EvaluationRequest.to_dict","title":"valor.schemas.evaluation.EvaluationRequest.to_dict()","text":"
Converts the request into a JSON-compatible dictionary.
 Source code in valor/schemas/evaluation.py 
def to_dict(self) -> dict:\n    \"\"\"\n    Converts the request into a JSON-compatible dictionary.\n    \"\"\"\n    return {\n        \"dataset_names\": self.dataset_names,\n        \"model_names\": self.model_names,\n        \"parameters\": asdict(self.parameters),\n        \"filters\": self.filters.to_dict(),\n    }\n
"},{"location":"client_api/Schemas/Spatial/Box/","title":"Box","text":"
               Bases: Polygon
 
A Box is a polygon that is constrained to 4 unique points.
 
Note that this does not need to be axis-aligned.
 
Parameters:
 Name Type Description Default value List[List[Tuple[float, float]]] 
An polygon value representing a box.
 required 
Attributes:
 Name Type Description area polygon boundary holes xmin xmax ymin ymax 
Examples:
 
>>> Box([[(0,0), (0,1), (1,1), (1,0), (0,0)]])\n
 
Create a Box using extrema.
 
>>> Box.from_extrema(\n...     xmin=0, xmax=1,\n...     ymin=0, ymax=1,\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class Box(Polygon):\n    \"\"\"\n    A Box is a polygon that is constrained to 4 unique points.\n\n    Note that this does not need to be axis-aligned.\n\n    Parameters\n    ----------\n    value : List[List[Tuple[float, float]]], optional\n        An polygon value representing a box.\n\n    Attributes\n    ----------\n    area\n    polygon\n    boundary\n    holes\n    xmin\n    xmax\n    ymin\n    ymax\n\n    Examples\n    --------\n    >>> Box([[(0,0), (0,1), (1,1), (1,0), (0,0)]])\n\n    Create a Box using extrema.\n    >>> Box.from_extrema(\n    ...     xmin=0, xmax=1,\n    ...     ymin=0, ymax=1,\n    ... )\n    \"\"\"\n\n    def __init__(\n        self, value: typing.List[typing.List[typing.Tuple[float, float]]]\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        \"\"\"\n        Validates typing.\n\n        Parameters\n        ----------\n        value : typing.Any\n            The value to validate.\n\n        Raises\n        ------\n        TypeError\n            If the value type is not supported.\n        \"\"\"\n        Polygon.__validate__(value)\n        if len(value) != 1:\n            raise ValueError(\"Box should not contain holes.\")\n        elif len(value[0]) != 5:\n            raise ValueError(\"Box should consist of four unique points.\")\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return super().decode_value(value)\n\n    @classmethod\n    def from_extrema(\n        cls,\n        xmin: float,\n        xmax: float,\n        ymin: float,\n        ymax: float,\n    ):\n        \"\"\"\n        Create a Box from extrema values.\n\n        Parameters\n        ----------\n        xmin : float\n            Minimum x-coordinate of the bounding box.\n        xmax : float\n            Maximum x-coordinate of the bounding box.\n        ymin : float\n            Minimum y-coordinate of the bounding box.\n        ymax : float\n            Maximum y-coordinate of the bounding box.\n\n        Returns\n        -------\n        Box\n            A Box created from the provided extrema values.\n        \"\"\"\n        points = [\n            [\n                (xmin, ymin),\n                (xmax, ymin),\n                (xmax, ymax),\n                (xmin, ymax),\n                (xmin, ymin),\n            ]\n        ]\n        return cls(value=points)\n\n    def to_polygon(self) -> Polygon:\n        \"\"\"\n        Converts box to a generic polygon.\n\n        Returns\n        -------\n        Polygon\n            The box as a Polygon.\n        \"\"\"\n        return Polygon(self.get_value())\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.__validate__","title":"valor.schemas.Box.__validate__(value)  classmethod","text":"
Validates typing.
 
Parameters:
 Name Type Description Default value Any 
The value to validate.
 required 
Raises:
 Type Description TypeError 
If the value type is not supported.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef __validate__(cls, value: typing.Any):\n    \"\"\"\n    Validates typing.\n\n    Parameters\n    ----------\n    value : typing.Any\n        The value to validate.\n\n    Raises\n    ------\n    TypeError\n        If the value type is not supported.\n    \"\"\"\n    Polygon.__validate__(value)\n    if len(value) != 1:\n        raise ValueError(\"Box should not contain holes.\")\n    elif len(value[0]) != 5:\n        raise ValueError(\"Box should consist of four unique points.\")\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.decode_value","title":"valor.schemas.Box.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return super().decode_value(value)\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.from_extrema","title":"valor.schemas.Box.from_extrema(xmin, xmax, ymin, ymax)  classmethod","text":"
Create a Box from extrema values.
 
Parameters:
 Name Type Description Default xmin float 
Minimum x-coordinate of the bounding box.
 required xmax float 
Maximum x-coordinate of the bounding box.
 required ymin float 
Minimum y-coordinate of the bounding box.
 required ymax float 
Maximum y-coordinate of the bounding box.
 required 
Returns:
 Type Description Box 
A Box created from the provided extrema values.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_extrema(\n    cls,\n    xmin: float,\n    xmax: float,\n    ymin: float,\n    ymax: float,\n):\n    \"\"\"\n    Create a Box from extrema values.\n\n    Parameters\n    ----------\n    xmin : float\n        Minimum x-coordinate of the bounding box.\n    xmax : float\n        Maximum x-coordinate of the bounding box.\n    ymin : float\n        Minimum y-coordinate of the bounding box.\n    ymax : float\n        Maximum y-coordinate of the bounding box.\n\n    Returns\n    -------\n    Box\n        A Box created from the provided extrema values.\n    \"\"\"\n    points = [\n        [\n            (xmin, ymin),\n            (xmax, ymin),\n            (xmax, ymax),\n            (xmin, ymax),\n            (xmin, ymin),\n        ]\n    ]\n    return cls(value=points)\n
"},{"location":"client_api/Schemas/Spatial/Box/#valor.schemas.Box.to_polygon","title":"valor.schemas.Box.to_polygon()","text":"
Converts box to a generic polygon.
 
Returns:
 Type Description Polygon 
The box as a Polygon.
 Source code in valor/schemas/symbolic/types.py 
def to_polygon(self) -> Polygon:\n    \"\"\"\n    Converts box to a generic polygon.\n\n    Returns\n    -------\n    Polygon\n        The box as a Polygon.\n    \"\"\"\n    return Polygon(self.get_value())\n
"},{"location":"client_api/Schemas/Spatial/LineString/","title":"LineString","text":"
               Bases: Spatial
 
Represents a line.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[Tuple[float, float]] 
A linestring.
 required 
Methods:
 Name Description colorspace 
Represent the photo in the given colorspace.
 gamma 
Change the photo's gamma exposure.
 
Examples:
 
Create a line.
 
>>> LineString([(0,0), (0,1), (1,1)])\n
 Source code in valor/schemas/symbolic/types.py 
class LineString(Spatial):\n    \"\"\"\n    Represents a line.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[Tuple[float, float]], optional\n        A linestring.\n\n    Methods\n    -------\n    colorspace(c='rgb')\n        Represent the photo in the given colorspace.\n    gamma(n=1.0)\n        Change the photo's gamma exposure.\n\n    Examples\n    --------\n    Create a line.\n    >>> LineString([(0,0), (0,1), (1,1)])\n    \"\"\"\n\n    def __init__(self, value: typing.List[typing.Tuple[float, float]]):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        MultiPoint.__validate__(value)\n        if len(value) < 2:\n            raise ValueError(\n                \"At least two points are required to make a line.\"\n            )\n\n    @classmethod\n    def decode_value(\n        cls, value: typing.Optional[typing.List[typing.List[float]]]\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/LineString/#valor.schemas.LineString-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/LineString/#valor.schemas.LineString.decode_value","title":"valor.schemas.LineString.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls, value: typing.Optional[typing.List[typing.List[float]]]\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/MultiLineString/","title":"MultiLineString","text":"
               Bases: Spatial
 
Represents a list of lines.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[List[Tuple[float, float]]] 
A multilinestring.
 required 
Examples:
 
Create a single line.
 
>>> MultiLineString([[(0,0), (0,1), (1,1), (0,0)]])\n
 
Create 3 lines.
 
>>> MultiLineString(\n...     [\n...         [(0,0), (0,1), (1,1)],\n...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2)],\n...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7)],\n...     ]\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class MultiLineString(Spatial):\n    \"\"\"\n    Represents a list of lines.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[List[Tuple[float, float]]], optional\n        A multilinestring.\n\n    Examples\n    --------\n    Create a single line.\n    >>> MultiLineString([[(0,0), (0,1), (1,1), (0,0)]])\n\n    Create 3 lines.\n    >>> MultiLineString(\n    ...     [\n    ...         [(0,0), (0,1), (1,1)],\n    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2)],\n    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7)],\n    ...     ]\n    ... )\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.List[typing.List[typing.Tuple[float, float]]],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, list):\n            raise TypeError(\n                f\"Expected type 'List[List[Tuple[float, float]]]' received type '{type(value).__name__}'\"\n            )\n        for line in value:\n            LineString.__validate__(line)\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls(\n            [[(point[0], point[1]) for point in line] for line in value]\n        )\n
"},{"location":"client_api/Schemas/Spatial/MultiLineString/#valor.schemas.MultiLineString-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/MultiLineString/#valor.schemas.MultiLineString.decode_value","title":"valor.schemas.MultiLineString.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls(\n        [[(point[0], point[1]) for point in line] for line in value]\n    )\n
"},{"location":"client_api/Schemas/Spatial/MultiPoint/","title":"MultiPoint","text":"
               Bases: Spatial
 
Represents a list of points.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[Tuple[float, float]] 
A multipoint.
 required 
Examples:
 
>>> MultiPoint([(0,0), (0,1), (1,1)])\n
 Source code in valor/schemas/symbolic/types.py 
class MultiPoint(Spatial):\n    \"\"\"\n    Represents a list of points.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[Tuple[float, float]], optional\n        A multipoint.\n\n    Examples\n    --------\n    >>> MultiPoint([(0,0), (0,1), (1,1)])\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.List[typing.Tuple[float, float]],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, list):\n            raise TypeError(\n                f\"Expected 'typing.List[typing.Tuple[float, float]]' received type '{type(value).__name__}'\"\n            )\n        for point in value:\n            Point.__validate__(point)\n\n    @classmethod\n    def decode_value(\n        cls, value: typing.Optional[typing.List[typing.List[float]]]\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/MultiPoint/#valor.schemas.MultiPoint-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/MultiPoint/#valor.schemas.MultiPoint.decode_value","title":"valor.schemas.MultiPoint.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls, value: typing.Optional[typing.List[typing.List[float]]]\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls([(point[0], point[1]) for point in value])\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/","title":"MultiPolygon","text":"
               Bases: Spatial
 
Represents a collection of polygons.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[List[List[Tuple[float, float]]]] 
A list of polygons.
 required 
Attributes:
 Name Type Description area polygons 
Examples:
 
>>> MultiPolygon(\n...     [\n...         [\n...             [(0,0), (0,1), (1,1), (0,0)]\n...         ],\n...         [\n...             [(0,0), (0,1), (1,1), (0,0)],\n...             [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n...             [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n...         ],\n...     ]\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class MultiPolygon(Spatial):\n    \"\"\"\n    Represents a collection of polygons.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[List[List[Tuple[float, float]]]], optional\n        A list of polygons.\n\n    Attributes\n    ----------\n    area\n    polygons\n\n    Examples\n    --------\n    >>> MultiPolygon(\n    ...     [\n    ...         [\n    ...             [(0,0), (0,1), (1,1), (0,0)]\n    ...         ],\n    ...         [\n    ...             [(0,0), (0,1), (1,1), (0,0)],\n    ...             [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n    ...             [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n    ...         ],\n    ...     ]\n    ... )\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.List[\n            typing.List[typing.List[typing.Tuple[float, float]]]\n        ],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, list):\n            raise TypeError(\n                f\"Expected type 'List[List[List[Tuple[float, float]]]]' received type '{type(value).__name__}'\"\n            )\n        for poly in value:\n            Polygon.__validate__(poly)\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[\n            typing.List[typing.List[typing.List[typing.List[float]]]]\n        ],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls(\n            [\n                [\n                    [(point[0], point[1]) for point in subpolygon]\n                    for subpolygon in polygon\n                ]\n                for polygon in value\n            ]\n        )\n\n    @property\n    def area(self):\n        \"\"\"\n        Symbolic representation of area.\n        \"\"\"\n        if not isinstance(self._value, Symbol):\n            raise ValueError(\n                \"attribute 'area' is reserved for symbolic variables.\"\n            )\n        return Float.symbolic(\n            name=self._value._name,\n            key=self._value._key,\n            attribute=\"area\",\n        )\n\n    def to_polygons(self) -> typing.List[Polygon]:\n        \"\"\"\n        Converts multipolygon to a list of Polygon instances.\n\n        Returns\n        -------\n        List[Polygon]\n        \"\"\"\n        return [Polygon(poly) for poly in self.get_value()]\n\n    @classmethod\n    def from_polygons(self, polygons: typing.List[Polygon]) -> \"MultiPolygon\":\n        \"\"\"\n        Converts a list of Polygon instances to a MultiPolygon.\n\n        Parameters\n        ----------\n        polygons : List[Polygon]\n            A list of Polygon instances.\n\n        Returns\n        -------\n        MultiPolygon\n            A MultiPolygon instance.\n        \"\"\"\n        return MultiPolygon([poly.get_value() for poly in polygons])\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon-attributes","title":"Attributes","text":""},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.area","title":"valor.schemas.MultiPolygon.area  property","text":"
Symbolic representation of area.
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.decode_value","title":"valor.schemas.MultiPolygon.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[\n        typing.List[typing.List[typing.List[typing.List[float]]]]\n    ],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls(\n        [\n            [\n                [(point[0], point[1]) for point in subpolygon]\n                for subpolygon in polygon\n            ]\n            for polygon in value\n        ]\n    )\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.from_polygons","title":"valor.schemas.MultiPolygon.from_polygons(polygons)  classmethod","text":"
Converts a list of Polygon instances to a MultiPolygon.
 
Parameters:
 Name Type Description Default polygons List[Polygon] 
A list of Polygon instances.
 required 
Returns:
 Type Description MultiPolygon 
A MultiPolygon instance.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_polygons(self, polygons: typing.List[Polygon]) -> \"MultiPolygon\":\n    \"\"\"\n    Converts a list of Polygon instances to a MultiPolygon.\n\n    Parameters\n    ----------\n    polygons : List[Polygon]\n        A list of Polygon instances.\n\n    Returns\n    -------\n    MultiPolygon\n        A MultiPolygon instance.\n    \"\"\"\n    return MultiPolygon([poly.get_value() for poly in polygons])\n
"},{"location":"client_api/Schemas/Spatial/MultiPolygon/#valor.schemas.MultiPolygon.to_polygons","title":"valor.schemas.MultiPolygon.to_polygons()","text":"
Converts multipolygon to a list of Polygon instances.
 
Returns:
 Type Description List[Polygon] Source code in valor/schemas/symbolic/types.py 
def to_polygons(self) -> typing.List[Polygon]:\n    \"\"\"\n    Converts multipolygon to a list of Polygon instances.\n\n    Returns\n    -------\n    List[Polygon]\n    \"\"\"\n    return [Polygon(poly) for poly in self.get_value()]\n
"},{"location":"client_api/Schemas/Spatial/Point/","title":"Point","text":"
               Bases: Spatial, Equatable
 
Represents a point in 2D space.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value Tuple[float, float] 
A point.
 required 
Examples:
 
>>> Point((1,2))\n
 Source code in valor/schemas/symbolic/types.py 
class Point(Spatial, Equatable):\n    \"\"\"\n    Represents a point in 2D space.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : Tuple[float, float], optional\n        A point.\n\n    Examples\n    --------\n    >>> Point((1,2))\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.Tuple[float, float],\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        if not isinstance(value, (tuple, list)):\n            raise TypeError(\n                f\"Expected type 'typing.Tuple[float, float]' received type '{type(value).__name__}'\"\n            )\n        elif len(value) != 2:\n            raise ValueError(\n                \"A point should contain only two x-y coordinates.\"\n            )\n        for item in value:\n            if not isinstance(item, (int, float, np.floating)):\n                raise TypeError(\n                    f\"Expected type '{float.__name__}' received type '{type(item).__name__}'\"\n                )\n\n    @classmethod\n    def decode_value(cls, value: typing.Optional[typing.List[float]]):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls((value[0], value[1]))\n\n    def encode_value(self) -> typing.Any:\n        \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n        value = self.get_value()\n        if value is None:\n            return None\n        return (float(value[0]), float(value[1]))\n\n    def tuple(self):\n        return self.get_value()\n\n    def resize(\n        self,\n        og_img_h=10,\n        og_img_w=10,\n        new_img_h=100,\n        new_img_w=100,\n    ):\n        value = self.get_value()\n        h_ratio = new_img_h / og_img_h\n        w_ratio = new_img_w / og_img_w\n        return Point((value[0] * h_ratio, value[1] * w_ratio))\n\n    @property\n    def x(self):\n        return self.get_value()[0]\n\n    @property\n    def y(self):\n        return self.get_value()[1]\n
"},{"location":"client_api/Schemas/Spatial/Point/#valor.schemas.Point-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Point/#valor.schemas.Point.decode_value","title":"valor.schemas.Point.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(cls, value: typing.Optional[typing.List[float]]):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls((value[0], value[1]))\n
"},{"location":"client_api/Schemas/Spatial/Point/#valor.schemas.Point.encode_value","title":"valor.schemas.Point.encode_value()","text":"
Encode object to JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
def encode_value(self) -> typing.Any:\n    \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n    value = self.get_value()\n    if value is None:\n        return None\n    return (float(value[0]), float(value[1]))\n
"},{"location":"client_api/Schemas/Spatial/Polygon/","title":"Polygon","text":"
               Bases: Spatial
 
Represents a polygon with a boundary and optional holes.
 
Follows the GeoJSON specification (RFC 7946).
 
Parameters:
 Name Type Description Default value List[List[Tuple[float, float]]] 
A polygon.
 required 
Attributes:
 Name Type Description area Float boundary List[Tuple[float, float]] holes List[List[Tuple[float, float]]] xmin float xmax float ymin float ymax float 
Examples:
 
Create a polygon without any holes.
 
>>> Polygon([[(0,0), (0,1), (1,1), (0,0)]])\n
 
Create a polygon with 2 holes.
 
>>> Polygon(\n...     [\n...         [(0,0), (0,1), (1,1), (0,0)],\n...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n...     ]\n... )\n
 Source code in valor/schemas/symbolic/types.py 
class Polygon(Spatial):\n    \"\"\"\n    Represents a polygon with a boundary and optional holes.\n\n    Follows the GeoJSON specification (RFC 7946).\n\n    Parameters\n    ----------\n    value : List[List[Tuple[float, float]]], optional\n        A polygon.\n\n    Attributes\n    ----------\n    area\n    boundary\n    holes\n    xmin\n    xmax\n    ymin\n    ymax\n\n    Examples\n    --------\n    Create a polygon without any holes.\n    >>> Polygon([[(0,0), (0,1), (1,1), (0,0)]])\n\n    Create a polygon with 2 holes.\n    >>> Polygon(\n    ...     [\n    ...         [(0,0), (0,1), (1,1), (0,0)],\n    ...         [(0.1, 0.1), (0.1, 0.2), (0.2, 0.2), (0.1, 0.1)],\n    ...         [(0.6, 0.6), (0.6, 0.7), (0.7, 0.7), (0.6, 0.6)],\n    ...     ]\n    ... )\n    \"\"\"\n\n    def __init__(\n        self, value: typing.List[typing.List[typing.Tuple[float, float]]]\n    ):\n        super().__init__(value=value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        MultiLineString.__validate__(value)\n        for line in value:\n            if not (len(line) >= 4 and line[0] == line[-1]):\n                raise ValueError(\n                    \"Polygons are defined by at least 4 points with the first point being repeated at the end.\"\n                )\n\n    @classmethod\n    def decode_value(\n        cls,\n        value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n    ):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        return cls(\n            [\n                [(point[0], point[1]) for point in subpolygon]\n                for subpolygon in value\n            ]\n        )\n\n    @property\n    def area(self) -> Float:\n        \"\"\"\n        Symbolic representation of area.\n        \"\"\"\n        if not isinstance(self._value, Symbol):\n            raise ValueError\n        return Float.symbolic(\n            name=self._value._name,\n            key=self._value._key,\n            attribute=\"area\",\n        )\n\n    @property\n    def boundary(self) -> typing.List[typing.Tuple[float, float]]:\n        \"\"\"\n        The boundary of the polygon.\n\n        Returns\n        -------\n        List[Tuple(float, float)]\n            A list of points.\n        \"\"\"\n        value = self.get_value()\n        if value is None:\n            raise ValueError(\"Polygon is 'None'\")\n        return value[0]\n\n    @property\n    def holes(self) -> typing.List[typing.List[typing.Tuple[float, float]]]:\n        \"\"\"\n        typing.Any holes in the polygon.\n\n        Returns\n        -------\n        List[List[Tuple(float, float)]]\n            A list of holes.\n        \"\"\"\n        value = self.get_value()\n        if value is None:\n            raise ValueError(\"Polygon is 'None'\")\n        return value[1:]\n\n    @property\n    def xmin(self) -> float:\n        \"\"\"\n        Minimum x-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return min([p[0] for p in self.boundary])\n\n    @property\n    def xmax(self) -> float:\n        \"\"\"\n        Maximum x-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return max([p[0] for p in self.boundary])\n\n    @property\n    def ymin(self) -> float:\n        \"\"\"\n        Minimum y-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return min([p[1] for p in self.boundary])\n\n    @property\n    def ymax(self) -> float:\n        \"\"\"\n        Maximum y-value.\n\n        Returns\n        -------\n        float\n        \"\"\"\n        return max([p[1] for p in self.boundary])\n
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon-attributes","title":"Attributes","text":""},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.area","title":"valor.schemas.Polygon.area: Float  property","text":"
Symbolic representation of area.
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.boundary","title":"valor.schemas.Polygon.boundary: typing.List[typing.Tuple[float, float]]  property","text":"
The boundary of the polygon.
 
Returns:
 Type Description List[Tuple(float, float)] 
A list of points.
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.holes","title":"valor.schemas.Polygon.holes: typing.List[typing.List[typing.Tuple[float, float]]]  property","text":"
typing.Any holes in the polygon.
 
Returns:
 Type Description List[List[Tuple(float, float)]] 
A list of holes.
"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.xmax","title":"valor.schemas.Polygon.xmax: float  property","text":"
Maximum x-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.xmin","title":"valor.schemas.Polygon.xmin: float  property","text":"
Minimum x-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.ymax","title":"valor.schemas.Polygon.ymax: float  property","text":"
Maximum y-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.ymin","title":"valor.schemas.Polygon.ymin: float  property","text":"
Minimum y-value.
 
Returns:
 Type Description float"},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Polygon/#valor.schemas.Polygon.decode_value","title":"valor.schemas.Polygon.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(\n    cls,\n    value: typing.Optional[typing.List[typing.List[typing.List[float]]]],\n):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    return cls(\n        [\n            [(point[0], point[1]) for point in subpolygon]\n            for subpolygon in value\n        ]\n    )\n
"},{"location":"client_api/Schemas/Spatial/Raster/","title":"Raster","text":"
               Bases: Spatial
 
Represents a binary mask.
 
Parameters:
 Name Type Description Default value Dict[str, Union[ndarray, str, None]] 
An raster value.
 required 
Attributes:
 Name Type Description area Float array ndarray geometry Union[Box, Polygon, MultiPolygon] height int width int 
Raises:
 Type Description TypeError 
If encoding is not a string.
 
Examples:
 
Generate a random mask.
 
>>> import numpy.random\n>>> height = 640\n>>> width = 480\n>>> array = numpy.random.rand(height, width)\n
 
Convert to binary mask.
 
>>> mask = (array > 0.5)\n
 
Create Raster.
 
>>> Raster.from_numpy(mask)\n
 Source code in valor/schemas/symbolic/types.py 
class Raster(Spatial):\n    \"\"\"\n    Represents a binary mask.\n\n    Parameters\n    ----------\n    value : Dict[str, typing.Union[np.ndarray, str, None]], optional\n        An raster value.\n\n    Attributes\n    ----------\n    area\n    array\n    geometry\n    height\n    width\n\n    Raises\n    ------\n    TypeError\n        If `encoding` is not a string.\n\n    Examples\n    --------\n    Generate a random mask.\n    >>> import numpy.random\n    >>> height = 640\n    >>> width = 480\n    >>> array = numpy.random.rand(height, width)\n\n    Convert to binary mask.\n    >>> mask = (array > 0.5)\n\n    Create Raster.\n    >>> Raster.from_numpy(mask)\n    \"\"\"\n\n    def __init__(\n        self,\n        value: typing.Dict[\n            str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]\n        ],\n    ):\n        \"\"\"\n        Initialize and instance of a raster.\n\n        Parameters\n        ----------\n        value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]\n            The raster in dictionary format {\"mask\": <np.ndarray>, \"geometry\": <geometry | None>}.\n        \"\"\"\n        super().__init__(value)\n\n    @classmethod\n    def __validate__(cls, value: typing.Any):\n        \"\"\"\n        Validates typing.\n\n        Parameters\n        ----------\n        value : Any\n            The value to validate.\n\n        Raises\n        ------\n        TypeError\n            If the value type is not supported.\n        \"\"\"\n        if not isinstance(value, dict):\n            raise TypeError(\n                \"Raster should contain a dictionary describing a mask and optionally a geometry.\"\n            )\n        elif set(value.keys()) != {\"mask\", \"geometry\"}:\n            raise ValueError(\n                \"Raster should be described by a dictionary with keys 'mask' and 'geometry'\"\n            )\n        elif not isinstance(value[\"mask\"], np.ndarray):\n            raise TypeError(\n                f\"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'\"\n            )\n        elif len(value[\"mask\"].shape) != 2:\n            raise ValueError(\"raster only supports 2d arrays\")\n        elif value[\"mask\"].dtype != bool:\n            raise ValueError(\n                f\"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}\"\n            )\n        elif (\n            value[\"geometry\"] is not None\n            and not Polygon.supports(value[\"geometry\"])\n            and not MultiPolygon.supports(value[\"geometry\"])\n        ):\n            raise TypeError(\n                \"Expected geometry to conform to either Polygon or MultiPolygon or be 'None'\"\n            )\n\n    def encode_value(self) -> typing.Any:\n        \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n        value = self.get_value()\n        if value is None:\n            return None\n        f = io.BytesIO()\n        PIL.Image.fromarray(value[\"mask\"]).save(f, format=\"PNG\")\n        f.seek(0)\n        mask_bytes = f.read()\n        f.close()\n        return {\n            \"mask\": b64encode(mask_bytes).decode(),\n            \"geometry\": value[\"geometry\"],\n        }\n\n    @classmethod\n    def decode_value(cls, value: typing.Any):\n        \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n        if value is None:\n            return None\n        if not (\n            isinstance(value, dict)\n            and set(value.keys()) == {\"mask\", \"geometry\"}\n        ):\n            raise ValueError(\n                f\"Improperly formatted raster encoding. Received '{value}'\"\n            )\n        mask_bytes = b64decode(value[\"mask\"])\n        with io.BytesIO(mask_bytes) as f:\n            img = PIL.Image.open(f)\n            value = {\n                \"mask\": np.array(img),\n                \"geometry\": value[\"geometry\"],\n            }\n        return cls(value=value)\n\n    @classmethod\n    def from_numpy(cls, mask: np.ndarray):\n        \"\"\"\n        Create a Raster object from a NumPy array.\n\n        Parameters\n        ----------\n        mask : np.ndarray\n            The 2D binary array representing the mask.\n\n        Returns\n        -------\n        Raster\n\n        Raises\n        ------\n        ValueError\n            If the input array is not 2D or not of dtype bool.\n        \"\"\"\n        return cls(value={\"mask\": mask, \"geometry\": None})\n\n    @classmethod\n    def from_geometry(\n        cls,\n        geometry: typing.Union[Box, Polygon, MultiPolygon],\n        height: int,\n        width: int,\n    ):\n        \"\"\"\n        Create a Raster object from a geometric mask.\n\n        Parameters\n        ----------\n        geometry : Union[Box, Polygon, MultiPolygon]\n            Defines the bitmask as a geometry. Overrides any existing mask.\n        height : int\n            The intended height of the binary mask.\n        width : int\n            The intended width of the binary mask.\n\n        Returns\n        -------\n        Raster\n        \"\"\"\n        bitmask = np.full((int(height), int(width)), False)\n        return cls(value={\"mask\": bitmask, \"geometry\": geometry.get_value()})\n\n    @property\n    def area(self) -> Float:\n        \"\"\"\n        Symbolic representation of area.\n        \"\"\"\n        if not isinstance(self._value, Symbol):\n            raise ValueError\n        return Float.symbolic(\n            name=self._value._name,\n            key=self._value._key,\n            attribute=\"area\",\n        )\n\n    @property\n    def array(self) -> np.ndarray:\n        \"\"\"\n        The bitmask as a numpy array.\n\n        Returns\n        -------\n        Optional[np.ndarray]\n            A 2D binary array representing the mask if it exists.\n        \"\"\"\n        value = self.get_value()\n        if value[\"geometry\"] is not None:\n            warnings.warn(\n                \"Raster array does not contain bitmask as this is a geometry-defined raster.\",\n                RuntimeWarning,\n            )\n        return value[\"mask\"]\n\n    @property\n    def geometry(self) -> typing.Union[Box, Polygon, MultiPolygon]:\n        \"\"\"\n        The geometric mask if it exists.\n\n        Returns\n        -------\n        Box | Polygon | MultiPolygon | None\n            The geometry if it exists.\n        \"\"\"\n        return self.get_value()[\"geometry\"]\n\n    @property\n    def height(self) -> int:\n        \"\"\"Returns the height of the raster if it exists.\"\"\"\n        return self.array.shape[0]\n\n    @property\n    def width(self) -> int:\n        \"\"\"Returns the width of the raster if it exists.\"\"\"\n        return self.array.shape[1]\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster-attributes","title":"Attributes","text":""},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.area","title":"valor.schemas.Raster.area: Float  property","text":"
Symbolic representation of area.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.array","title":"valor.schemas.Raster.array: np.ndarray  property","text":"
The bitmask as a numpy array.
 
Returns:
 Type Description Optional[ndarray] 
A 2D binary array representing the mask if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.geometry","title":"valor.schemas.Raster.geometry: typing.Union[Box, Polygon, MultiPolygon]  property","text":"
The geometric mask if it exists.
 
Returns:
 Type Description Box | Polygon | MultiPolygon | None 
The geometry if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.height","title":"valor.schemas.Raster.height: int  property","text":"
Returns the height of the raster if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.width","title":"valor.schemas.Raster.width: int  property","text":"
Returns the width of the raster if it exists.
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster-functions","title":"Functions","text":""},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.__init__","title":"valor.schemas.Raster.__init__(value)","text":"
Initialize and instance of a raster.
 
Parameters:
 Name Type Description Default value Dict[str, Union[ndarray, Box, Polygon, MultiPolygon, None]] 
The raster in dictionary format {\"mask\": , \"geometry\": }. required Source code in valor/schemas/symbolic/types.py 
def __init__(\n    self,\n    value: typing.Dict[\n        str, typing.Union[np.ndarray, Box, Polygon, MultiPolygon, None]\n    ],\n):\n    \"\"\"\n    Initialize and instance of a raster.\n\n    Parameters\n    ----------\n    value : Dict[str, Union[np.ndarray, Box, Polygon, MultiPolygon, None]]\n        The raster in dictionary format {\"mask\": <np.ndarray>, \"geometry\": <geometry | None>}.\n    \"\"\"\n    super().__init__(value)\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.__validate__","title":"valor.schemas.Raster.__validate__(value)  classmethod","text":"
Validates typing.
 
Parameters:
 Name Type Description Default value Any 
The value to validate.
 required 
Raises:
 Type Description TypeError 
If the value type is not supported.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef __validate__(cls, value: typing.Any):\n    \"\"\"\n    Validates typing.\n\n    Parameters\n    ----------\n    value : Any\n        The value to validate.\n\n    Raises\n    ------\n    TypeError\n        If the value type is not supported.\n    \"\"\"\n    if not isinstance(value, dict):\n        raise TypeError(\n            \"Raster should contain a dictionary describing a mask and optionally a geometry.\"\n        )\n    elif set(value.keys()) != {\"mask\", \"geometry\"}:\n        raise ValueError(\n            \"Raster should be described by a dictionary with keys 'mask' and 'geometry'\"\n        )\n    elif not isinstance(value[\"mask\"], np.ndarray):\n        raise TypeError(\n            f\"Expected mask to have type '{np.ndarray}' receieved type '{value['mask']}'\"\n        )\n    elif len(value[\"mask\"].shape) != 2:\n        raise ValueError(\"raster only supports 2d arrays\")\n    elif value[\"mask\"].dtype != bool:\n        raise ValueError(\n            f\"Expecting a binary mask (i.e. of dtype bool) but got dtype {value['mask'].dtype}\"\n        )\n    elif (\n        value[\"geometry\"] is not None\n        and not Polygon.supports(value[\"geometry\"])\n        and not MultiPolygon.supports(value[\"geometry\"])\n    ):\n        raise TypeError(\n            \"Expected geometry to conform to either Polygon or MultiPolygon or be 'None'\"\n        )\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.decode_value","title":"valor.schemas.Raster.decode_value(value)  classmethod","text":"
Decode object from JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef decode_value(cls, value: typing.Any):\n    \"\"\"Decode object from JSON compatible dictionary.\"\"\"\n    if value is None:\n        return None\n    if not (\n        isinstance(value, dict)\n        and set(value.keys()) == {\"mask\", \"geometry\"}\n    ):\n        raise ValueError(\n            f\"Improperly formatted raster encoding. Received '{value}'\"\n        )\n    mask_bytes = b64decode(value[\"mask\"])\n    with io.BytesIO(mask_bytes) as f:\n        img = PIL.Image.open(f)\n        value = {\n            \"mask\": np.array(img),\n            \"geometry\": value[\"geometry\"],\n        }\n    return cls(value=value)\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.encode_value","title":"valor.schemas.Raster.encode_value()","text":"
Encode object to JSON compatible dictionary.
 Source code in valor/schemas/symbolic/types.py 
def encode_value(self) -> typing.Any:\n    \"\"\"Encode object to JSON compatible dictionary.\"\"\"\n    value = self.get_value()\n    if value is None:\n        return None\n    f = io.BytesIO()\n    PIL.Image.fromarray(value[\"mask\"]).save(f, format=\"PNG\")\n    f.seek(0)\n    mask_bytes = f.read()\n    f.close()\n    return {\n        \"mask\": b64encode(mask_bytes).decode(),\n        \"geometry\": value[\"geometry\"],\n    }\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.from_geometry","title":"valor.schemas.Raster.from_geometry(geometry, height, width)  classmethod","text":"
Create a Raster object from a geometric mask.
 
Parameters:
 Name Type Description Default geometry Union[Box, Polygon, MultiPolygon] 
Defines the bitmask as a geometry. Overrides any existing mask.
 required height int 
The intended height of the binary mask.
 required width int 
The intended width of the binary mask.
 required 
Returns:
 Type Description Raster Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_geometry(\n    cls,\n    geometry: typing.Union[Box, Polygon, MultiPolygon],\n    height: int,\n    width: int,\n):\n    \"\"\"\n    Create a Raster object from a geometric mask.\n\n    Parameters\n    ----------\n    geometry : Union[Box, Polygon, MultiPolygon]\n        Defines the bitmask as a geometry. Overrides any existing mask.\n    height : int\n        The intended height of the binary mask.\n    width : int\n        The intended width of the binary mask.\n\n    Returns\n    -------\n    Raster\n    \"\"\"\n    bitmask = np.full((int(height), int(width)), False)\n    return cls(value={\"mask\": bitmask, \"geometry\": geometry.get_value()})\n
"},{"location":"client_api/Schemas/Spatial/Raster/#valor.schemas.Raster.from_numpy","title":"valor.schemas.Raster.from_numpy(mask)  classmethod","text":"
Create a Raster object from a NumPy array.
 
Parameters:
 Name Type Description Default mask ndarray 
The 2D binary array representing the mask.
 required 
Returns:
 Type Description Raster 
Raises:
 Type Description ValueError 
If the input array is not 2D or not of dtype bool.
 Source code in valor/schemas/symbolic/types.py 
@classmethod\ndef from_numpy(cls, mask: np.ndarray):\n    \"\"\"\n    Create a Raster object from a NumPy array.\n\n    Parameters\n    ----------\n    mask : np.ndarray\n        The 2D binary array representing the mask.\n\n    Returns\n    -------\n    Raster\n\n    Raises\n    ------\n    ValueError\n        If the input array is not 2D or not of dtype bool.\n    \"\"\"\n    return cls(value={\"mask\": mask, \"geometry\": None})\n
"}]}
\ No newline at end of file