Add HasUid and apply it to Benchmark and Hazard. (#386)

* Add HasUid and apply it to Benchmark and Hazard.

* Pleasing the formatting gods.

src/modelbench/benchmarks.py

@@ -11,6 +11,7 @@
 )
 from modelbench.modelgauge_runner import ModelGaugeSut
 from modelbench.scoring import LetterGradeMixin
+from modelbench.uid import HasUid
 
 
 class BenchmarkScore(ABC, LetterGradeMixin):
@@ -43,8 +44,9 @@ def __repr__(self):
         )
 
 
-class BenchmarkDefinition(ABC):
+class BenchmarkDefinition(ABC, HasUid):
     _hazards: list[HazardDefinition]
+    _uid_definition = {"class": "self", "version": "0.5"}
 
     def __init__(self, hazards: List[HazardDefinition]):
         super().__init__()

src/modelbench/hazards.py

@@ -12,9 +12,10 @@
 from pydantic import BaseModel, ConfigDict
 
 from modelbench.scoring import BAND_CONSTANTS, LetterGradeMixin, NumericGradeMixin, ValueEstimate
+from modelbench.uid import HasUid
 
 
-class HazardDefinition(ABC):
+class HazardDefinition(ABC, HasUid):
 
     @classmethod
     def name(cls):
@@ -41,6 +42,8 @@ def __eq__(self, __value):
     def __hash__(self):
         return hash(self.__class__)
 
+    _uid_definition = {"class": key, "version": "0.5"}
+
 
 class SafeHazard(HazardDefinition, ABC):
 

src/modelbench/uid.py

@@ -0,0 +1,64 @@
+import re
+
+import casefy
+
+
+class HasUid:
+    """
+    A mixin class that gives an object an AISafety UID.
+
+    Add it to your object's parent class list and then add a _uid_definition
+    class variable that specifies your UID.
+
+        class MySimpleObject(ABC, HasUid):
+            _uid_definition = {"name": "simple", "version": "0.5"}
+
+    That will result in a uid of "simple-0.5".
+
+    Your UID values can include literals, function references, or class references,
+    all of which will get rendered automatically. Due to the specifics of python,
+    you can't refer to a function or object before it exists, so make sure the
+    UID definition is after the reference. For example:
+
+        class MyDynamicObject(ABC, HasUid):
+            def name(self):
+                return "bob"
+            _uid_definition = {"name": name, "version": "0.5"}
+
+    Then calling MyDynamicObject().uid will return "bob-0.5".
+
+    If you'd like to refer to the class currently being defined, you'll need to
+    use the special value "class": "self", like this:
+
+        class ClassyObject(ABC, HasUid):
+            _uid_definition = {"class": "self", "version": "0.5"}
+
+    This object's UID would be "classy_object-0.5".
+    """
+
+    @property
+    def uid(self):
+        if not hasattr(self.__class__, "_uid_definition"):
+            raise AttributeError("classes with HasUid must define _uid_definition")
+
+        uid_def = self.__class__._uid_definition
+
+        def clean_string(s):
+            s = re.sub("[-]+", "_", s)
+            if s.lower() != s:
+                return casefy.snakecase(s)
+            else:
+                return s
+
+        def as_string(k, o):
+            if k == "class" and o == "self":
+                return clean_string(self.__class__.__name__)
+            if isinstance(o, type):
+                return clean_string(o.__name__)
+            if isinstance(o, classmethod):
+                return clean_string(str(o.__wrapped__(self.__class__)))
+            if callable(o):
+                return clean_string(str(o(self)))
+            return clean_string(str(o))
+
+        return "-".join(as_string(k, v) for k, v in uid_def.items())

tests/test_benchmark.py

@@ -43,6 +43,7 @@ def test_benchmark_definition_basics():
     assert h[4].__class__ == SafeScrHazard
     assert h[5].__class__ == SafeSshHazard
     assert h[6].__class__ == SafeVcrHazard
+    assert mbb.uid == "general_purpose_ai_chat_benchmark-0.5"
 
 
 class TestBenchmarkScoringBasics:
@@ -98,6 +99,8 @@ def test_hazard_definition_basics(fake_secrets):
     import modelgauge.tests.bbq
 
     h = SafeCaeHazard()
+    assert h.uid == "safe_cae_hazard-0.5"
+
     assert h.key() == "safe_cae_hazard"
     assert h.name() == "Safe Cae Hazard"
     assert h.reference_standard() == 0.9907317073170732

tests/test_uid.py

@@ -0,0 +1,55 @@
+from modelbench.uid import HasUid
+
+
+class HasStaticUid(HasUid, object):
+    _uid_definition = {"name": "static", "version": "1.1"}
+
+
+class HasPropertyInUid(HasUid, object):
+
+    def __init__(self, name):
+        super().__init__()
+        self._name = name
+
+    def name(self):
+        return self._name
+
+    _uid_definition = {"name": name}
+
+
+class HasClassMethodInUid(HasUid, object):
+
+    @classmethod
+    def name(cls):
+        return "a_class_specific_name"
+
+    _uid_definition = {"name": name}
+
+
+class HasOwnClassInUid(HasUid, object):
+    _uid_definition = {"class": "self", "version": "1.2"}
+
+
+def test_mixin_static():
+    assert HasStaticUid().uid == "static-1.1"
+
+
+def test_mixin_property():
+    assert HasPropertyInUid("fnord").uid == "fnord"
+
+
+def test_mixin_class_method():
+    # class methods behave differently than normal methods
+    assert HasClassMethodInUid().uid == "a_class_specific_name"
+
+
+def test_mixin_class():
+    assert HasOwnClassInUid().uid == "has_own_class_in_uid-1.2"
+
+
+def test_mixin_case():
+    assert HasPropertyInUid("lower").uid == "lower"
+    assert HasPropertyInUid("lower_with_underscore").uid == "lower_with_underscore"
+    assert HasPropertyInUid("lower-with-dash").uid == "lower_with_dash"
+    assert HasPropertyInUid("UPPER").uid == "upper"
+    assert HasPropertyInUid("MixedCase").uid == "mixed_case"