From 8eafb12fc2d0ddfbb7f1ff5a5f5eb899b0c71a55 Mon Sep 17 00:00:00 2001 From: William Pietri Date: Thu, 18 Jul 2024 14:37:53 -0700 Subject: [PATCH] Add HasUid and apply it to Benchmark and Hazard. (#386) * Add HasUid and apply it to Benchmark and Hazard. * Pleasing the formatting gods. --- src/modelbench/benchmarks.py | 4 ++- src/modelbench/hazards.py | 5 ++- src/modelbench/uid.py | 64 ++++++++++++++++++++++++++++++++++++ tests/test_benchmark.py | 3 ++ tests/test_uid.py | 55 +++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 src/modelbench/uid.py create mode 100644 tests/test_uid.py diff --git a/src/modelbench/benchmarks.py b/src/modelbench/benchmarks.py index 96e1d74e..059f6420 100644 --- a/src/modelbench/benchmarks.py +++ b/src/modelbench/benchmarks.py @@ -11,6 +11,7 @@ ) from modelbench.modelgauge_runner import ModelGaugeSut from modelbench.scoring import LetterGradeMixin +from modelbench.uid import HasUid class BenchmarkScore(ABC, LetterGradeMixin): @@ -43,8 +44,9 @@ def __repr__(self): ) -class BenchmarkDefinition(ABC): +class BenchmarkDefinition(ABC, HasUid): _hazards: list[HazardDefinition] + _uid_definition = {"class": "self", "version": "0.5"} def __init__(self, hazards: List[HazardDefinition]): super().__init__() diff --git a/src/modelbench/hazards.py b/src/modelbench/hazards.py index 92a08db2..258accc5 100644 --- a/src/modelbench/hazards.py +++ b/src/modelbench/hazards.py @@ -12,9 +12,10 @@ from pydantic import BaseModel, ConfigDict from modelbench.scoring import BAND_CONSTANTS, LetterGradeMixin, NumericGradeMixin, ValueEstimate +from modelbench.uid import HasUid -class HazardDefinition(ABC): +class HazardDefinition(ABC, HasUid): @classmethod def name(cls): @@ -41,6 +42,8 @@ def __eq__(self, __value): def __hash__(self): return hash(self.__class__) + _uid_definition = {"class": key, "version": "0.5"} + class SafeHazard(HazardDefinition, ABC): diff --git a/src/modelbench/uid.py b/src/modelbench/uid.py new file mode 100644 index 00000000..91d35b21 --- /dev/null +++ b/src/modelbench/uid.py @@ -0,0 +1,64 @@ +import re + +import casefy + + +class HasUid: + """ + A mixin class that gives an object an AISafety UID. + + Add it to your object's parent class list and then add a _uid_definition + class variable that specifies your UID. + + class MySimpleObject(ABC, HasUid): + _uid_definition = {"name": "simple", "version": "0.5"} + + That will result in a uid of "simple-0.5". + + Your UID values can include literals, function references, or class references, + all of which will get rendered automatically. Due to the specifics of python, + you can't refer to a function or object before it exists, so make sure the + UID definition is after the reference. For example: + + class MyDynamicObject(ABC, HasUid): + def name(self): + return "bob" + _uid_definition = {"name": name, "version": "0.5"} + + Then calling MyDynamicObject().uid will return "bob-0.5". + + If you'd like to refer to the class currently being defined, you'll need to + use the special value "class": "self", like this: + + class ClassyObject(ABC, HasUid): + _uid_definition = {"class": "self", "version": "0.5"} + + This object's UID would be "classy_object-0.5". + """ + + @property + def uid(self): + if not hasattr(self.__class__, "_uid_definition"): + raise AttributeError("classes with HasUid must define _uid_definition") + + uid_def = self.__class__._uid_definition + + def clean_string(s): + s = re.sub("[-]+", "_", s) + if s.lower() != s: + return casefy.snakecase(s) + else: + return s + + def as_string(k, o): + if k == "class" and o == "self": + return clean_string(self.__class__.__name__) + if isinstance(o, type): + return clean_string(o.__name__) + if isinstance(o, classmethod): + return clean_string(str(o.__wrapped__(self.__class__))) + if callable(o): + return clean_string(str(o(self))) + return clean_string(str(o)) + + return "-".join(as_string(k, v) for k, v in uid_def.items()) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 67782bb2..85733a3f 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -43,6 +43,7 @@ def test_benchmark_definition_basics(): assert h[4].__class__ == SafeScrHazard assert h[5].__class__ == SafeSshHazard assert h[6].__class__ == SafeVcrHazard + assert mbb.uid == "general_purpose_ai_chat_benchmark-0.5" class TestBenchmarkScoringBasics: @@ -98,6 +99,8 @@ def test_hazard_definition_basics(fake_secrets): import modelgauge.tests.bbq h = SafeCaeHazard() + assert h.uid == "safe_cae_hazard-0.5" + assert h.key() == "safe_cae_hazard" assert h.name() == "Safe Cae Hazard" assert h.reference_standard() == 0.9907317073170732 diff --git a/tests/test_uid.py b/tests/test_uid.py new file mode 100644 index 00000000..4dcfbfa5 --- /dev/null +++ b/tests/test_uid.py @@ -0,0 +1,55 @@ +from modelbench.uid import HasUid + + +class HasStaticUid(HasUid, object): + _uid_definition = {"name": "static", "version": "1.1"} + + +class HasPropertyInUid(HasUid, object): + + def __init__(self, name): + super().__init__() + self._name = name + + def name(self): + return self._name + + _uid_definition = {"name": name} + + +class HasClassMethodInUid(HasUid, object): + + @classmethod + def name(cls): + return "a_class_specific_name" + + _uid_definition = {"name": name} + + +class HasOwnClassInUid(HasUid, object): + _uid_definition = {"class": "self", "version": "1.2"} + + +def test_mixin_static(): + assert HasStaticUid().uid == "static-1.1" + + +def test_mixin_property(): + assert HasPropertyInUid("fnord").uid == "fnord" + + +def test_mixin_class_method(): + # class methods behave differently than normal methods + assert HasClassMethodInUid().uid == "a_class_specific_name" + + +def test_mixin_class(): + assert HasOwnClassInUid().uid == "has_own_class_in_uid-1.2" + + +def test_mixin_case(): + assert HasPropertyInUid("lower").uid == "lower" + assert HasPropertyInUid("lower_with_underscore").uid == "lower_with_underscore" + assert HasPropertyInUid("lower-with-dash").uid == "lower_with_dash" + assert HasPropertyInUid("UPPER").uid == "upper" + assert HasPropertyInUid("MixedCase").uid == "mixed_case"