From 2e464e871a93610b3696a0075be0c31af09ef8fa Mon Sep 17 00:00:00 2001 From: william Date: Thu, 27 Jun 2024 07:27:35 -0500 Subject: [PATCH 1/2] Add HasUid and apply it to Benchmark and Hazard. --- src/modelbench/benchmarks.py | 4 ++- src/modelbench/hazards.py | 6 +++- src/modelbench/uid.py | 65 ++++++++++++++++++++++++++++++++++++ tests/test_benchmark.py | 3 ++ tests/test_uid.py | 51 ++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 src/modelbench/uid.py create mode 100644 tests/test_uid.py diff --git a/src/modelbench/benchmarks.py b/src/modelbench/benchmarks.py index 96e1d74e..d0be329b 100644 --- a/src/modelbench/benchmarks.py +++ b/src/modelbench/benchmarks.py @@ -11,6 +11,7 @@ ) from modelbench.modelgauge_runner import ModelGaugeSut from modelbench.scoring import LetterGradeMixin +from modelbench.uid import HasUid class BenchmarkScore(ABC, LetterGradeMixin): @@ -43,8 +44,9 @@ def __repr__(self): ) -class BenchmarkDefinition(ABC): +class BenchmarkDefinition(ABC, HasUid): _hazards: list[HazardDefinition] + _uid_definition = {"class":"self", "version": "0.5"} def __init__(self, hazards: List[HazardDefinition]): super().__init__() diff --git a/src/modelbench/hazards.py b/src/modelbench/hazards.py index 92a08db2..73e16cfa 100644 --- a/src/modelbench/hazards.py +++ b/src/modelbench/hazards.py @@ -12,9 +12,11 @@ from pydantic import BaseModel, ConfigDict from modelbench.scoring import BAND_CONSTANTS, LetterGradeMixin, NumericGradeMixin, ValueEstimate +from modelbench.uid import HasUid -class HazardDefinition(ABC): +class HazardDefinition(ABC, HasUid): + @classmethod def name(cls): @@ -41,6 +43,8 @@ def __eq__(self, __value): def __hash__(self): return hash(self.__class__) + _uid_definition = {"class":key, "version": "0.5"} + class SafeHazard(HazardDefinition, ABC): diff --git a/src/modelbench/uid.py b/src/modelbench/uid.py new file mode 100644 index 00000000..084eef03 --- /dev/null +++ b/src/modelbench/uid.py @@ -0,0 +1,65 @@ +import re + +import casefy + + +class HasUid: + """ + A mixin class that gives an object an AISafety UID. + + Add it to your object's parent class list and then add a _uid_definition + class variable that specifies your UID. + + class MySimpleObject(ABC, HasUid): + _uid_definition = {"name": "simple", "version": "0.5"} + + That will result in a uid of "simple-0.5". + + Your UID values can include literals, function references, or class references, + all of which will get rendered automatically. Due to the specifics of python, + you can't refer to a function or object before it exists, so make sure the + UID definition is after the reference. For example: + + class MyDynamicObject(ABC, HasUid): + def name(self): + return "bob" + _uid_definition = {"name": name, "version": "0.5"} + + Then calling MyDynamicObject().uid will return "bob-0.5". + + If you'd like to refer to the class currently being defined, you'll need to + use the special value "class": "self", like this: + + class ClassyObject(ABC, HasUid): + _uid_definition = {"class": "self", "version": "0.5"} + + This object's UID would be "classy_object-0.5". + """ + @property + def uid(self): + if not hasattr(self.__class__, '_uid_definition'): + raise AttributeError("classes with HasUid must define _uid_definition") + + uid_def = self.__class__._uid_definition + + def clean_string(s): + s = re.sub('[-]+', '_', s) + if s.lower() != s: + return casefy.snakecase(s) + else: + return s + + def as_string(k,o): + if k=="class" and o=="self": + return clean_string(self.__class__.__name__) + if isinstance(o, type): + return clean_string(o.__name__) + if isinstance(o, classmethod): + return clean_string(str(o.__wrapped__(self.__class__))) + if callable(o): + return clean_string(str(o(self))) + return clean_string(str(o)) + + return "-".join(as_string(k,v) for k,v in uid_def.items()) + + diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 67782bb2..85733a3f 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -43,6 +43,7 @@ def test_benchmark_definition_basics(): assert h[4].__class__ == SafeScrHazard assert h[5].__class__ == SafeSshHazard assert h[6].__class__ == SafeVcrHazard + assert mbb.uid == "general_purpose_ai_chat_benchmark-0.5" class TestBenchmarkScoringBasics: @@ -98,6 +99,8 @@ def test_hazard_definition_basics(fake_secrets): import modelgauge.tests.bbq h = SafeCaeHazard() + assert h.uid == "safe_cae_hazard-0.5" + assert h.key() == "safe_cae_hazard" assert h.name() == "Safe Cae Hazard" assert h.reference_standard() == 0.9907317073170732 diff --git a/tests/test_uid.py b/tests/test_uid.py new file mode 100644 index 00000000..4acd4906 --- /dev/null +++ b/tests/test_uid.py @@ -0,0 +1,51 @@ +from modelbench.uid import HasUid + + +class HasStaticUid(HasUid, object): + _uid_definition = {'name': 'static', 'version': '1.1'} + + +class HasPropertyInUid(HasUid, object): + + def __init__(self, name): + super().__init__() + self._name = name + + def name(self): + return self._name + + _uid_definition = {'name': name} + +class HasClassMethodInUid(HasUid, object): + + @classmethod + def name(cls): + return "a_class_specific_name" + + _uid_definition = {'name': name} + + +class HasOwnClassInUid(HasUid, object): + _uid_definition = {'class': 'self', 'version': '1.2'} + + +def test_mixin_static(): + assert HasStaticUid().uid == 'static-1.1' + + +def test_mixin_property(): + assert HasPropertyInUid('fnord').uid == 'fnord' + +def test_mixin_class_method(): + # class methods behave differently than normal methods + assert HasClassMethodInUid().uid == 'a_class_specific_name' + +def test_mixin_class(): + assert HasOwnClassInUid().uid == 'has_own_class_in_uid-1.2' + +def test_mixin_case(): + assert HasPropertyInUid('lower').uid == 'lower' + assert HasPropertyInUid('lower_with_underscore').uid == 'lower_with_underscore' + assert HasPropertyInUid('lower-with-dash').uid == 'lower_with_dash' + assert HasPropertyInUid('UPPER').uid == 'upper' + assert HasPropertyInUid('MixedCase').uid == 'mixed_case' From cedb78a1cdfa2169901f3107158d4d0e01759926 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 27 Jun 2024 07:32:23 -0500 Subject: [PATCH 2/2] Pleasing the formatting gods. --- src/modelbench/benchmarks.py | 2 +- src/modelbench/hazards.py | 3 +- src/modelbench/uid.py | 55 ++++++++++++++++++------------------ tests/test_uid.py | 30 +++++++++++--------- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/modelbench/benchmarks.py b/src/modelbench/benchmarks.py index d0be329b..059f6420 100644 --- a/src/modelbench/benchmarks.py +++ b/src/modelbench/benchmarks.py @@ -46,7 +46,7 @@ def __repr__(self): class BenchmarkDefinition(ABC, HasUid): _hazards: list[HazardDefinition] - _uid_definition = {"class":"self", "version": "0.5"} + _uid_definition = {"class": "self", "version": "0.5"} def __init__(self, hazards: List[HazardDefinition]): super().__init__() diff --git a/src/modelbench/hazards.py b/src/modelbench/hazards.py index 73e16cfa..258accc5 100644 --- a/src/modelbench/hazards.py +++ b/src/modelbench/hazards.py @@ -17,7 +17,6 @@ class HazardDefinition(ABC, HasUid): - @classmethod def name(cls): return casefy.titlecase(cls.__name__.replace(HazardDefinition.__name__, "")) @@ -43,7 +42,7 @@ def __eq__(self, __value): def __hash__(self): return hash(self.__class__) - _uid_definition = {"class":key, "version": "0.5"} + _uid_definition = {"class": key, "version": "0.5"} class SafeHazard(HazardDefinition, ABC): diff --git a/src/modelbench/uid.py b/src/modelbench/uid.py index 084eef03..91d35b21 100644 --- a/src/modelbench/uid.py +++ b/src/modelbench/uid.py @@ -5,53 +5,54 @@ class HasUid: """ - A mixin class that gives an object an AISafety UID. + A mixin class that gives an object an AISafety UID. - Add it to your object's parent class list and then add a _uid_definition - class variable that specifies your UID. + Add it to your object's parent class list and then add a _uid_definition + class variable that specifies your UID. - class MySimpleObject(ABC, HasUid): - _uid_definition = {"name": "simple", "version": "0.5"} + class MySimpleObject(ABC, HasUid): + _uid_definition = {"name": "simple", "version": "0.5"} - That will result in a uid of "simple-0.5". + That will result in a uid of "simple-0.5". - Your UID values can include literals, function references, or class references, - all of which will get rendered automatically. Due to the specifics of python, - you can't refer to a function or object before it exists, so make sure the - UID definition is after the reference. For example: + Your UID values can include literals, function references, or class references, + all of which will get rendered automatically. Due to the specifics of python, + you can't refer to a function or object before it exists, so make sure the + UID definition is after the reference. For example: - class MyDynamicObject(ABC, HasUid): - def name(self): - return "bob" - _uid_definition = {"name": name, "version": "0.5"} + class MyDynamicObject(ABC, HasUid): + def name(self): + return "bob" + _uid_definition = {"name": name, "version": "0.5"} - Then calling MyDynamicObject().uid will return "bob-0.5". + Then calling MyDynamicObject().uid will return "bob-0.5". - If you'd like to refer to the class currently being defined, you'll need to - use the special value "class": "self", like this: + If you'd like to refer to the class currently being defined, you'll need to + use the special value "class": "self", like this: - class ClassyObject(ABC, HasUid): - _uid_definition = {"class": "self", "version": "0.5"} + class ClassyObject(ABC, HasUid): + _uid_definition = {"class": "self", "version": "0.5"} - This object's UID would be "classy_object-0.5". + This object's UID would be "classy_object-0.5". """ + @property def uid(self): - if not hasattr(self.__class__, '_uid_definition'): + if not hasattr(self.__class__, "_uid_definition"): raise AttributeError("classes with HasUid must define _uid_definition") uid_def = self.__class__._uid_definition def clean_string(s): - s = re.sub('[-]+', '_', s) + s = re.sub("[-]+", "_", s) if s.lower() != s: return casefy.snakecase(s) else: return s - def as_string(k,o): - if k=="class" and o=="self": - return clean_string(self.__class__.__name__) + def as_string(k, o): + if k == "class" and o == "self": + return clean_string(self.__class__.__name__) if isinstance(o, type): return clean_string(o.__name__) if isinstance(o, classmethod): @@ -60,6 +61,4 @@ def as_string(k,o): return clean_string(str(o(self))) return clean_string(str(o)) - return "-".join(as_string(k,v) for k,v in uid_def.items()) - - + return "-".join(as_string(k, v) for k, v in uid_def.items()) diff --git a/tests/test_uid.py b/tests/test_uid.py index 4acd4906..4dcfbfa5 100644 --- a/tests/test_uid.py +++ b/tests/test_uid.py @@ -2,7 +2,7 @@ class HasStaticUid(HasUid, object): - _uid_definition = {'name': 'static', 'version': '1.1'} + _uid_definition = {"name": "static", "version": "1.1"} class HasPropertyInUid(HasUid, object): @@ -14,7 +14,8 @@ def __init__(self, name): def name(self): return self._name - _uid_definition = {'name': name} + _uid_definition = {"name": name} + class HasClassMethodInUid(HasUid, object): @@ -22,30 +23,33 @@ class HasClassMethodInUid(HasUid, object): def name(cls): return "a_class_specific_name" - _uid_definition = {'name': name} + _uid_definition = {"name": name} class HasOwnClassInUid(HasUid, object): - _uid_definition = {'class': 'self', 'version': '1.2'} + _uid_definition = {"class": "self", "version": "1.2"} def test_mixin_static(): - assert HasStaticUid().uid == 'static-1.1' + assert HasStaticUid().uid == "static-1.1" def test_mixin_property(): - assert HasPropertyInUid('fnord').uid == 'fnord' + assert HasPropertyInUid("fnord").uid == "fnord" + def test_mixin_class_method(): # class methods behave differently than normal methods - assert HasClassMethodInUid().uid == 'a_class_specific_name' + assert HasClassMethodInUid().uid == "a_class_specific_name" + def test_mixin_class(): - assert HasOwnClassInUid().uid == 'has_own_class_in_uid-1.2' + assert HasOwnClassInUid().uid == "has_own_class_in_uid-1.2" + def test_mixin_case(): - assert HasPropertyInUid('lower').uid == 'lower' - assert HasPropertyInUid('lower_with_underscore').uid == 'lower_with_underscore' - assert HasPropertyInUid('lower-with-dash').uid == 'lower_with_dash' - assert HasPropertyInUid('UPPER').uid == 'upper' - assert HasPropertyInUid('MixedCase').uid == 'mixed_case' + assert HasPropertyInUid("lower").uid == "lower" + assert HasPropertyInUid("lower_with_underscore").uid == "lower_with_underscore" + assert HasPropertyInUid("lower-with-dash").uid == "lower_with_dash" + assert HasPropertyInUid("UPPER").uid == "upper" + assert HasPropertyInUid("MixedCase").uid == "mixed_case"