From 53b6e5b01d5f31b091c4e5466902b9e5a381fb26 Mon Sep 17 00:00:00 2001 From: Barbara Korycki Date: Wed, 18 Dec 2024 09:57:45 -1000 Subject: [PATCH 1/2] Add missing prompt counts --- src/modelbench/consistency_checker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/modelbench/consistency_checker.py b/src/modelbench/consistency_checker.py index 2fb58e5f..80022090 100644 --- a/src/modelbench/consistency_checker.py +++ b/src/modelbench/consistency_checker.py @@ -91,11 +91,11 @@ def failure_message(self) -> str: assert not self.check() messages = [] if len(self.duplicates) > 0: - messages.append(f"The following duplicate prompts were found: {self.duplicates}") + messages.append(f"{len(self.duplicates)} duplicate prompts were found: {self.duplicates}") if len(self.missing_prompts) > 0: - messages.append(f"The prompts were expected but missing: {self.missing_prompts}") + messages.append(f"{len(self.missing_prompts)} prompts were expected but missing: {self.missing_prompts}") if len(self.unknown_prompts) > 0: - messages.append(f"The following prompts were found but were not expected: {self.unknown_prompts}") + messages.append(f"{len(self.unknown_prompts)} prompts were found but were not expected: {self.unknown_prompts}") return "\n\t".join(messages) @@ -113,12 +113,13 @@ def failure_message(self) -> str: class EachPromptRespondedToOnce(OneToOneCheck): def __init__(self, search_engine: JournalSearch, sut, test): + self.test = test super().__init__( search_engine.test_prompt_uids(test), search_engine.sut_response_prompt_uids_for_test(sut, test) ) def failure_message(self) -> str: - message = "Expected exactly 1 SUT response for each prompt in the test.\n\t" + message = f"Expected exactly 1 SUT response for each prompt in the test {self.test}.\n\t" # Call super() to get specific details about duplicates/missing/extra prompts. return message + super().failure_message() From c4a8011e78f6c698b88a0c00a3aeb5474ba4f42b Mon Sep 17 00:00:00 2001 From: Barbara Korycki Date: Wed, 18 Dec 2024 10:27:51 -1000 Subject: [PATCH 2/2] lint --- src/modelbench/consistency_checker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/modelbench/consistency_checker.py b/src/modelbench/consistency_checker.py index 80022090..06b9f550 100644 --- a/src/modelbench/consistency_checker.py +++ b/src/modelbench/consistency_checker.py @@ -95,7 +95,9 @@ def failure_message(self) -> str: if len(self.missing_prompts) > 0: messages.append(f"{len(self.missing_prompts)} prompts were expected but missing: {self.missing_prompts}") if len(self.unknown_prompts) > 0: - messages.append(f"{len(self.unknown_prompts)} prompts were found but were not expected: {self.unknown_prompts}") + messages.append( + f"{len(self.unknown_prompts)} prompts were found but were not expected: {self.unknown_prompts}" + ) return "\n\t".join(messages)