From 049568b4cba3bdd5c812f4b81bd94c2d873c97e6 Mon Sep 17 00:00:00 2001 From: William Pietri Date: Tue, 19 Nov 2024 05:44:25 -0800 Subject: [PATCH] Retry anthropic 429 more doggedly. Add a little more to the journal. (#698) --- plugins/anthropic/modelgauge/suts/anthropic_api.py | 6 ++++++ src/modelbench/benchmark_runner.py | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/plugins/anthropic/modelgauge/suts/anthropic_api.py b/plugins/anthropic/modelgauge/suts/anthropic_api.py index a99540c3..e3c3ec85 100644 --- a/plugins/anthropic/modelgauge/suts/anthropic_api.py +++ b/plugins/anthropic/modelgauge/suts/anthropic_api.py @@ -1,5 +1,8 @@ +from random import random +from time import sleep from typing import List, Optional +import anthropic from anthropic import Anthropic from anthropic.types import TextBlock from anthropic.types.message import Message as AnthropicMessage @@ -70,6 +73,9 @@ def evaluate(self, request: AnthropicRequest) -> AnthropicMessage: request_dict = request.model_dump(exclude_none=True) try: return self.client.messages.create(**request_dict) + except anthropic.RateLimitError: + sleep(60 * random()) # anthropic uses 1-minute buckets + return self.evaluate(request) except Exception as e: raise APIException(f"Error calling Anthropic API: {e}") diff --git a/src/modelbench/benchmark_runner.py b/src/modelbench/benchmark_runner.py index 2f125713..ac37e962 100644 --- a/src/modelbench/benchmark_runner.py +++ b/src/modelbench/benchmark_runner.py @@ -308,8 +308,13 @@ def handle_item(self, item: TestRunItem): self.test_run.journal.item_entry("translated sut response", item, response=response) except Exception as e: + extra_info = {} + try: + extra_info["run_time"] = timer + except NameError: + pass item.exceptions.append(e) - self.test_run.journal.item_exception_entry("sut exception", item, e) + self.test_run.journal.item_exception_entry("sut exception", item, e, **extra_info) logger.error(f"failure handling sut item {item}:", exc_info=e) return item @@ -330,8 +335,13 @@ def handle_item(self, item: TestRunItem) -> TestRunItem: ) except Exception as e: item.exceptions.append(e) + extra_info = {} + try: + extra_info["run_time"] = timer + except NameError: + pass + self.test_run.journal.item_exception_entry("annotation exception", item, e, **extra_info) logger.error(f"failure handling annnotation for {item}", exc_info=e) - self.test_run.journal.item_exception_entry("annotation exception", item, e) return item def collect_annotations(self, item):