Retry anthropic 429 more doggedly. Add a little more to the journal.

mlcommons · Nov 19, 2024 · dab3694 · dab3694
1 parent e469bc8
commit dab3694
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/plugins/anthropic/modelgauge/suts/anthropic_api.py b/plugins/anthropic/modelgauge/suts/anthropic_api.py
@@ -1,5 +1,8 @@
+from random import random
+from time import sleep
 from typing import List, Optional
 
+import anthropic
 from anthropic import Anthropic
 from anthropic.types import TextBlock
 from anthropic.types.message import Message as AnthropicMessage
@@ -70,6 +73,9 @@ def evaluate(self, request: AnthropicRequest) -> AnthropicMessage:
         request_dict = request.model_dump(exclude_none=True)
         try:
             return self.client.messages.create(**request_dict)
+        except anthropic.RateLimitError:
+            sleep(60 * random())  # anthropic uses 1-minute buckets
+            return self.evaluate(request)
         except Exception as e:
             raise APIException(f"Error calling Anthropic API: {e}")
 

diff --git a/src/modelbench/benchmark_runner.py b/src/modelbench/benchmark_runner.py
@@ -308,8 +308,13 @@ def handle_item(self, item: TestRunItem):
             self.test_run.journal.item_entry("translated sut response", item, response=response)
 
         except Exception as e:
+            extra_info = {}
+            try:
+                extra_info["run_time"] = timer
+            except NameError:
+                pass
             item.exceptions.append(e)
-            self.test_run.journal.item_exception_entry("sut exception", item, e)
+            self.test_run.journal.item_exception_entry("sut exception", item, e, **extra_info)
             logger.error(f"failure handling sut item {item}:", exc_info=e)
         return item
 
@@ -330,8 +335,13 @@ def handle_item(self, item: TestRunItem) -> TestRunItem:
                 )
         except Exception as e:
             item.exceptions.append(e)
+            extra_info = {}
+            try:
+                extra_info["run_time"] = timer
+            except NameError:
+                pass
+            self.test_run.journal.item_exception_entry("annotation exception", item, e, **extra_info)
             logger.error(f"failure handling annnotation for {item}", exc_info=e)
-            self.test_run.journal.item_exception_entry("annotation exception", item, e)
         return item
 
     def collect_annotations(self, item):