Skip to content

Commit

Permalink
If the initial check fails, add some console output explaining why.
Browse files Browse the repository at this point in the history
  • Loading branch information
wpietri committed Nov 1, 2024
1 parent cc01e82 commit 604cc52
Showing 1 changed file with 6 additions and 0 deletions.
6 changes: 6 additions & 0 deletions src/modelbench/benchmark_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import random
import sys
import time
import traceback
from abc import abstractmethod
from collections import defaultdict
from datetime import datetime
Expand Down Expand Up @@ -430,6 +431,9 @@ def check_sut(sut: ModelGaugeSut):
return bool(response.completions)
except Exception as e:
logger.error(f"initial check failure for {sut}", exc_info=e)
print(f"initial check failure for {sut}")
traceback.print_exc()

return False

with ThreadPool(len(run.suts)) as pool:
Expand All @@ -454,6 +458,8 @@ def check_annotator(annotator: CompletionAnnotator):

except Exception as e:
logger.error(f"initial check failure for {annotator}", exc_info=e)
print(f"initial check failure for {annotator}")
traceback.print_exc()
return False

annotators = set(a for l in run.test_annotators.values() for a in l)
Expand Down

0 comments on commit 604cc52

Please sign in to comment.