-
{{ content(benchmark_score.benchmark_definition, "uid") }}
+
{{ benchmark_score.benchmark_definition.uid }}
diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py
index 85733a3f..9c10eb59 100644
--- a/tests/test_benchmark.py
+++ b/tests/test_benchmark.py
@@ -1,11 +1,13 @@
import pathlib
from datetime import datetime
from typing import List, Mapping
+from unittest.mock import MagicMock
import pytest
from modelgauge.base_test import BaseTest
from modelgauge.records import TestRecord
from modelgauge.secret_values import RawSecrets
+from modelgauge.tests.safe import SafeTestResult, PersonaType, PersonaResult
from modelbench.benchmarks import (
BenchmarkScore,
@@ -111,13 +113,27 @@ def test_hazard_definition_basics(fake_secrets):
def test_hazard_score_basics():
- bd = SafeCaeHazard()
- a_score = bd.reference_standard()
- hs = HazardScore(hazard_definition=bd, score=ValueEstimate.make(a_score, 50), test_scores={})
- assert hs.hazard_definition == bd
+ hd = SafeCaeHazard()
+ a_score = hd.reference_standard()
+ hs = HazardScore(hazard_definition=hd, score=ValueEstimate.make(a_score, 50), test_scores={})
+ assert hs.hazard_definition == hd
assert hs.score.estimate == a_score
+def test_hazard_score_test_scores():
+ hd = SafeCaeHazard()
+ mock_test_record = MagicMock()
+ frac_safe = 0.5
+ mock_test_record.result.to_instance.return_value = SafeTestResult(
+ persona_results={PersonaType.TYPICAL: PersonaResult(frac_safe=frac_safe, num_items=100)}
+ )
+ result = hd.score({"foo": mock_test_record})
+ print(result)
+ score_key = next(iter(result.test_scores))
+ assert score_key == hd.uid
+ assert result.test_scores[score_key].estimate == frac_safe
+
+
def test_modelgauge_sut_display_name_and_name():
assert ModelGaugeSut.LLAMA_2_7B.display_name == "Meta Llama 2, 7b parameters"
assert ModelGaugeSut.LLAMA_2_7B.name == "LLAMA_2_7B"
diff --git a/tests/test_record.py b/tests/test_record.py
new file mode 100644
index 00000000..f7ca2e3e
--- /dev/null
+++ b/tests/test_record.py
@@ -0,0 +1,177 @@
+import json
+import platform
+import re
+from datetime import datetime, timezone
+from unittest.mock import Mock, MagicMock, patch
+
+from modelgauge.record_init import InitializationRecord
+
+from modelbench.benchmarks import GeneralPurposeAiChatBenchmark
+from modelbench.hazards import HazardScore, SafeCaeHazard
+from modelbench.modelgauge_runner import ModelGaugeSut
+from modelbench.record import (
+ BenchmarkScoreEncoder,
+ benchmark_run_record,
+ dump_json,
+ benchmark_code_info,
+ benchmark_library_info,
+)
+from modelbench.run import FakeSut
+from modelbench.scoring import ValueEstimate
+from test_static_site_generator import benchmark_score
+
+
+def encode(o):
+ return json.dumps(o, cls=BenchmarkScoreEncoder, indent=4)
+
+
+def encode_and_parse(o):
+ s = encode(o)
+ return json.loads(s)
+
+
+def test_sut():
+ assert encode_and_parse(ModelGaugeSut.ALPACA_7B) == {"uid": "alpaca-7b"}
+
+ ModelGaugeSut.ALPACA_7B.instance(MagicMock())
+ with_initialization = encode_and_parse(ModelGaugeSut.ALPACA_7B)
+ assert "uid" in with_initialization
+ assert "initialization" in with_initialization
+ assert encode_and_parse(ModelGaugeSut.ALPACA_7B) == with_initialization
+
+
+def test_anonymous_sut():
+ j = encode_and_parse(FakeSut("a_sut-v1.0", "A SUT of some sort"))
+ print(j)
+
+
+def test_value_estimate():
+ ve = ValueEstimate.make(0.5, 1000)
+ j = encode_and_parse(ve)
+ assert j["lower"] == ve.lower
+ assert j["estimate"] == ve.estimate
+ assert j["upper"] == ve.upper
+ assert j["samples"] == ve.samples
+
+
+def test_hazard_definition_without_tests_loaded():
+ hazard = SafeCaeHazard()
+ j = encode_and_parse(hazard)
+ assert j["uid"] == hazard.uid
+ assert "tests" not in j
+ assert j["reference_standard"] == hazard.reference_standard()
+
+
+def test_hazard_definition_with_tests_loaded():
+ hazard = SafeCaeHazard()
+ hazard.tests({"together": {"api_key": "ignored"}})
+ j = encode_and_parse(hazard)
+ assert j["uid"] == hazard.uid
+ assert j["tests"] == ["safe-cae"]
+ assert j["reference_standard"] == hazard.reference_standard()
+
+
+def test_benchmark_definition():
+ j = encode_and_parse(GeneralPurposeAiChatBenchmark())
+ assert j["uid"] == "general_purpose_ai_chat_benchmark-0.5"
+ assert "safe_cae_hazard-0.5" in [i["uid"] for i in j["hazards"]]
+
+
+def test_hazard_score():
+ ve = ValueEstimate.make(1.0, 100000)
+ hs = HazardScore(hazard_definition=SafeCaeHazard(), score=ve, test_scores={"cae": ve})
+ j = encode_and_parse(hs)
+ assert j["hazard_definition"]["uid"] == SafeCaeHazard().uid
+ assert j["score"]["samples"] == ve.samples
+ assert j["test_scores"]["cae"]["samples"] == ve.samples
+ assert j["numeric_grade"] == 5
+ assert j["text_grade"] == "L"
+
+
+def test_benchmark_score(benchmark_score):
+ j = encode_and_parse(benchmark_score)
+ assert "benchmark_definition" not in j # it's already higher up in the tree; no need to duplicate
+ assert j["sut"]["uid"] == benchmark_score.sut.key
+ assert len(j["hazard_scores"]) == len(benchmark_score.hazard_scores)
+ assert j["end_time"] == str(benchmark_score.end_time)
+ assert j["numeric_grade"] == benchmark_score.numeric_grade()
+ assert j["text_grade"] == benchmark_score.text_grade()
+
+
+def test_benchmark_score_initialization_record(benchmark_score):
+ mock_method = Mock()
+ mock_method.return_value = InitializationRecord(
+ module="a_module", class_name="a_class", args=["arg1", "arg2"], kwargs={"kwarg1": "a_value"}
+ )
+ benchmark_score.sut.instance_initialization = mock_method
+ j = encode_and_parse(benchmark_score)
+ print(j)
+ assert j["sut"]["initialization"]["module"] == "a_module"
+
+
+def test_benchmark_run_record(benchmark_score):
+ r = benchmark_run_record(benchmark_score)
+ assert r["score"] == benchmark_score
+ assert r["_metadata"]["format_version"] == 1
+
+ run_info = r["_metadata"]["run"]
+ assert re.match(r"\w+", run_info["user"])
+ assert re.match(r"20\d\d-.+UTC", run_info["timestamp"])
+ assert run_info["platform"] == platform.platform()
+ assert run_info["system"]
+ assert run_info["node"] == platform.node()
+ assert run_info["python"] == platform.python_version()
+
+
+def test_benchmark_code_record(benchmark_score):
+ r = benchmark_run_record(benchmark_score)
+ source = r["_metadata"]["code"]["source"]
+ assert source["git_version"].startswith("git version 2")
+ assert source["origin"] in ["git@github.com:mlcommons/modelbench.git", "https://github.com/mlcommons/modelbench"]
+ assert re.match(r"(v[.0-9]+-\d+-)?[a-z0-9]{8}", source["code_version"])
+ assert isinstance(source["changed_files"], list) # hard to be more specific here
+
+
+def test_benchmark_code_record_without_git_command(benchmark_score):
+ with patch("modelbench.record.run_command") as f:
+ f.side_effect = FileNotFoundError()
+ j = benchmark_code_info()
+ print(j)
+ assert j["error"].startswith("git command not found")
+
+
+def test_benchmark_code_record_without_git_repo(benchmark_score, cwd_tmpdir):
+ j = benchmark_code_info()
+ print(j)
+ assert j["error"].startswith("couldn't find git dir")
+
+
+def test_benchmark_code_record_without_git(benchmark_score):
+ with patch("modelbench.record.run_command") as f:
+ f.side_effect = FileNotFoundError()
+ r = benchmark_run_record(benchmark_score)
+ source = r["_metadata"]["code"]["source"]
+ assert source["error"] == "git command not found"
+
+
+def test_pip_list():
+ i = benchmark_library_info()
+ print(i)
+ assert "modelgauge" in i
+
+
+def test_dump_json(benchmark_score, tmp_path):
+ # just a smoke test; everything substantial should be tested above.
+ json_path = tmp_path / "foo.json"
+ dump_json(
+ json_path,
+ datetime.fromtimestamp(1700000000, timezone.utc),
+ benchmark_score.benchmark_definition,
+ [benchmark_score],
+ )
+ with open(json_path) as f:
+ j = json.load(f)
+ assert j["benchmark"]["uid"] == benchmark_score.benchmark_definition.uid
+ assert j["run_uid"] == "run-" + benchmark_score.benchmark_definition.uid + "-20231114-221320"
+ assert "grades" in j["content"]
+ assert len(j["scores"]) == 1
diff --git a/tests/test_run.py b/tests/test_run.py
index 1e608d20..1fdf298d 100644
--- a/tests/test_run.py
+++ b/tests/test_run.py
@@ -1,16 +1,17 @@
import json
import pathlib
import unittest.mock
+from datetime import datetime
from unittest.mock import MagicMock, patch
import click
import pytest
from click.testing import CliRunner
-from modelbench.benchmarks import BenchmarkDefinition
+from modelbench.benchmarks import BenchmarkDefinition, BenchmarkScore, GeneralPurposeAiChatBenchmark
from modelbench.hazards import HazardScore, SafeCbrHazard
from modelbench.hazards import SafeHazard
-from modelbench.modelgauge_runner import ModelGaugeSut
+from modelbench.modelgauge_runner import ModelGaugeSut, SutDescription
from modelbench.run import benchmark, cli, find_suts_for_sut_argument, update_standards_to
from modelbench.scoring import ValueEstimate
@@ -51,14 +52,26 @@ def test_find_suts():
class TestCli:
+ def mock_score(self):
+ benchmark = GeneralPurposeAiChatBenchmark()
+ return BenchmarkScore(
+ benchmark,
+ ModelGaugeSut.ALPACA_7B,
+ [
+ HazardScore(
+ hazard_definition=benchmark.hazards()[0], score=ValueEstimate.make(0.123456, 100), test_scores={}
+ ),
+ ],
+ datetime.now(),
+ )
+
@pytest.fixture(autouse=True)
def mock_score_benchmarks(self, monkeypatch):
import modelbench
- mock_obj = MagicMock()
-
- monkeypatch.setattr(modelbench.run, "score_benchmarks", mock_obj)
- return mock_obj
+ mock = MagicMock(return_value=[self.mock_score()])
+ monkeypatch.setattr(modelbench.run, "score_benchmarks", mock)
+ return mock
@pytest.fixture(autouse=True)
def do_not_make_static_site(self, monkeypatch):
@@ -70,6 +83,38 @@ def do_not_make_static_site(self, monkeypatch):
def runner(self):
return CliRunner()
+ def test_benchmark_basic_run_produces_json(self, runner, tmp_path):
+ with unittest.mock.patch("modelbench.run.find_suts_for_sut_argument") as mock_find_suts:
+ mock_find_suts.return_value = [SutDescription("fake", "Fake Sut")]
+ result = runner.invoke(
+ cli,
+ ["benchmark", "-m", "1", "--sut", "fake", "--output-dir", str(tmp_path.absolute())],
+ catch_exceptions=False,
+ )
+ assert result.exit_code == 0
+ assert (tmp_path / f"benchmark_record-{GeneralPurposeAiChatBenchmark().uid}.json").exists
+
+ def test_benchmark_anonymous_run_produces_json(self, runner, tmp_path):
+ with unittest.mock.patch("modelbench.run.find_suts_for_sut_argument") as mock_find_suts:
+ mock_find_suts.return_value = [SutDescription("fake", "Fake Sut")]
+ result = runner.invoke(
+ cli,
+ [
+ "benchmark",
+ "--anonymize",
+ "42",
+ "-m",
+ "1",
+ "--sut",
+ "fake",
+ "--output-dir",
+ str(tmp_path.absolute()),
+ ],
+ catch_exceptions=False,
+ )
+ assert result.exit_code == 0, result.stdout
+ assert (tmp_path / f"benchmark_record-{GeneralPurposeAiChatBenchmark().uid}.json").exists
+
def test_nonexistent_benchmarks_can_not_be_called(self, runner):
result = runner.invoke(cli, ["benchmark", "--benchmark", "NotARealBenchmark"])
assert result.exit_code == 2