Use final practice prompts

src/modelbench/run.py

@@ -89,7 +89,7 @@ def cli() -> None:
 @click.option(
     "--locale",
     "-l",
-    type=click.Choice([l.value.lower() for l in Locale] + ["all"], case_sensitive=False),
+    type=click.Choice(["en_us"], case_sensitive=False),
     default="en_us",
     help=f"Locale for v1.0 benchmark (Default: en_us)",
     multiple=False,

src/modelgauge/annotators/llama_guard_annotator.py

@@ -1,3 +1,11 @@
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from string import Template
+from typing import Callable, Dict, List, Optional
+
+from pydantic import BaseModel
+
 from modelgauge.annotator import CompletionAnnotator
 from modelgauge.annotator_registry import ANNOTATORS
 from modelgauge.secret_values import InjectSecret
@@ -9,11 +17,8 @@
     TogetherCompletionsResponse,
     TogetherCompletionsSUT,
 )
-from dataclasses import dataclass
-from enum import Enum
-from pydantic import BaseModel
-from string import Template
-from typing import Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
 
 
 class LlamaGuardAnnotation(BaseModel):
@@ -273,9 +278,12 @@ def translate_response(
         elif lines[0] == "unsafe":
             decoder = _make_llamaguard_mapping(self.llama_guard_data) if self.decoder is None else self.decoder
             categories = []
-            assert len(lines) > 1
-            for raw in lines[1].split(","):
-                categories.append(decoder[raw])
+            if len(lines) > 1:
+                for raw in lines[1].split(","):
+                    categories.append(decoder[raw])
+            else:
+                logger.warning(f"categories missing from '{together_response.text}' for: {request}")
+
             return LlamaGuardAnnotation(is_safe=False, is_safe_logprob=logprob, violation_categories=categories)
         else:
             raise AssertionError("Unexpected response: ", together_response.text)

src/modelgauge/dependency_helper.py

@@ -3,11 +3,13 @@
 import shutil
 import tempfile
 from abc import ABC, abstractmethod
+from typing import Dict, Mapping, Optional
+
+from pydantic import BaseModel, Field
+
 from modelgauge.data_packing import DataUnpacker
 from modelgauge.external_data import ExternalData
 from modelgauge.general import current_timestamp_millis, hash_file, normalize_filename
-from pydantic import BaseModel, Field
-from typing import Dict, Mapping, Optional
 
 
 class DependencyHelper(ABC):
@@ -78,11 +80,19 @@ def get_local_path(self, dependency_key: str) -> str:
 
         normalized_key = normalize_filename(dependency_key)
         version: str
-        if dependency_key in self.required_versions:
-            version = self.required_versions[dependency_key]
-            self._ensure_required_version_exists(dependency_key, normalized_key, external_data, version)
-        else:
-            version = self._get_latest_version(normalized_key, external_data)
+        try:
+            if dependency_key in self.required_versions:
+                version = self.required_versions[dependency_key]
+                self._ensure_required_version_exists(dependency_key, normalized_key, external_data, version)
+            else:
+                version = self._get_latest_version(normalized_key, external_data)
+        except Exception as e:
+            if isinstance(e, RuntimeError):
+                raise e
+            else:
+                raise RuntimeError(
+                    f"Failure loading dependency metadata from source for {self.dependencies[dependency_key]}"
+                ) from e
         self.used_dependencies[dependency_key] = version
         return self._get_version_path(normalized_key, version)
 

src/modelgauge/tests/safe_v1.py

@@ -104,10 +104,10 @@ def _check_annotators(cls):
             raise NotImplementedError("Concrete SafeTestVersion1 classes must set class-attribute `annotators`.")
 
     def _get_prompt_set_names(self):
-        return ["all_practice_skilled_unskilled"]
+        return ["airr_official_1.0_practice_prompt_set_release"]
 
     def get_dependencies(self) -> Mapping[str, ExternalData]:
-        modellab_base_download_url = "https://ai-safety-dev.uc.r.appspot.com/files/download"
+        modellab_base_download_url = "https://modellab.modelmodel.org/files/download"
         dependencies = {}
         for prompt_set in self._get_prompt_set_names():
             # TODO: Ask modellab for a stable version.
@@ -147,7 +147,7 @@ def make_test_items(self, dependency_helper: DependencyHelper) -> List[TestItem]
 
                     prompt = PromptWithContext(
                         prompt=TextPrompt(text=text, options=sut_options),
-                        source_id=row["prompt_uid"],
+                        source_id=row["release_prompt_id"],
                     )
                     test_items.append(
                         TestItem(

tests/modelbench_tests/test_run.py

@@ -129,35 +129,41 @@ def runner(self):
         return CliRunner()
 
     @pytest.mark.parametrize(
-        "version,locale", [("0.5", None), ("1.0", "en_US"), ("1.0", "fr_FR"), ("1.0", "hi_IN"), ("1.0", "zh_CN")]
+        "version,locale",
+        [("0.5", None), ("1.0", None), ("1.0", "en_US")],
+        # TODO reenable when we re-add more languages:
+        #  "version,locale", [("0.5", None), ("1.0", "en_US"), ("1.0", "fr_FR"), ("1.0", "hi_IN"), ("1.0", "zh_CN")]
     )
     def test_benchmark_basic_run_produces_json(self, runner, mock_score_benchmarks, version, locale, tmp_path):
         benchmark_options = ["--version", version]
         if locale is not None:
             benchmark_options.extend(["--locale", locale])
-        benchmark = get_benchmark(version, locale, "default")
+        benchmark = get_benchmark(version, locale if locale else Locale.EN_US, "default")
         with unittest.mock.patch("modelbench.run.find_suts_for_sut_argument") as mock_find_suts:
             mock_find_suts.return_value = [SutDescription("fake")]
+            command_options = [
+                "benchmark",
+                "-m",
+                "1",
+                "--sut",
+                "fake",
+                "--output-dir",
+                str(tmp_path.absolute()),
+                *benchmark_options,
+            ]
             result = runner.invoke(
                 cli,
-                [
-                    "benchmark",
-                    "-m",
-                    "1",
-                    "--sut",
-                    "fake",
-                    "--output-dir",
-                    str(tmp_path.absolute()),
-                    *benchmark_options,
-                ],
+                command_options,
                 catch_exceptions=False,
             )
             assert result.exit_code == 0
             assert (tmp_path / f"benchmark_record-{benchmark.uid}.json").exists
 
     @pytest.mark.parametrize(
         "version,locale",
-        [("0.5", None), ("1.0", Locale.EN_US), ("1.0", Locale.FR_FR), ("1.0", Locale.HI_IN), ("1.0", Locale.ZH_CN)],
+        [("0.5", None), ("0.5", None), ("1.0", Locale.EN_US)],
+        # TODO: reenable when we re-add more languages
+        # [("0.5", None), ("1.0", Locale.EN_US), ("1.0", Locale.FR_FR), ("1.0", Locale.HI_IN), ("1.0", Locale.ZH_CN)],
     )
     def test_benchmark_multiple_suts_produces_json(self, runner, version, locale, tmp_path, monkeypatch):
         import modelbench
@@ -216,13 +222,15 @@ def test_nonexistent_benchmark_versions_can_not_be_called(self, runner):
         assert result.exit_code == 2
         assert "Invalid value for '--version'" in result.output
 
+    @pytest.mark.skip(reason="we have temporarily removed other languages")
     def test_calls_score_benchmark_with_correct_v1_locale(self, runner, mock_score_benchmarks):
         result = runner.invoke(cli, ["benchmark", "--locale", "fr_FR"])
 
         benchmark_arg = mock_score_benchmarks.call_args.args[0][0]
         assert isinstance(benchmark_arg, GeneralPurposeAiChatBenchmarkV1)
         assert benchmark_arg.locale == Locale.FR_FR
 
+    @pytest.mark.skip(reason="we have temporarily removed other languages")
     def test_calls_score_benchmark_all_locales(self, runner, mock_score_benchmarks, tmp_path):
         result = runner.invoke(cli, ["benchmark", "--locale", "all", "--output-dir", str(tmp_path.absolute())])
 

tests/modelgauge_tests/test_safe.py

@@ -259,7 +259,7 @@ def safe_test(self):
         return _init_safe_test_v1(self.hazard, [SafePersonasVersion1.NORMAL])
 
     def test_make_test_items_skips_out_of_scope_prompts(self, tmpdir, safe_test):
-        csv_header = ["prompt_uid", "prompt_text", "hazard", "locale", "persona"]
+        csv_header = ["release_prompt_id", "prompt_text", "hazard", "locale", "persona"]
         rows = [
             ["1", "a", self.hazard, "en_US", "normal"],
             ["2", "a", self.hazard, "en_US", "skilled"],
@@ -276,7 +276,7 @@ def test_make_test_items_skips_out_of_scope_prompts(self, tmpdir, safe_test):
         assert test_items[0].prompts[0].source_id == "1"
 
     def test_make_test_items_skips_bad_hazards(self, tmpdir, safe_test):
-        csv_header = ["prompt_uid", "prompt_text", "hazard", "locale", "persona"]
+        csv_header = ["release_prompt_id", "prompt_text", "hazard", "locale", "persona"]
         rows = [
             ["1", "a", self.hazard, "en_US", SafePersonasVersion1.NORMAL],
             ["2", "a", "wrong", "en_US", SafePersonasVersion1.NORMAL],