fix version string; switch mistral-large to MistralAI and disable it …

…in the VertexAI sut, with comment explaining why
mlcommons · Nov 20, 2024 · c450624 · c450624
1 parent e10a470
commit c450624
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 3 deletions.
diff --git a/plugins/mistral/modelgauge/suts/mistral_sut.py b/plugins/mistral/modelgauge/suts/mistral_sut.py
@@ -67,6 +67,12 @@ def translate_response(self, request: MistralAIRequest, response: MistralAIRespo
 MISTRAL_API_KEY = InjectSecret(MistralAIAPIKey)
 
 model_name = "ministral-8b"
-model_version = "latest"
-model_uid = f"{model_name}-{model_version}"
+model_version = "2410"
+model_uid = f"{model_name}-latest"
+SUTS.register(MistralAISut, model_uid, model_uid, model_version, MISTRAL_API_KEY)
+
+
+model_name = "mistral-large"
+model_version = "2411"
+model_uid = f"{model_name}-latest"
 SUTS.register(MistralAISut, model_uid, model_uid, model_version, MISTRAL_API_KEY)
diff --git a/plugins/vertexai/modelgauge/suts/vertexai_mistral_sut.py b/plugins/vertexai/modelgauge/suts/vertexai_mistral_sut.py
@@ -101,4 +101,6 @@ def translate_response(self, request: VertexAIMistralRequest, response: VertexAI
 model_name = "mistral-large"
 model_version = "2407"
 model_uid = f"{model_name}-{model_version}"
-SUTS.register(VertexAIMistralAISut, model_uid, model_name, model_version, VERTEX_KEY, VERTEX_PROJECT_ID, VERTEX_REGION)
+# Uncomment this if you want to run mistral-large on VertexAI.
+# See plugins/mistral for an implementation that runs on MistralAI instead.
+# SUTS.register(VertexAIMistralAISut, model_uid, model_name, model_version, VERTEX_KEY, VERTEX_PROJECT_ID, VERTEX_REGION)