feat: evaluate arctic v2 models in MTEB(Medical) (#66)

* feat: evaluate arctic v2 models in MTEB(Medical) * feat: track co2 emissions * feat: new model_meta.json * fix: set `use_instructions` to true
embeddings-benchmark · Dec 11, 2024 · edb303f · edb303f
1 parent e9db296
commit edb303f
Show file tree

Hide file tree

Showing 26 changed files with 15,102 additions and 0 deletions.
diff --git a/...lake-arctic-embed-l-v2.0/edc2df7b6c25794b340229ca082e7c78782e6374/CMedQAv2-reranking.json b/...lake-arctic-embed-l-v2.0/edc2df7b6c25794b340229ca082e7c78782e6374/CMedQAv2-reranking.json
@@ -0,0 +1,26 @@
+{
+  "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
+  "task_name": "CMedQAv2-reranking",
+  "mteb_version": "1.24.0",
+  "scores": {
+    "test": [
+      {
+        "map": 0.70069,
+        "mrr": 0.748533,
+        "nAUC_map_max": 0.598828,
+        "nAUC_map_std": 0.31886,
+        "nAUC_map_diff1": 0.541323,
+        "nAUC_mrr_max": 0.694851,
+        "nAUC_mrr_std": 0.382436,
+        "nAUC_mrr_diff1": 0.634483,
+        "main_score": 0.70069,
+        "hf_subset": "default",
+        "languages": [
+          "cmn-Hans"
+        ]
+      }
+    ]
+  },
+  "evaluation_time": 25.8760507106781,
+  "kg_co2_emissions": 0.0010782871468822304
+}