[mieb] use Logistic Regression classifier for AbsTaskImageMultilabelC…

…lassification (#1420)

* use moc-lr classifier

* set n_experiments=5

* run dinov2 and some laion models

* add dinov2-giant results

mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py

@@ -9,7 +9,8 @@
 from sklearn.base import ClassifierMixin, clone
 from sklearn.metrics import f1_score, label_ranking_average_precision_score
 from sklearn.model_selection import train_test_split
-from sklearn.neighbors import KNeighborsClassifier
+from sklearn.multioutput import MultiOutputClassifier
+from sklearn.linear_model import LogisticRegression
 from sklearn.preprocessing import MultiLabelBinarizer
 
 from mteb.abstasks.TaskMetadata import HFSubset
@@ -52,7 +53,7 @@ class AbsTaskImageMultilabelClassification(AbsTask):
     image_column_name: str = "image"
     label_column_name: str = "labels"
 
-    classifier = KNeighborsClassifier(n_neighbors=5)
+    classifier = MultiOutputClassifier(estimator=LogisticRegression())
 
     def __init__(
         self,

mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py

@@ -54,4 +54,4 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification):
     label_column_name: str = "classes"
 
     # To be removed when we want full results
-    n_experiments: int = 1
+    n_experiments: int = 5

results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json

@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 71.94576096534729,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.3903069466882068,
+        "f1": 0.6257925811318822,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.5316062533656477,
+        "main_score": 0.3903069466882068,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.3905492730210016,
+            "f1": 0.6235333142876379,
+            "lrap": 0.5233687847783186
+          },
+          {
+            "accuracy": 0.401453957996769,
+            "f1": 0.6273491281205146,
+            "lrap": 0.5416105726081543
+          },
+          {
+            "accuracy": 0.37621163166397414,
+            "f1": 0.6161039070570788,
+            "lrap": 0.5324994951534777
+          },
+          {
+            "accuracy": 0.38126009693053314,
+            "f1": 0.6219616890190912,
+            "lrap": 0.5204900376952104
+          },
+          {
+            "accuracy": 0.4020597738287561,
+            "f1": 0.6400148671750886,
+            "lrap": 0.540062376593077
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}

results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json

@@ -1 +1 @@
-{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"}
+{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}

results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json

@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 408.0938787460327,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.3851373182552503,
+        "f1": 0.6315092630757977,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.5298672253634935,
+        "main_score": 0.3851373182552503,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.3939822294022617,
+            "f1": 0.6271304662163256,
+            "lrap": 0.53264393735416
+          },
+          {
+            "accuracy": 0.3945880452342488,
+            "f1": 0.6263229032757067,
+            "lrap": 0.5370607835218142
+          },
+          {
+            "accuracy": 0.3784329563812601,
+            "f1": 0.6339676248065612,
+            "lrap": 0.5346142411595813
+          },
+          {
+            "accuracy": 0.3689418416801292,
+            "f1": 0.6304914034835388,
+            "lrap": 0.5144189777418785
+          },
+          {
+            "accuracy": 0.3897415185783522,
+            "f1": 0.6396339175968564,
+            "lrap": 0.5305981870400338
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}

results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json

@@ -0,0 +1 @@
+{"name": "facebook/dinov2-giant", "revision": "611a9d42f2335e0f921f1e313ad3c1b7178d206d", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}

results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json

@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 146.90911436080933,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.35315024232633274,
+        "f1": 0.5817890400372185,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.49049121567043397,
+        "main_score": 0.35315024232633274,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.3667205169628433,
+            "f1": 0.591090519122089,
+            "lrap": 0.5039753859271227
+          },
+          {
+            "accuracy": 0.34995961227786754,
+            "f1": 0.5618180857706963,
+            "lrap": 0.48571452611739013
+          },
+          {
+            "accuracy": 0.34693053311793215,
+            "f1": 0.5774757520432393,
+            "lrap": 0.49124652216837067
+          },
+          {
+            "accuracy": 0.33582390953150243,
+            "f1": 0.5718289254309215,
+            "lrap": 0.4711502647639497
+          },
+          {
+            "accuracy": 0.36631663974151857,
+            "f1": 0.606731917819147,
+            "lrap": 0.5003693793753363
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}

results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json

@@ -1 +1 @@
-{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"}
+{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}

results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json

@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 52.176159620285034,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.4340064620355412,
+        "f1": 0.6759098897017933,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.5801337843295732,
+        "main_score": 0.4340064620355412,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.4184168012924071,
+            "f1": 0.6581607635477135,
+            "lrap": 0.563152373900564
+          },
+          {
+            "accuracy": 0.44810177705977383,
+            "f1": 0.6892652132242753,
+            "lrap": 0.5981864790881464
+          },
+          {
+            "accuracy": 0.44426494345718903,
+            "f1": 0.6797744937645096,
+            "lrap": 0.5992208535272061
+          },
+          {
+            "accuracy": 0.43073505654281097,
+            "f1": 0.6706176542382969,
+            "lrap": 0.5713583737210639
+          },
+          {
+            "accuracy": 0.42851373182552505,
+            "f1": 0.6817313237341718,
+            "lrap": 0.5687508414108857
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}

results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json

@@ -1 +1 @@
-{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"}
+{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"}

results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json

@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 39.63397932052612,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.5000807754442649,
+        "f1": 0.6763180027107614,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.6410416666666808,
+        "main_score": 0.5000807754442649,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.5038368336025848,
+            "f1": 0.6876380960919419,
+            "lrap": 0.6470522572249287
+          },
+          {
+            "accuracy": 0.5232229402261712,
+            "f1": 0.6850605274296695,
+            "lrap": 0.6585363938251828
+          },
+          {
+            "accuracy": 0.4888933764135703,
+            "f1": 0.6592473172415618,
+            "lrap": 0.6350399950637378
+          },
+          {
+            "accuracy": 0.502827140549273,
+            "f1": 0.6767015784006876,
+            "lrap": 0.6400778024591767
+          },
+          {
+            "accuracy": 0.48162358642972536,
+            "f1": 0.6729424943899456,
+            "lrap": 0.6245018847603783
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}

results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json

@@ -0,0 +1 @@
+{"name": "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", "revision": "f0e2ffa09cbadab3db6a261ec1ec56407ce42912", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"}

results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json

@@ -0,0 +1,48 @@
+{
+  "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50",
+  "evaluation_time": 106.33361673355103,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.16.5",
+  "scores": {
+    "test": [
+      {
+        "accuracy": 0.5382875605815831,
+        "f1": 0.7099573953664177,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "lrap": 0.6770555667743827,
+        "main_score": 0.5382875605815831,
+        "scores_per_experiment": [
+          {
+            "accuracy": 0.5484652665589661,
+            "f1": 0.7264512368883388,
+            "lrap": 0.6877928109854764
+          },
+          {
+            "accuracy": 0.5581583198707593,
+            "f1": 0.7220847968939508,
+            "lrap": 0.6972356847962836
+          },
+          {
+            "accuracy": 0.5282714054927302,
+            "f1": 0.6913016268300175,
+            "lrap": 0.6676300821217165
+          },
+          {
+            "accuracy": 0.5401857835218093,
+            "f1": 0.7000594523316099,
+            "lrap": 0.6753413323460928
+          },
+          {
+            "accuracy": 0.5163570274636511,
+            "f1": 0.7098898638881714,
+            "lrap": 0.6572779236223446
+          }
+        ]
+      }
+    ]
+  },
+  "task_name": "VOC2007"
+}

results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json

@@ -1 +1 @@
-{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"}
+{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"}