From f42bcec1e573783b1e3753228bb1eb9f52a96a30 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sat, 9 Nov 2024 18:38:58 +0000 Subject: [PATCH 1/4] use moc-lr classifier --- .../AbsTaskImageMultilabelClassification.py | 5 ++-- .../VOC2007.json | 28 +++++++++++++++++++ .../VOC2007.json | 18 ++++++------ .../VOC2007.json | 28 +++++++++++++++++++ 4 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json create mode 100644 results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json diff --git a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py index 3cbd33ab53..d680f5d990 100644 --- a/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py +++ b/mteb/abstasks/Image/AbsTaskImageMultilabelClassification.py @@ -9,7 +9,8 @@ from sklearn.base import ClassifierMixin, clone from sklearn.metrics import f1_score, label_ranking_average_precision_score from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier +from sklearn.multioutput import MultiOutputClassifier +from sklearn.linear_model import LogisticRegression from sklearn.preprocessing import MultiLabelBinarizer from mteb.abstasks.TaskMetadata import HFSubset @@ -52,7 +53,7 @@ class AbsTaskImageMultilabelClassification(AbsTask): image_column_name: str = "image" label_column_name: str = "labels" - classifier = KNeighborsClassifier(n_neighbors=5) + classifier = MultiOutputClassifier(estimator=LogisticRegression()) def __init__( self, diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json new file mode 100644 index 0000000000..46fb7c85b0 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 57.04910588264465, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5351373182552503, + "f1": 0.7048436479181375, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6794869637408174, + "main_score": 0.5351373182552503, + "scores_per_experiment": [ + { + "accuracy": 0.5351373182552503, + "f1": 0.7048436479181375, + "lrap": 0.6794869637408174 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json index dafbb1f01e..d547da378b 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json @@ -1,24 +1,24 @@ { "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 55.247910261154175, + "evaluation_time": 41.00741267204285, "kg_co2_emissions": null, - "mteb_version": "1.12.80", + "mteb_version": "1.16.5", "scores": { "test": [ { - "accuracy": 0.5026252019386107, - "f1": 0.6780762548765893, + "accuracy": 0.52140549273021, + "f1": 0.6981947346229871, "hf_subset": "default", "languages": [ "eng-Latn" ], - "lrap": 0.6642591096751196, - "main_score": 0.5026252019386107, + "lrap": 0.6660473882606517, + "main_score": 0.52140549273021, "scores_per_experiment": [ { - "accuracy": 0.5026252019386107, - "f1": 0.6780762548765893, - "lrap": 0.6642591096751196 + "accuracy": 0.52140549273021, + "f1": 0.6981947346229871, + "lrap": 0.6660473882606517 } ] } diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json new file mode 100644 index 0000000000..19871dcdec --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json @@ -0,0 +1,28 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 129.10212993621826, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5310985460420032, + "f1": 0.696133632301483, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6760691527553577, + "main_score": 0.5310985460420032, + "scores_per_experiment": [ + { + "accuracy": 0.5310985460420032, + "f1": 0.696133632301483, + "lrap": 0.6760691527553577 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file From b5e9fef46a04acaf7a2ac79535e01f18127dc26e Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sat, 9 Nov 2024 18:59:18 +0000 Subject: [PATCH 2/4] set n_experiments=5 --- .../eng/PascalVOC2007.py | 2 +- .../VOC2007.json | 34 +++++++++++++++---- .../model_meta.json | 2 +- .../VOC2007.json | 30 +++++++++++++--- .../VOC2007.json | 30 +++++++++++++--- .../model_meta.json | 2 +- 6 files changed, 80 insertions(+), 20 deletions(-) diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 4ceae17ff9..0ce54da8eb 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -54,4 +54,4 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): label_column_name: str = "classes" # To be removed when we want full results - n_experiments: int = 1 + n_experiments: int = 5 diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json index 46fb7c85b0..50150f4e71 100644 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/VOC2007.json @@ -1,24 +1,44 @@ { "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 57.04910588264465, + "evaluation_time": 65.64880728721619, "kg_co2_emissions": null, "mteb_version": "1.16.5", "scores": { "test": [ { - "accuracy": 0.5351373182552503, - "f1": 0.7048436479181375, + "accuracy": 0.523546042003231, + "f1": 0.6886687471443521, "hf_subset": "default", "languages": [ "eng-Latn" ], - "lrap": 0.6794869637408174, - "main_score": 0.5351373182552503, + "lrap": 0.6682884019924772, + "main_score": 0.523546042003231, "scores_per_experiment": [ { "accuracy": 0.5351373182552503, - "f1": 0.7048436479181375, - "lrap": 0.6794869637408174 + "f1": 0.7049899989401454, + "lrap": 0.6795329608688017 + }, + { + "accuracy": 0.5300888529886915, + "f1": 0.6875495899200211, + "lrap": 0.6688218003949185 + }, + { + "accuracy": 0.5119143780290791, + "f1": 0.673686551264662, + "lrap": 0.6658084275713686 + }, + { + "accuracy": 0.5216074313408724, + "f1": 0.6802113731239721, + "lrap": 0.6663693681565408 + }, + { + "accuracy": 0.5189822294022617, + "f1": 0.6969062224729599, + "lrap": 0.6609094529707563 } ] } diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json index 0b8a2cecb0..2bf553db27 100644 --- a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json @@ -1 +1 @@ -{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file +{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json index d547da378b..44558b6f3d 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/VOC2007.json @@ -1,24 +1,44 @@ { "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 41.00741267204285, + "evaluation_time": 48.62162375450134, "kg_co2_emissions": null, "mteb_version": "1.16.5", "scores": { "test": [ { - "accuracy": 0.52140549273021, - "f1": 0.6981947346229871, + "accuracy": 0.5166801292407108, + "f1": 0.6851483326294392, "hf_subset": "default", "languages": [ "eng-Latn" ], - "lrap": 0.6660473882606517, - "main_score": 0.52140549273021, + "lrap": 0.6602331269072138, + "main_score": 0.5166801292407108, "scores_per_experiment": [ { "accuracy": 0.52140549273021, "f1": 0.6981947346229871, "lrap": 0.6660473882606517 + }, + { + "accuracy": 0.5357431340872375, + "f1": 0.6897556454047876, + "lrap": 0.6734304882427027 + }, + { + "accuracy": 0.5102988691437803, + "f1": 0.6665291163888459, + "lrap": 0.6582500336564507 + }, + { + "accuracy": 0.5149434571890146, + "f1": 0.6906546111504063, + "lrap": 0.6572852158499527 + }, + { + "accuracy": 0.5010096930533118, + "f1": 0.6806075555801697, + "lrap": 0.6461525085263116 } ] } diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json index 19871dcdec..151085b676 100644 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/VOC2007.json @@ -1,24 +1,44 @@ { "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", - "evaluation_time": 129.10212993621826, + "evaluation_time": 143.64798212051392, "kg_co2_emissions": null, "mteb_version": "1.16.5", "scores": { "test": [ { - "accuracy": 0.5310985460420032, - "f1": 0.696133632301483, + "accuracy": 0.5198707592891761, + "f1": 0.6818774389774342, "hf_subset": "default", "languages": [ "eng-Latn" ], - "lrap": 0.6760691527553577, - "main_score": 0.5310985460420032, + "lrap": 0.6617288749775786, + "main_score": 0.5198707592891761, "scores_per_experiment": [ { "accuracy": 0.5310985460420032, "f1": 0.696133632301483, "lrap": 0.6760691527553577 + }, + { + "accuracy": 0.5274636510500808, + "f1": 0.6754522344136807, + "lrap": 0.6636039310716377 + }, + { + "accuracy": 0.507875605815832, + "f1": 0.6621408823160164, + "lrap": 0.6531808135882394 + }, + { + "accuracy": 0.5359450726978998, + "f1": 0.6914673370418792, + "lrap": 0.673578576557188 + }, + { + "accuracy": 0.4969709208400646, + "f1": 0.684193108814112, + "lrap": 0.6422119009154702 } ] } diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json index 8c47f12e0c..69b52c701c 100644 --- a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/model_meta.json @@ -1 +1 @@ -{"name": "openai/clip-vit-large-patch14", "revision": "32bd64288804d66eefd0ccbe215aa642df71cc41", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "CLIPModelWrapper"} \ No newline at end of file +{"name": "openai/clip-vit-large-patch14", "revision": "32bd64288804d66eefd0ccbe215aa642df71cc41", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "CLIPModelWrapper"} \ No newline at end of file From 2021987f7e59629624b5019a48aeddb218264074 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sat, 9 Nov 2024 19:22:49 +0000 Subject: [PATCH 3/4] run dinov2 and some laion models --- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../model_meta.json | 1 + .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 1 + .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- .../VOC2007.json | 48 +++++++++++++++++++ .../model_meta.json | 2 +- 13 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json create mode 100644 results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json create mode 100644 results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json create mode 100644 results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json create mode 100644 results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json create mode 100644 results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json create mode 100644 results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json create mode 100644 results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json new file mode 100644 index 0000000000..1240b81aa4 --- /dev/null +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 71.94576096534729, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.3903069466882068, + "f1": 0.6257925811318822, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.5316062533656477, + "main_score": 0.3903069466882068, + "scores_per_experiment": [ + { + "accuracy": 0.3905492730210016, + "f1": 0.6235333142876379, + "lrap": 0.5233687847783186 + }, + { + "accuracy": 0.401453957996769, + "f1": 0.6273491281205146, + "lrap": 0.5416105726081543 + }, + { + "accuracy": 0.37621163166397414, + "f1": 0.6161039070570788, + "lrap": 0.5324994951534777 + }, + { + "accuracy": 0.38126009693053314, + "f1": 0.6219616890190912, + "lrap": 0.5204900376952104 + }, + { + "accuracy": 0.4020597738287561, + "f1": 0.6400148671750886, + "lrap": 0.540062376593077 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json index 6ae5f937c4..4d9fd59747 100644 --- a/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json +++ b/results-mieb/facebook__dinov2-base/f9e44c814b77203eaa57a6bdbbd535f21ede1415/model_meta.json @@ -1 +1 @@ -{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file +{"name": "facebook/dinov2-base", "revision": "f9e44c814b77203eaa57a6bdbbd535f21ede1415", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json new file mode 100644 index 0000000000..87a8751f95 --- /dev/null +++ b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/model_meta.json @@ -0,0 +1 @@ +{"name": "facebook/dinov2-giant", "revision": "611a9d42f2335e0f921f1e313ad3c1b7178d206d", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json new file mode 100644 index 0000000000..a9111cc617 --- /dev/null +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 146.90911436080933, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.35315024232633274, + "f1": 0.5817890400372185, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.49049121567043397, + "main_score": 0.35315024232633274, + "scores_per_experiment": [ + { + "accuracy": 0.3667205169628433, + "f1": 0.591090519122089, + "lrap": 0.5039753859271227 + }, + { + "accuracy": 0.34995961227786754, + "f1": 0.5618180857706963, + "lrap": 0.48571452611739013 + }, + { + "accuracy": 0.34693053311793215, + "f1": 0.5774757520432393, + "lrap": 0.49124652216837067 + }, + { + "accuracy": 0.33582390953150243, + "f1": 0.5718289254309215, + "lrap": 0.4711502647639497 + }, + { + "accuracy": 0.36631663974151857, + "f1": 0.606731917819147, + "lrap": 0.5003693793753363 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json index 1de6c7be30..a84e0519fe 100644 --- a/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json +++ b/results-mieb/facebook__dinov2-large/47b73eefe95e8d44ec3623f8890bd894b6ea2d6c/model_meta.json @@ -1 +1 @@ -{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file +{"name": "facebook/dinov2-large", "revision": "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json new file mode 100644 index 0000000000..e036d66bea --- /dev/null +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 52.176159620285034, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.4340064620355412, + "f1": 0.6759098897017933, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.5801337843295732, + "main_score": 0.4340064620355412, + "scores_per_experiment": [ + { + "accuracy": 0.4184168012924071, + "f1": 0.6581607635477135, + "lrap": 0.563152373900564 + }, + { + "accuracy": 0.44810177705977383, + "f1": 0.6892652132242753, + "lrap": 0.5981864790881464 + }, + { + "accuracy": 0.44426494345718903, + "f1": 0.6797744937645096, + "lrap": 0.5992208535272061 + }, + { + "accuracy": 0.43073505654281097, + "f1": 0.6706176542382969, + "lrap": 0.5713583737210639 + }, + { + "accuracy": 0.42851373182552505, + "f1": 0.6817313237341718, + "lrap": 0.5687508414108857 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json index 2eca9d0047..5789f8b54f 100644 --- a/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json +++ b/results-mieb/facebook__dinov2-small/ed25f3a31f01632728cabb09d1542f84ab7b0056/model_meta.json @@ -1 +1 @@ -{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "DINOModelWrapper"} \ No newline at end of file +{"name": "facebook/dinov2-small", "revision": "ed25f3a31f01632728cabb09d1542f84ab7b0056", "release_date": "2023-07-18", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "DINOModelWrapper"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json new file mode 100644 index 0000000000..4257296d6f --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 39.63397932052612, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5000807754442649, + "f1": 0.6763180027107614, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6410416666666808, + "main_score": 0.5000807754442649, + "scores_per_experiment": [ + { + "accuracy": 0.5038368336025848, + "f1": 0.6876380960919419, + "lrap": 0.6470522572249287 + }, + { + "accuracy": 0.5232229402261712, + "f1": 0.6850605274296695, + "lrap": 0.6585363938251828 + }, + { + "accuracy": 0.4888933764135703, + "f1": 0.6592473172415618, + "lrap": 0.6350399950637378 + }, + { + "accuracy": 0.502827140549273, + "f1": 0.6767015784006876, + "lrap": 0.6400778024591767 + }, + { + "accuracy": 0.48162358642972536, + "f1": 0.6729424943899456, + "lrap": 0.6245018847603783 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json new file mode 100644 index 0000000000..5b161eaaff --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-B-32-DataComp.XL-s13B-b90K/f0e2ffa09cbadab3db6a261ec1ec56407ce42912/model_meta.json @@ -0,0 +1 @@ +{"name": "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K", "revision": "f0e2ffa09cbadab3db6a261ec1ec56407ce42912", "release_date": "2023-04-26", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json new file mode 100644 index 0000000000..b0d4cc3355 --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 106.33361673355103, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5382875605815831, + "f1": 0.7099573953664177, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6770555667743827, + "main_score": 0.5382875605815831, + "scores_per_experiment": [ + { + "accuracy": 0.5484652665589661, + "f1": 0.7264512368883388, + "lrap": 0.6877928109854764 + }, + { + "accuracy": 0.5581583198707593, + "f1": 0.7220847968939508, + "lrap": 0.6972356847962836 + }, + { + "accuracy": 0.5282714054927302, + "f1": 0.6913016268300175, + "lrap": 0.6676300821217165 + }, + { + "accuracy": 0.5401857835218093, + "f1": 0.7000594523316099, + "lrap": 0.6753413323460928 + }, + { + "accuracy": 0.5163570274636511, + "f1": 0.7098898638881714, + "lrap": 0.6572779236223446 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json index 36d3659874..4ba9199948 100644 --- a/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json +++ b/results-mieb/laion__CLIP-ViT-H-14-laion2B-s32B-b79K/de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b/model_meta.json @@ -1 +1 @@ -{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file +{"name": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "revision": "de081ac0a0ca8dc9d1533eed1ae884bb8ae1404b", "release_date": "2022-09-15", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json new file mode 100644 index 0000000000..0cc887a6ba --- /dev/null +++ b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 139.95830011367798, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.5426898222940226, + "f1": 0.7116762127648723, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.6792292676359876, + "main_score": 0.5426898222940226, + "scores_per_experiment": [ + { + "accuracy": 0.5609854604200323, + "f1": 0.7310080393154441, + "lrap": 0.6944724914737191 + }, + { + "accuracy": 0.559168012924071, + "f1": 0.7125996315013976, + "lrap": 0.6941942649434737 + }, + { + "accuracy": 0.5220113085621971, + "f1": 0.6999211686407317, + "lrap": 0.6659638081134593 + }, + { + "accuracy": 0.5387722132471728, + "f1": 0.702612150503481, + "lrap": 0.6734136600251451 + }, + { + "accuracy": 0.5325121163166397, + "f1": 0.7122400738633065, + "lrap": 0.6681021136241404 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file diff --git a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json index 5465d2e2eb..852d6e492a 100644 --- a/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json +++ b/results-mieb/laion__CLIP-ViT-g-14-laion2B-s34B-b88K/15efd0f6ac0c40c0f9da7becca03c974d7012604/model_meta.json @@ -1 +1 @@ -{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "openclip_loader"} \ No newline at end of file +{"name": "laion/CLIP-ViT-g-14-laion2B-s34B-b88K", "revision": "15efd0f6ac0c40c0f9da7becca03c974d7012604", "release_date": "2023-03-06", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instuctions": null, "zero_shot_benchmarks": null, "loader": "openclip_loader"} \ No newline at end of file From e4002e944e30ab365b46422ed0bb4d1ae5295509 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sat, 9 Nov 2024 19:29:10 +0000 Subject: [PATCH 4/4] add dinov2-giant results --- .../VOC2007.json | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json diff --git a/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json new file mode 100644 index 0000000000..730e213e2b --- /dev/null +++ b/results-mieb/facebook__dinov2-giant/611a9d42f2335e0f921f1e313ad3c1b7178d206d/VOC2007.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "evaluation_time": 408.0938787460327, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "accuracy": 0.3851373182552503, + "f1": 0.6315092630757977, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "lrap": 0.5298672253634935, + "main_score": 0.3851373182552503, + "scores_per_experiment": [ + { + "accuracy": 0.3939822294022617, + "f1": 0.6271304662163256, + "lrap": 0.53264393735416 + }, + { + "accuracy": 0.3945880452342488, + "f1": 0.6263229032757067, + "lrap": 0.5370607835218142 + }, + { + "accuracy": 0.3784329563812601, + "f1": 0.6339676248065612, + "lrap": 0.5346142411595813 + }, + { + "accuracy": 0.3689418416801292, + "f1": 0.6304914034835388, + "lrap": 0.5144189777418785 + }, + { + "accuracy": 0.3897415185783522, + "f1": 0.6396339175968564, + "lrap": 0.5305981870400338 + } + ] + } + ] + }, + "task_name": "VOC2007" +} \ No newline at end of file