diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py index 6ba591cf12..eb4a4d692c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py @@ -10,8 +10,8 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve captions based on images.", reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", dataset={ - "path": "JamieSJS/flickr30k", - "revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629", + "path": "isaacchung/flickr30ki2t", + "revision": "6984df6bd4380034e7766d9a992d8907df363efb", }, type="Any2AnyRetrieval", category="i2t", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py index be56a554e4..b9775528c8 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py @@ -10,8 +10,8 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): description="Retrieve images based on captions.", reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31", dataset={ - "path": "JamieSJS/flickr30k", - "revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629", + "path": "isaacchung/flickr30kt2i", + "revision": "e819702b287bfbe084e129a61f308a802b7c108e", }, type="Any2AnyRetrieval", category="t2i", @@ -39,17 +39,3 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval): "n_samples": {"default": 31014}, # qrels }, ) - - def load_data(self, **kwargs): - super().load_data(**kwargs) - # swap corpus and query - for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]): - self.queries[split], self.corpus[split] = ( - self.corpus[split], - self.queries[split], - ) - self.relevant_docs[split] = { - cid: {qid: score} - for qid, cid_score in self.relevant_docs[split].items() - for cid, score in cid_score.items() - } diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..b5e7e4bff6 --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 12.655046463012695, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.806, + "cv_recall_at_10": 0.981, + "cv_recall_at_100": 0.999, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.994, + "cv_recall_at_3": 0.922, + "cv_recall_at_5": 0.961, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.75656, + "map_at_1": 0.1612, + "map_at_10": 0.65737, + "map_at_100": 0.6941, + "map_at_1000": 0.69494, + "map_at_20": 0.68213, + "map_at_3": 0.42077, + "map_at_5": 0.5824, + "mrr_at_1": 0.806, + "mrr_at_10": 0.8702051587301588, + "mrr_at_100": 0.8713558453277496, + "mrr_at_1000": 0.8713621346359256, + "mrr_at_20": 0.8711786018505061, + "mrr_at_3": 0.8581666666666669, + "mrr_at_5": 0.8672666666666671, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.3580765639589976, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_10_diff1": 0.4775664651825631, + "nauc_cv_recall_at_10_max": 0.7300113027667245, + "nauc_cv_recall_at_10_std": -0.05830753353973558, + "nauc_cv_recall_at_1_diff1": 0.3443688532171371, + "nauc_cv_recall_at_1_max": 0.4866993124852272, + "nauc_cv_recall_at_1_std": -0.1314705005806373, + "nauc_cv_recall_at_20_diff1": 0.5864456893868585, + "nauc_cv_recall_at_20_max": 0.9782135076252602, + "nauc_cv_recall_at_20_std": 0.5981948334889493, + "nauc_cv_recall_at_3_diff1": 0.3899961694079347, + "nauc_cv_recall_at_3_max": 0.5140295434413065, + "nauc_cv_recall_at_3_std": -0.12323732911968292, + "nauc_cv_recall_at_5_diff1": 0.48242715889774873, + "nauc_cv_recall_at_5_max": 0.5554957025545274, + "nauc_cv_recall_at_5_std": -0.16390145801910372, + "nauc_map_at_1000_diff1": 0.07860412999292432, + "nauc_map_at_1000_max": 0.5193407172877343, + "nauc_map_at_1000_std": -0.03554548015806451, + "nauc_map_at_100_diff1": 0.07860332936377284, + "nauc_map_at_100_max": 0.5202209025325473, + "nauc_map_at_100_std": -0.03402804669053566, + "nauc_map_at_10_diff1": 0.06738608544555293, + "nauc_map_at_10_max": 0.5046712341845588, + "nauc_map_at_10_std": -0.052089661267638164, + "nauc_map_at_1_diff1": 0.344368853217137, + "nauc_map_at_1_max": 0.4866993124852267, + "nauc_map_at_1_std": -0.1314705005806365, + "nauc_map_at_20_diff1": 0.07540132773467392, + "nauc_map_at_20_max": 0.5194163232471378, + "nauc_map_at_20_std": -0.03561110594882118, + "nauc_map_at_3_diff1": 0.13827655728488158, + "nauc_map_at_3_max": 0.48609865047620054, + "nauc_map_at_3_std": -0.09536745328619929, + "nauc_map_at_5_diff1": 0.07059479448464502, + "nauc_map_at_5_max": 0.48086096767511555, + "nauc_map_at_5_std": -0.08160089788651494, + "nauc_mrr_at_1000_diff1": 0.36023929557475076, + "nauc_mrr_at_1000_max": 0.5035860304880058, + "nauc_mrr_at_1000_std": -0.1249299862258316, + "nauc_mrr_at_100_diff1": 0.3602391842803307, + "nauc_mrr_at_100_max": 0.5036115760084787, + "nauc_mrr_at_100_std": -0.12487209719821057, + "nauc_mrr_at_10_diff1": 0.3611514720783506, + "nauc_mrr_at_10_max": 0.5051263876997938, + "nauc_mrr_at_10_std": -0.1257752596646823, + "nauc_mrr_at_1_diff1": 0.3443688532171371, + "nauc_mrr_at_1_max": 0.4866993124852272, + "nauc_mrr_at_1_std": -0.1314705005806373, + "nauc_mrr_at_20_diff1": 0.3606869616253337, + "nauc_mrr_at_20_max": 0.5042795551778488, + "nauc_mrr_at_20_std": -0.1239507002161601, + "nauc_mrr_at_3_diff1": 0.35911327275172067, + "nauc_mrr_at_3_max": 0.49885073455301077, + "nauc_mrr_at_3_std": -0.12597845586993137, + "nauc_mrr_at_5_diff1": 0.36427948002844535, + "nauc_mrr_at_5_max": 0.5025432319868158, + "nauc_mrr_at_5_std": -0.12876002101412964, + "nauc_ndcg_at_1000_diff1": 0.1404445479237733, + "nauc_ndcg_at_1000_max": 0.5347818240932689, + "nauc_ndcg_at_1000_std": -0.03316587882098286, + "nauc_ndcg_at_100_diff1": 0.1358934054564792, + "nauc_ndcg_at_100_max": 0.5460446827666793, + "nauc_ndcg_at_100_std": -0.007831409654410723, + "nauc_ndcg_at_10_diff1": 0.10422124872393643, + "nauc_ndcg_at_10_max": 0.522718634663143, + "nauc_ndcg_at_10_std": -0.042784056691292496, + "nauc_ndcg_at_1_diff1": 0.3443688532171371, + "nauc_ndcg_at_1_max": 0.4866993124852272, + "nauc_ndcg_at_1_std": -0.1314705005806373, + "nauc_ndcg_at_20_diff1": 0.12477162836638274, + "nauc_ndcg_at_20_max": 0.5506038459831472, + "nauc_ndcg_at_20_std": -0.0058328462109728755, + "nauc_ndcg_at_3_diff1": 0.1727965348989873, + "nauc_ndcg_at_3_max": 0.49295039267335533, + "nauc_ndcg_at_3_std": -0.09392106684131309, + "nauc_ndcg_at_5_diff1": 0.1101745223344572, + "nauc_ndcg_at_5_max": 0.4913548608912362, + "nauc_ndcg_at_5_std": -0.08489198959710985, + "nauc_precision_at_1000_diff1": -0.08286647992530825, + "nauc_precision_at_1000_max": 0.854341736694646, + "nauc_precision_at_1000_std": 0.8109243697478463, + "nauc_precision_at_100_diff1": 0.02976321846328062, + "nauc_precision_at_100_max": 0.6817348214942566, + "nauc_precision_at_100_std": 0.4525517335603402, + "nauc_precision_at_10_diff1": 0.03152919182802702, + "nauc_precision_at_10_max": 0.5209015914034817, + "nauc_precision_at_10_std": 0.012785208286159547, + "nauc_precision_at_1_diff1": 0.3443688532171371, + "nauc_precision_at_1_max": 0.4866993124852272, + "nauc_precision_at_1_std": -0.1314705005806373, + "nauc_precision_at_20_diff1": 0.058403294374015056, + "nauc_precision_at_20_max": 0.5976698949057889, + "nauc_precision_at_20_std": 0.15342898310071018, + "nauc_precision_at_3_diff1": 0.13112541027685642, + "nauc_precision_at_3_max": 0.4910052478350366, + "nauc_precision_at_3_std": -0.0846106773972222, + "nauc_precision_at_5_diff1": 0.057600655958168046, + "nauc_precision_at_5_max": 0.4823384872903635, + "nauc_precision_at_5_std": -0.07416100661323119, + "nauc_recall_at_1000_diff1": -0.08286647992533601, + "nauc_recall_at_1000_max": 0.8543417366946594, + "nauc_recall_at_1000_std": 0.8109243697478747, + "nauc_recall_at_100_diff1": 0.029763218463283347, + "nauc_recall_at_100_max": 0.6817348214942569, + "nauc_recall_at_100_std": 0.4525517335603416, + "nauc_recall_at_10_diff1": 0.03152919182802702, + "nauc_recall_at_10_max": 0.5209015914034817, + "nauc_recall_at_10_std": 0.012785208286159547, + "nauc_recall_at_1_diff1": 0.344368853217137, + "nauc_recall_at_1_max": 0.4866993124852267, + "nauc_recall_at_1_std": -0.1314705005806365, + "nauc_recall_at_20_diff1": 0.058403294374015056, + "nauc_recall_at_20_max": 0.5976698949057889, + "nauc_recall_at_20_std": 0.15342898310071018, + "nauc_recall_at_3_diff1": 0.1311254102768568, + "nauc_recall_at_3_max": 0.49100524783503674, + "nauc_recall_at_3_std": -0.0846106773972213, + "nauc_recall_at_5_diff1": 0.057600655958168046, + "nauc_recall_at_5_max": 0.4823384872903635, + "nauc_recall_at_5_std": -0.07416100661323119, + "ndcg_at_1": 0.806, + "ndcg_at_10": 0.75656, + "ndcg_at_100": 0.82771, + "ndcg_at_1000": 0.83522, + "ndcg_at_20": 0.79537, + "ndcg_at_3": 0.7524, + "ndcg_at_5": 0.67901, + "precision_at_1": 0.806, + "precision_at_10": 0.3869, + "precision_at_100": 0.04818, + "precision_at_1000": 0.00499, + "precision_at_20": 0.2159, + "precision_at_3": 0.735, + "precision_at_5": 0.6344, + "recall_at_1": 0.1612, + "recall_at_10": 0.7738, + "recall_at_100": 0.9636, + "recall_at_1000": 0.9976, + "recall_at_20": 0.8636, + "recall_at_3": 0.441, + "recall_at_5": 0.6344 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..2f5c0c550a --- /dev/null +++ b/results-mieb/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 21.522692918777466, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.678, + "cv_recall_at_10": 0.9334, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9632, + "cv_recall_at_3": 0.8422, + "cv_recall_at_5": 0.89, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8094, + "map_at_1": 0.678, + "map_at_10": 0.76922, + "map_at_100": 0.77224, + "map_at_1000": 0.77227, + "map_at_20": 0.7714, + "map_at_3": 0.7522, + "map_at_5": 0.76322, + "mrr_at_1": 0.678, + "mrr_at_10": 0.7692153968253979, + "mrr_at_100": 0.7722356838203779, + "mrr_at_1000": 0.7722705791355338, + "mrr_at_20": 0.771396252397414, + "mrr_at_3": 0.752200000000002, + "mrr_at_5": 0.7632200000000005, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.7008714596949883, + "nauc_cv_recall_at_100_max": 0.7499999999999728, + "nauc_cv_recall_at_100_std": -0.37489884842825993, + "nauc_cv_recall_at_10_diff1": 0.5953937130407739, + "nauc_cv_recall_at_10_max": 0.5879408820585317, + "nauc_cv_recall_at_10_std": -0.2350459703400894, + "nauc_cv_recall_at_1_diff1": 0.7388944519574545, + "nauc_cv_recall_at_1_max": 0.5035033567670426, + "nauc_cv_recall_at_1_std": -0.10506388352655013, + "nauc_cv_recall_at_20_diff1": 0.6257002801120445, + "nauc_cv_recall_at_20_max": 0.6655198311208591, + "nauc_cv_recall_at_20_std": -0.19918402143466227, + "nauc_cv_recall_at_3_diff1": 0.6472713623239569, + "nauc_cv_recall_at_3_max": 0.5578714550455595, + "nauc_cv_recall_at_3_std": -0.13548415964522142, + "nauc_cv_recall_at_5_diff1": 0.6339326518340327, + "nauc_cv_recall_at_5_max": 0.5687226183317582, + "nauc_cv_recall_at_5_std": -0.15658878103255752, + "nauc_map_at_1000_diff1": 0.7108121576697071, + "nauc_map_at_1000_max": 0.5195954518905794, + "nauc_map_at_1000_std": -0.11880914292849726, + "nauc_map_at_100_diff1": 0.7108064511289186, + "nauc_map_at_100_max": 0.5196339191480183, + "nauc_map_at_100_std": -0.1188526782447686, + "nauc_map_at_10_diff1": 0.7089638573224181, + "nauc_map_at_10_max": 0.5198626919789708, + "nauc_map_at_10_std": -0.12106795154834728, + "nauc_map_at_1_diff1": 0.7388944519574545, + "nauc_map_at_1_max": 0.5035033567670426, + "nauc_map_at_1_std": -0.10506388352655013, + "nauc_map_at_20_diff1": 0.7104379728179321, + "nauc_map_at_20_max": 0.5201383461745716, + "nauc_map_at_20_std": -0.1193396676956454, + "nauc_map_at_3_diff1": 0.7078639395398761, + "nauc_map_at_3_max": 0.5200771632890915, + "nauc_map_at_3_std": -0.11708354472686522, + "nauc_map_at_5_diff1": 0.7087628559704173, + "nauc_map_at_5_max": 0.5201026397276295, + "nauc_map_at_5_std": -0.11894921046259736, + "nauc_mrr_at_1000_diff1": 0.7108121576697071, + "nauc_mrr_at_1000_max": 0.5195954518905794, + "nauc_mrr_at_1000_std": -0.11880914292849726, + "nauc_mrr_at_100_diff1": 0.7108064511289186, + "nauc_mrr_at_100_max": 0.5196339191480183, + "nauc_mrr_at_100_std": -0.1188526782447686, + "nauc_mrr_at_10_diff1": 0.7089638573224181, + "nauc_mrr_at_10_max": 0.5198626919789708, + "nauc_mrr_at_10_std": -0.12106795154834728, + "nauc_mrr_at_1_diff1": 0.7388944519574545, + "nauc_mrr_at_1_max": 0.5035033567670426, + "nauc_mrr_at_1_std": -0.10506388352655013, + "nauc_mrr_at_20_diff1": 0.7104379728179321, + "nauc_mrr_at_20_max": 0.5201383461745716, + "nauc_mrr_at_20_std": -0.1193396676956454, + "nauc_mrr_at_3_diff1": 0.7078639395398761, + "nauc_mrr_at_3_max": 0.5200771632890915, + "nauc_mrr_at_3_std": -0.11708354472686522, + "nauc_mrr_at_5_diff1": 0.7087628559704173, + "nauc_mrr_at_5_max": 0.5201026397276295, + "nauc_mrr_at_5_std": -0.11894921046259736, + "nauc_ndcg_at_1000_diff1": 0.7067180020680146, + "nauc_ndcg_at_1000_max": 0.5231648168494609, + "nauc_ndcg_at_1000_std": -0.12126906171818087, + "nauc_ndcg_at_100_diff1": 0.7066643145995184, + "nauc_ndcg_at_100_max": 0.5243091596475544, + "nauc_ndcg_at_100_std": -0.12259715494304686, + "nauc_ndcg_at_10_diff1": 0.6970018286625054, + "nauc_ndcg_at_10_max": 0.5270891134868764, + "nauc_ndcg_at_10_std": -0.13181013070978706, + "nauc_ndcg_at_1_diff1": 0.7388944519574545, + "nauc_ndcg_at_1_max": 0.5035033567670426, + "nauc_ndcg_at_1_std": -0.10506388352655013, + "nauc_ndcg_at_20_diff1": 0.7033385159678784, + "nauc_ndcg_at_20_max": 0.5287035840738418, + "nauc_ndcg_at_20_std": -0.12480446128344788, + "nauc_ndcg_at_3_diff1": 0.6960393372878148, + "nauc_ndcg_at_3_max": 0.5273632969286177, + "nauc_ndcg_at_3_std": -0.12075786696488118, + "nauc_ndcg_at_5_diff1": 0.6970584144874077, + "nauc_ndcg_at_5_max": 0.5276209122981296, + "nauc_ndcg_at_5_std": -0.1247281409835251, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7008714596949908, + "nauc_precision_at_100_max": 0.749999999999983, + "nauc_precision_at_100_std": -0.37489884842824256, + "nauc_precision_at_10_diff1": 0.5953937130407729, + "nauc_precision_at_10_max": 0.5879408820585307, + "nauc_precision_at_10_std": -0.23504597034008565, + "nauc_precision_at_1_diff1": 0.7388944519574545, + "nauc_precision_at_1_max": 0.5035033567670426, + "nauc_precision_at_1_std": -0.10506388352655013, + "nauc_precision_at_20_diff1": 0.6257002801120534, + "nauc_precision_at_20_max": 0.6655198311208589, + "nauc_precision_at_20_std": -0.19918402143465108, + "nauc_precision_at_3_diff1": 0.6472713623239564, + "nauc_precision_at_3_max": 0.5578714550455599, + "nauc_precision_at_3_std": -0.13548415964522165, + "nauc_precision_at_5_diff1": 0.6339326518340342, + "nauc_precision_at_5_max": 0.568722618331759, + "nauc_precision_at_5_std": -0.15658878103255713, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7008714596949883, + "nauc_recall_at_100_max": 0.7499999999999728, + "nauc_recall_at_100_std": -0.37489884842825993, + "nauc_recall_at_10_diff1": 0.5953937130407739, + "nauc_recall_at_10_max": 0.5879408820585317, + "nauc_recall_at_10_std": -0.2350459703400894, + "nauc_recall_at_1_diff1": 0.7388944519574545, + "nauc_recall_at_1_max": 0.5035033567670426, + "nauc_recall_at_1_std": -0.10506388352655013, + "nauc_recall_at_20_diff1": 0.6257002801120445, + "nauc_recall_at_20_max": 0.6655198311208591, + "nauc_recall_at_20_std": -0.19918402143466227, + "nauc_recall_at_3_diff1": 0.6472713623239569, + "nauc_recall_at_3_max": 0.5578714550455595, + "nauc_recall_at_3_std": -0.13548415964522142, + "nauc_recall_at_5_diff1": 0.6339326518340327, + "nauc_recall_at_5_max": 0.5687226183317582, + "nauc_recall_at_5_std": -0.15658878103255752, + "ndcg_at_1": 0.678, + "ndcg_at_10": 0.8094, + "ndcg_at_100": 0.82289, + "ndcg_at_1000": 0.82368, + "ndcg_at_20": 0.81708, + "ndcg_at_3": 0.77539, + "ndcg_at_5": 0.79516, + "precision_at_1": 0.678, + "precision_at_10": 0.09334, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04816, + "precision_at_3": 0.28073, + "precision_at_5": 0.178, + "recall_at_1": 0.678, + "recall_at_10": 0.9334, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "recall_at_20": 0.9632, + "recall_at_3": 0.8422, + "recall_at_5": 0.89 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..a90131d30b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 13.612479209899902, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.82, + "cv_recall_at_10": 0.986, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.994, + "cv_recall_at_3": 0.931, + "cv_recall_at_5": 0.97, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.74273, + "map_at_1": 0.164, + "map_at_10": 0.63451, + "map_at_100": 0.67573, + "map_at_1000": 0.67665, + "map_at_20": 0.66234, + "map_at_3": 0.4162, + "map_at_5": 0.56242, + "mrr_at_1": 0.82, + "mrr_at_10": 0.8819170634920639, + "mrr_at_100": 0.8826449941029878, + "mrr_at_1000": 0.8826449941029878, + "mrr_at_20": 0.8824860171691058, + "mrr_at_3": 0.8710000000000003, + "mrr_at_5": 0.8797000000000006, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.19891289849274035, + "nauc_cv_recall_at_10_max": 0.5265772975856937, + "nauc_cv_recall_at_10_std": 0.3455382152861277, + "nauc_cv_recall_at_1_diff1": 0.4480945037999776, + "nauc_cv_recall_at_1_max": 0.3922045379447073, + "nauc_cv_recall_at_1_std": -0.12505231853728344, + "nauc_cv_recall_at_20_diff1": 0.3018985371926489, + "nauc_cv_recall_at_20_max": 0.6433239962651742, + "nauc_cv_recall_at_20_std": 0.9537037037037052, + "nauc_cv_recall_at_3_diff1": 0.2737655448652898, + "nauc_cv_recall_at_3_max": 0.4099108242330735, + "nauc_cv_recall_at_3_std": -0.1392102734813745, + "nauc_cv_recall_at_5_diff1": 0.2479458450046711, + "nauc_cv_recall_at_5_max": 0.45326797385621426, + "nauc_cv_recall_at_5_std": 0.024727668845316842, + "nauc_map_at_1000_diff1": 0.12299692538637216, + "nauc_map_at_1000_max": 0.40383972971788085, + "nauc_map_at_1000_std": -0.03877241232841388, + "nauc_map_at_100_diff1": 0.1224103741610186, + "nauc_map_at_100_max": 0.4045083510460462, + "nauc_map_at_100_std": -0.036813876580229825, + "nauc_map_at_10_diff1": 0.11422185007928484, + "nauc_map_at_10_max": 0.39191511510815347, + "nauc_map_at_10_std": -0.056740898404007624, + "nauc_map_at_1_diff1": 0.4480945037999775, + "nauc_map_at_1_max": 0.39220453794470633, + "nauc_map_at_1_std": -0.12505231853728427, + "nauc_map_at_20_diff1": 0.11978487595245653, + "nauc_map_at_20_max": 0.4010120010081494, + "nauc_map_at_20_std": -0.038459183289332906, + "nauc_map_at_3_diff1": 0.19814408531027977, + "nauc_map_at_3_max": 0.4018668277901667, + "nauc_map_at_3_std": -0.12036750531181802, + "nauc_map_at_5_diff1": 0.11645918188256225, + "nauc_map_at_5_max": 0.38608545461679483, + "nauc_map_at_5_std": -0.08112260515216518, + "nauc_mrr_at_1000_diff1": 0.4117843439551849, + "nauc_mrr_at_1000_max": 0.39603449433424776, + "nauc_mrr_at_1000_std": -0.1224537228873275, + "nauc_mrr_at_100_diff1": 0.4117843439551849, + "nauc_mrr_at_100_max": 0.39603449433424776, + "nauc_mrr_at_100_std": -0.1224537228873275, + "nauc_mrr_at_10_diff1": 0.40988550434587345, + "nauc_mrr_at_10_max": 0.39584154636889374, + "nauc_mrr_at_10_std": -0.12111989031979221, + "nauc_mrr_at_1_diff1": 0.4480945037999776, + "nauc_mrr_at_1_max": 0.3922045379447073, + "nauc_mrr_at_1_std": -0.12505231853728344, + "nauc_mrr_at_20_diff1": 0.41146243588617815, + "nauc_mrr_at_20_max": 0.3962597833666116, + "nauc_mrr_at_20_std": -0.12095286988661123, + "nauc_mrr_at_3_diff1": 0.39902923729559436, + "nauc_mrr_at_3_max": 0.3943872791235097, + "nauc_mrr_at_3_std": -0.13306276199025832, + "nauc_mrr_at_5_diff1": 0.4076485858771895, + "nauc_mrr_at_5_max": 0.39577465779193577, + "nauc_mrr_at_5_std": -0.12265544640203102, + "nauc_ndcg_at_1000_diff1": 0.1844301804081895, + "nauc_ndcg_at_1000_max": 0.42130933536662635, + "nauc_ndcg_at_1000_std": -0.035065914910614955, + "nauc_ndcg_at_100_diff1": 0.17496821020126946, + "nauc_ndcg_at_100_max": 0.43029672300582816, + "nauc_ndcg_at_100_std": -0.002507684096230846, + "nauc_ndcg_at_10_diff1": 0.14481662979912127, + "nauc_ndcg_at_10_max": 0.40520681410698756, + "nauc_ndcg_at_10_std": -0.04220162909542145, + "nauc_ndcg_at_1_diff1": 0.4480945037999776, + "nauc_ndcg_at_1_max": 0.3922045379447073, + "nauc_ndcg_at_1_std": -0.12505231853728344, + "nauc_ndcg_at_20_diff1": 0.15803572920290762, + "nauc_ndcg_at_20_max": 0.42054444917147027, + "nauc_ndcg_at_20_std": -0.0019159299606582648, + "nauc_ndcg_at_3_diff1": 0.21765219679209533, + "nauc_ndcg_at_3_max": 0.40379267362903226, + "nauc_ndcg_at_3_std": -0.12304328773334444, + "nauc_ndcg_at_5_diff1": 0.1462495129764977, + "nauc_ndcg_at_5_max": 0.3934225656825324, + "nauc_ndcg_at_5_std": -0.0786882910212851, + "nauc_precision_at_1000_diff1": 0.62427637721762, + "nauc_precision_at_1000_max": 0.7399626517273514, + "nauc_precision_at_1000_std": 0.7899159663865152, + "nauc_precision_at_100_diff1": 0.028283567165904248, + "nauc_precision_at_100_max": 0.5215659239456432, + "nauc_precision_at_100_std": 0.5279993895443634, + "nauc_precision_at_10_diff1": 0.06152272326044776, + "nauc_precision_at_10_max": 0.3828014005531608, + "nauc_precision_at_10_std": 0.018326040537019386, + "nauc_precision_at_1_diff1": 0.4480945037999776, + "nauc_precision_at_1_max": 0.3922045379447073, + "nauc_precision_at_1_std": -0.12505231853728344, + "nauc_precision_at_20_diff1": 0.06089982228042279, + "nauc_precision_at_20_max": 0.4065587877654105, + "nauc_precision_at_20_std": 0.1612227106912349, + "nauc_precision_at_3_diff1": 0.16178283766113552, + "nauc_precision_at_3_max": 0.3991300609614105, + "nauc_precision_at_3_std": -0.12286918672342817, + "nauc_precision_at_5_diff1": 0.08039511083398675, + "nauc_precision_at_5_max": 0.3789339125780607, + "nauc_precision_at_5_std": -0.05814725911556473, + "nauc_recall_at_1000_diff1": 0.624276377217619, + "nauc_recall_at_1000_max": 0.7399626517273947, + "nauc_recall_at_1000_std": 0.7899159663865074, + "nauc_recall_at_100_diff1": 0.028283567165905115, + "nauc_recall_at_100_max": 0.5215659239456432, + "nauc_recall_at_100_std": 0.5279993895443671, + "nauc_recall_at_10_diff1": 0.06152272326044776, + "nauc_recall_at_10_max": 0.3828014005531608, + "nauc_recall_at_10_std": 0.018326040537019386, + "nauc_recall_at_1_diff1": 0.4480945037999775, + "nauc_recall_at_1_max": 0.39220453794470633, + "nauc_recall_at_1_std": -0.12505231853728427, + "nauc_recall_at_20_diff1": 0.06089982228042279, + "nauc_recall_at_20_max": 0.4065587877654105, + "nauc_recall_at_20_std": 0.1612227106912349, + "nauc_recall_at_3_diff1": 0.161782837661136, + "nauc_recall_at_3_max": 0.3991300609614111, + "nauc_recall_at_3_std": -0.12286918672342777, + "nauc_recall_at_5_diff1": 0.08039511083398675, + "nauc_recall_at_5_max": 0.3789339125780607, + "nauc_recall_at_5_std": -0.05814725911556473, + "ndcg_at_1": 0.82, + "ndcg_at_10": 0.74273, + "ndcg_at_100": 0.82168, + "ndcg_at_1000": 0.82961, + "ndcg_at_20": 0.78527, + "ndcg_at_3": 0.7496, + "ndcg_at_5": 0.6661, + "precision_at_1": 0.82, + "precision_at_10": 0.3759, + "precision_at_100": 0.04818, + "precision_at_1000": 0.00499, + "precision_at_20": 0.2125, + "precision_at_3": 0.72567, + "precision_at_5": 0.6134, + "recall_at_1": 0.164, + "recall_at_10": 0.7518, + "recall_at_100": 0.9636, + "recall_at_1000": 0.999, + "recall_at_20": 0.85, + "recall_at_3": 0.4354, + "recall_at_5": 0.6134 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..8b907392b6 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 22.22239637374878, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.6214, + "cv_recall_at_10": 0.915, + "cv_recall_at_100": 0.9932, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.9532, + "cv_recall_at_3": 0.7944, + "cv_recall_at_5": 0.854, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.76748, + "map_at_1": 0.6214, + "map_at_10": 0.72031, + "map_at_100": 0.72409, + "map_at_1000": 0.72412, + "map_at_20": 0.72299, + "map_at_3": 0.69867, + "map_at_5": 0.71205, + "mrr_at_1": 0.6214, + "mrr_at_10": 0.7203121428571442, + "mrr_at_100": 0.724085885840924, + "mrr_at_1000": 0.7241243831788744, + "mrr_at_20": 0.7229865462203786, + "mrr_at_3": 0.6986666666666691, + "mrr_at_5": 0.7120466666666677, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6346185533036849, + "nauc_cv_recall_at_100_max": 0.6848052946668853, + "nauc_cv_recall_at_100_std": 0.4664002856044359, + "nauc_cv_recall_at_10_diff1": 0.5758345691217669, + "nauc_cv_recall_at_10_max": 0.5616378316032299, + "nauc_cv_recall_at_10_std": 0.15670456417861245, + "nauc_cv_recall_at_1_diff1": 0.6964868193650136, + "nauc_cv_recall_at_1_max": 0.4658840817263219, + "nauc_cv_recall_at_1_std": 0.06009948824657468, + "nauc_cv_recall_at_20_diff1": 0.5942604962212811, + "nauc_cv_recall_at_20_max": 0.5904757914561845, + "nauc_cv_recall_at_20_std": 0.18555826889159976, + "nauc_cv_recall_at_3_diff1": 0.6016933324728311, + "nauc_cv_recall_at_3_max": 0.4996714444125018, + "nauc_cv_recall_at_3_std": 0.11144390402265514, + "nauc_cv_recall_at_5_diff1": 0.596326854581594, + "nauc_cv_recall_at_5_max": 0.5435354775022344, + "nauc_cv_recall_at_5_std": 0.1647132585408802, + "nauc_map_at_1000_diff1": 0.6681451778281712, + "nauc_map_at_1000_max": 0.48153054698905345, + "nauc_map_at_1000_std": 0.08084200415260853, + "nauc_map_at_100_diff1": 0.6681421969134242, + "nauc_map_at_100_max": 0.4815602459122235, + "nauc_map_at_100_std": 0.08089746305416494, + "nauc_map_at_10_diff1": 0.66637927944471, + "nauc_map_at_10_max": 0.4824993978185994, + "nauc_map_at_10_std": 0.08173000407846276, + "nauc_map_at_1_diff1": 0.6964868193650136, + "nauc_map_at_1_max": 0.4658840817263219, + "nauc_map_at_1_std": 0.06009948824657468, + "nauc_map_at_20_diff1": 0.6677072473838044, + "nauc_map_at_20_max": 0.4820334855898056, + "nauc_map_at_20_std": 0.08112000061939652, + "nauc_map_at_3_diff1": 0.6629064040013647, + "nauc_map_at_3_max": 0.47830764546568383, + "nauc_map_at_3_std": 0.07908104385148582, + "nauc_map_at_5_diff1": 0.6651150515375697, + "nauc_map_at_5_max": 0.48320424406293105, + "nauc_map_at_5_std": 0.0841376160310789, + "nauc_mrr_at_1000_diff1": 0.6681451778281712, + "nauc_mrr_at_1000_max": 0.48153054698905345, + "nauc_mrr_at_1000_std": 0.08084200415260853, + "nauc_mrr_at_100_diff1": 0.6681421969134242, + "nauc_mrr_at_100_max": 0.4815602459122235, + "nauc_mrr_at_100_std": 0.08089746305416494, + "nauc_mrr_at_10_diff1": 0.66637927944471, + "nauc_mrr_at_10_max": 0.4824993978185994, + "nauc_mrr_at_10_std": 0.08173000407846276, + "nauc_mrr_at_1_diff1": 0.6964868193650136, + "nauc_mrr_at_1_max": 0.4658840817263219, + "nauc_mrr_at_1_std": 0.06009948824657468, + "nauc_mrr_at_20_diff1": 0.6677072473838044, + "nauc_mrr_at_20_max": 0.4820334855898056, + "nauc_mrr_at_20_std": 0.08112000061939652, + "nauc_mrr_at_3_diff1": 0.6629064040013647, + "nauc_mrr_at_3_max": 0.47830764546568383, + "nauc_mrr_at_3_std": 0.07908104385148582, + "nauc_mrr_at_5_diff1": 0.6651150515375697, + "nauc_mrr_at_5_max": 0.48320424406293105, + "nauc_mrr_at_5_std": 0.0841376160310789, + "nauc_ndcg_at_1000_diff1": 0.6644414321594331, + "nauc_ndcg_at_1000_max": 0.48494330141081976, + "nauc_ndcg_at_1000_std": 0.08499549812045258, + "nauc_ndcg_at_100_diff1": 0.6643084643885205, + "nauc_ndcg_at_100_max": 0.4859086696461096, + "nauc_ndcg_at_100_std": 0.08683324562972554, + "nauc_ndcg_at_10_diff1": 0.6558132357644398, + "nauc_ndcg_at_10_max": 0.4912776808457643, + "nauc_ndcg_at_10_std": 0.09056604431255127, + "nauc_ndcg_at_1_diff1": 0.6964868193650136, + "nauc_ndcg_at_1_max": 0.4658840817263219, + "nauc_ndcg_at_1_std": 0.06009948824657468, + "nauc_ndcg_at_20_diff1": 0.6611087197877977, + "nauc_ndcg_at_20_max": 0.48954890256829625, + "nauc_ndcg_at_20_std": 0.08876502997875993, + "nauc_ndcg_at_3_diff1": 0.6501669512812934, + "nauc_ndcg_at_3_max": 0.4827750854554256, + "nauc_ndcg_at_3_std": 0.08585341799650138, + "nauc_ndcg_at_5_diff1": 0.6534210015801749, + "nauc_ndcg_at_5_max": 0.49293204109100786, + "nauc_ndcg_at_5_std": 0.09712640857173467, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6346185533036952, + "nauc_precision_at_100_max": 0.6848052946668934, + "nauc_precision_at_100_std": 0.46640028560445, + "nauc_precision_at_10_diff1": 0.5758345691217664, + "nauc_precision_at_10_max": 0.5616378316032302, + "nauc_precision_at_10_std": 0.1567045641786131, + "nauc_precision_at_1_diff1": 0.6964868193650136, + "nauc_precision_at_1_max": 0.4658840817263219, + "nauc_precision_at_1_std": 0.06009948824657468, + "nauc_precision_at_20_diff1": 0.59426049622128, + "nauc_precision_at_20_max": 0.5904757914561855, + "nauc_precision_at_20_std": 0.18555826889160457, + "nauc_precision_at_3_diff1": 0.6016933324728304, + "nauc_precision_at_3_max": 0.49967144441250244, + "nauc_precision_at_3_std": 0.1114439040226548, + "nauc_precision_at_5_diff1": 0.5963268545815941, + "nauc_precision_at_5_max": 0.5435354775022362, + "nauc_precision_at_5_std": 0.16471325854088018, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6346185533036849, + "nauc_recall_at_100_max": 0.6848052946668853, + "nauc_recall_at_100_std": 0.4664002856044359, + "nauc_recall_at_10_diff1": 0.5758345691217669, + "nauc_recall_at_10_max": 0.5616378316032299, + "nauc_recall_at_10_std": 0.15670456417861245, + "nauc_recall_at_1_diff1": 0.6964868193650136, + "nauc_recall_at_1_max": 0.4658840817263219, + "nauc_recall_at_1_std": 0.06009948824657468, + "nauc_recall_at_20_diff1": 0.5942604962212811, + "nauc_recall_at_20_max": 0.5904757914561845, + "nauc_recall_at_20_std": 0.18555826889159976, + "nauc_recall_at_3_diff1": 0.6016933324728311, + "nauc_recall_at_3_max": 0.4996714444125018, + "nauc_recall_at_3_std": 0.11144390402265514, + "nauc_recall_at_5_diff1": 0.596326854581594, + "nauc_recall_at_5_max": 0.5435354775022344, + "nauc_recall_at_5_std": 0.1647132585408802, + "ndcg_at_1": 0.6214, + "ndcg_at_10": 0.76748, + "ndcg_at_100": 0.78475, + "ndcg_at_1000": 0.78565, + "ndcg_at_20": 0.77717, + "ndcg_at_3": 0.7233, + "ndcg_at_5": 0.74763, + "precision_at_1": 0.6214, + "precision_at_10": 0.0915, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "precision_at_20": 0.04766, + "precision_at_3": 0.2648, + "precision_at_5": 0.1708, + "recall_at_1": 0.6214, + "recall_at_10": 0.915, + "recall_at_100": 0.9932, + "recall_at_1000": 1.0, + "recall_at_20": 0.9532, + "recall_at_3": 0.7944, + "recall_at_5": 0.854 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json new file mode 100644 index 0000000000..c27e7fd151 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kI2TRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb", + "evaluation_time": 11.527727365493774, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.782, + "cv_recall_at_10": 0.979, + "cv_recall_at_100": 0.998, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.994, + "cv_recall_at_3": 0.919, + "cv_recall_at_5": 0.951, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.71535, + "map_at_1": 0.1564, + "map_at_10": 0.6016, + "map_at_100": 0.64142, + "map_at_1000": 0.64252, + "map_at_20": 0.62777, + "map_at_3": 0.38783, + "map_at_5": 0.52369, + "mrr_at_1": 0.782, + "mrr_at_10": 0.8533876984126986, + "mrr_at_100": 0.8545914967755422, + "mrr_at_1000": 0.8546010863340449, + "mrr_at_20": 0.8544995892995896, + "mrr_at_3": 0.842166666666667, + "mrr_at_5": 0.849416666666667, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.6790382819794335, + "nauc_cv_recall_at_100_max": 0.6136788048552808, + "nauc_cv_recall_at_100_std": 0.5401493930905519, + "nauc_cv_recall_at_10_diff1": 0.5072695744964671, + "nauc_cv_recall_at_10_max": 0.6689342403628125, + "nauc_cv_recall_at_10_std": 0.3829754123871726, + "nauc_cv_recall_at_1_diff1": 0.488735446449004, + "nauc_cv_recall_at_1_max": 0.5141133834291504, + "nauc_cv_recall_at_1_std": -0.03618247455901551, + "nauc_cv_recall_at_20_diff1": 0.34002489884843273, + "nauc_cv_recall_at_20_max": 0.6156240273887368, + "nauc_cv_recall_at_20_std": 0.7568471833177612, + "nauc_cv_recall_at_3_diff1": 0.5087895240400684, + "nauc_cv_recall_at_3_max": 0.5634459545134927, + "nauc_cv_recall_at_3_std": 0.10033890099249525, + "nauc_cv_recall_at_5_diff1": 0.5434650050496416, + "nauc_cv_recall_at_5_max": 0.5627870195697343, + "nauc_cv_recall_at_5_std": 0.2778158882600648, + "nauc_map_at_1000_diff1": 0.15465269435050452, + "nauc_map_at_1000_max": 0.46711312406477384, + "nauc_map_at_1000_std": 0.048575938256388396, + "nauc_map_at_100_diff1": 0.15431140329951903, + "nauc_map_at_100_max": 0.46719979617191026, + "nauc_map_at_100_std": 0.05052363203080754, + "nauc_map_at_10_diff1": 0.1529787812866786, + "nauc_map_at_10_max": 0.45711233128671735, + "nauc_map_at_10_std": 0.02449973700621325, + "nauc_map_at_1_diff1": 0.4887354464490032, + "nauc_map_at_1_max": 0.5141133834291495, + "nauc_map_at_1_std": -0.03618247455901751, + "nauc_map_at_20_diff1": 0.15076154146537132, + "nauc_map_at_20_max": 0.46444349461535234, + "nauc_map_at_20_std": 0.04479449662456778, + "nauc_map_at_3_diff1": 0.21351903584890045, + "nauc_map_at_3_max": 0.45698690208877696, + "nauc_map_at_3_std": -0.010887987588539814, + "nauc_map_at_5_diff1": 0.14578035683987364, + "nauc_map_at_5_max": 0.4515880427226945, + "nauc_map_at_5_std": -0.0026874220062081747, + "nauc_mrr_at_1000_diff1": 0.4893310611891473, + "nauc_mrr_at_1000_max": 0.5261133355559909, + "nauc_mrr_at_1000_std": 0.0020315610859939267, + "nauc_mrr_at_100_diff1": 0.4893377672062905, + "nauc_mrr_at_100_max": 0.5261248596328446, + "nauc_mrr_at_100_std": 0.0020657882998518138, + "nauc_mrr_at_10_diff1": 0.48988261824797846, + "nauc_mrr_at_10_max": 0.5277240085344459, + "nauc_mrr_at_10_std": 0.00453451162925253, + "nauc_mrr_at_1_diff1": 0.488735446449004, + "nauc_mrr_at_1_max": 0.5141133834291504, + "nauc_mrr_at_1_std": -0.03618247455901551, + "nauc_mrr_at_20_diff1": 0.48916248000131224, + "nauc_mrr_at_20_max": 0.5262057614175848, + "nauc_mrr_at_20_std": 0.0026394141831601174, + "nauc_mrr_at_3_diff1": 0.48985167550576125, + "nauc_mrr_at_3_max": 0.5300441674581229, + "nauc_mrr_at_3_std": 0.0027730733995076284, + "nauc_mrr_at_5_diff1": 0.4918762189399869, + "nauc_mrr_at_5_max": 0.52702946776199, + "nauc_mrr_at_5_std": 0.011169701853700587, + "nauc_ndcg_at_1000_diff1": 0.22766089836536801, + "nauc_ndcg_at_1000_max": 0.492189505568212, + "nauc_ndcg_at_1000_std": 0.06504915436760614, + "nauc_ndcg_at_100_diff1": 0.2199335774609376, + "nauc_ndcg_at_100_max": 0.4953948669765448, + "nauc_ndcg_at_100_std": 0.0980744945605265, + "nauc_ndcg_at_10_diff1": 0.2044319581978452, + "nauc_ndcg_at_10_max": 0.4763899911130591, + "nauc_ndcg_at_10_std": 0.04503381614955524, + "nauc_ndcg_at_1_diff1": 0.488735446449004, + "nauc_ndcg_at_1_max": 0.5141133834291504, + "nauc_ndcg_at_1_std": -0.03618247455901551, + "nauc_ndcg_at_20_diff1": 0.200495407478228, + "nauc_ndcg_at_20_max": 0.4889831247980535, + "nauc_ndcg_at_20_std": 0.0867421312607528, + "nauc_ndcg_at_3_diff1": 0.25423026936814114, + "nauc_ndcg_at_3_max": 0.4728682169678516, + "nauc_ndcg_at_3_std": 0.001298277045082591, + "nauc_ndcg_at_5_diff1": 0.19418768049527063, + "nauc_ndcg_at_5_max": 0.46699616414823136, + "nauc_ndcg_at_5_std": 0.01093649505873361, + "nauc_precision_at_1000_diff1": 0.12759103641456715, + "nauc_precision_at_1000_max": 0.4844070961717251, + "nauc_precision_at_1000_std": 0.7185340802988052, + "nauc_precision_at_100_diff1": 0.08522333406731375, + "nauc_precision_at_100_max": 0.48179115797715183, + "nauc_precision_at_100_std": 0.5722441057258686, + "nauc_precision_at_10_diff1": 0.13469873984556588, + "nauc_precision_at_10_max": 0.4428990975706562, + "nauc_precision_at_10_std": 0.08769758273258854, + "nauc_precision_at_1_diff1": 0.488735446449004, + "nauc_precision_at_1_max": 0.5141133834291504, + "nauc_precision_at_1_std": -0.03618247455901551, + "nauc_precision_at_20_diff1": 0.09427425323646658, + "nauc_precision_at_20_max": 0.4578914983843951, + "nauc_precision_at_20_std": 0.2235341685316957, + "nauc_precision_at_3_diff1": 0.2010575428533153, + "nauc_precision_at_3_max": 0.46236331299632105, + "nauc_precision_at_3_std": 0.013547375486496331, + "nauc_precision_at_5_diff1": 0.1317349745546741, + "nauc_precision_at_5_max": 0.4500305482173576, + "nauc_precision_at_5_std": 0.022958692069647763, + "nauc_recall_at_1000_diff1": 0.12759103641454075, + "nauc_recall_at_1000_max": 0.48440709617180283, + "nauc_recall_at_1000_std": 0.7185340802988699, + "nauc_recall_at_100_diff1": 0.0852233340673131, + "nauc_recall_at_100_max": 0.48179115797715144, + "nauc_recall_at_100_std": 0.5722441057258676, + "nauc_recall_at_10_diff1": 0.13469873984556588, + "nauc_recall_at_10_max": 0.4428990975706562, + "nauc_recall_at_10_std": 0.08769758273258854, + "nauc_recall_at_1_diff1": 0.4887354464490032, + "nauc_recall_at_1_max": 0.5141133834291495, + "nauc_recall_at_1_std": -0.03618247455901751, + "nauc_recall_at_20_diff1": 0.09427425323646658, + "nauc_recall_at_20_max": 0.4578914983843951, + "nauc_recall_at_20_std": 0.2235341685316957, + "nauc_recall_at_3_diff1": 0.20105754285331554, + "nauc_recall_at_3_max": 0.4623633129963219, + "nauc_recall_at_3_std": 0.013547375486496333, + "nauc_recall_at_5_diff1": 0.1317349745546741, + "nauc_recall_at_5_max": 0.4500305482173576, + "nauc_recall_at_5_std": 0.022958692069647763, + "ndcg_at_1": 0.782, + "ndcg_at_10": 0.71535, + "ndcg_at_100": 0.79531, + "ndcg_at_1000": 0.80538, + "ndcg_at_20": 0.75694, + "ndcg_at_3": 0.70872, + "ndcg_at_5": 0.63009, + "precision_at_1": 0.782, + "precision_at_10": 0.3677, + "precision_at_100": 0.04762, + "precision_at_1000": 0.00499, + "precision_at_20": 0.2081, + "precision_at_3": 0.68767, + "precision_at_5": 0.5814, + "recall_at_1": 0.1564, + "recall_at_10": 0.7354, + "recall_at_100": 0.9524, + "recall_at_1000": 0.998, + "recall_at_20": 0.8324, + "recall_at_3": 0.4126, + "recall_at_5": 0.5814 + } + ] + }, + "task_name": "Flickr30kI2TRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json new file mode 100644 index 0000000000..8ab9b00d39 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/Flickr30kT2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "e819702b287bfbe084e129a61f308a802b7c108e", + "evaluation_time": 19.96243405342102, + "kg_co2_emissions": null, + "mteb_version": "1.16.5", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.582, + "cv_recall_at_10": 0.892, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.94, + "cv_recall_at_3": 0.7656, + "cv_recall_at_5": 0.8288, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.73691, + "map_at_1": 0.582, + "map_at_10": 0.6872, + "map_at_100": 0.69194, + "map_at_1000": 0.692, + "map_at_20": 0.69052, + "map_at_3": 0.664, + "map_at_5": 0.6786, + "mrr_at_1": 0.582, + "mrr_at_10": 0.6872000793650814, + "mrr_at_100": 0.6919418796590151, + "mrr_at_1000": 0.6920043726572235, + "mrr_at_20": 0.6905210107419258, + "mrr_at_3": 0.6640000000000026, + "mrr_at_5": 0.6786000000000014, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.5202707749766565, + "nauc_cv_recall_at_100_max": 0.5908309990662858, + "nauc_cv_recall_at_100_std": 0.2939682539682632, + "nauc_cv_recall_at_10_diff1": 0.5617933093550299, + "nauc_cv_recall_at_10_max": 0.5589071830149802, + "nauc_cv_recall_at_10_std": 0.2244342982854361, + "nauc_cv_recall_at_1_diff1": 0.696381939657304, + "nauc_cv_recall_at_1_max": 0.47579049630166126, + "nauc_cv_recall_at_1_std": 0.0856787171723342, + "nauc_cv_recall_at_20_diff1": 0.5454154995331454, + "nauc_cv_recall_at_20_max": 0.575666044195457, + "nauc_cv_recall_at_20_std": 0.27880018674136237, + "nauc_cv_recall_at_3_diff1": 0.6071203712031832, + "nauc_cv_recall_at_3_max": 0.5034522530911553, + "nauc_cv_recall_at_3_std": 0.1294339186732062, + "nauc_cv_recall_at_5_diff1": 0.5786903531517902, + "nauc_cv_recall_at_5_max": 0.5299681868096371, + "nauc_cv_recall_at_5_std": 0.18341665359380663, + "nauc_map_at_1000_diff1": 0.6637636015118767, + "nauc_map_at_1000_max": 0.48813186871534664, + "nauc_map_at_1000_std": 0.10620231745962286, + "nauc_map_at_100_diff1": 0.663725501715657, + "nauc_map_at_100_max": 0.48815298386872974, + "nauc_map_at_100_std": 0.10623045894817788, + "nauc_map_at_10_diff1": 0.6621084063953249, + "nauc_map_at_10_max": 0.48930142276760086, + "nauc_map_at_10_std": 0.10756855111381708, + "nauc_map_at_1_diff1": 0.696381939657304, + "nauc_map_at_1_max": 0.47579049630166126, + "nauc_map_at_1_std": 0.0856787171723342, + "nauc_map_at_20_diff1": 0.6630269532116994, + "nauc_map_at_20_max": 0.48862647994909303, + "nauc_map_at_20_std": 0.10717682940880707, + "nauc_map_at_3_diff1": 0.662292666771372, + "nauc_map_at_3_max": 0.48547341513990666, + "nauc_map_at_3_std": 0.10081169237632849, + "nauc_map_at_5_diff1": 0.6602863231828925, + "nauc_map_at_5_max": 0.4883925967971498, + "nauc_map_at_5_std": 0.10706113722330816, + "nauc_mrr_at_1000_diff1": 0.6637636015118767, + "nauc_mrr_at_1000_max": 0.48813186871534664, + "nauc_mrr_at_1000_std": 0.10620231745962286, + "nauc_mrr_at_100_diff1": 0.663725501715657, + "nauc_mrr_at_100_max": 0.48815298386872974, + "nauc_mrr_at_100_std": 0.10623045894817788, + "nauc_mrr_at_10_diff1": 0.6621084063953249, + "nauc_mrr_at_10_max": 0.48930142276760086, + "nauc_mrr_at_10_std": 0.10756855111381708, + "nauc_mrr_at_1_diff1": 0.696381939657304, + "nauc_mrr_at_1_max": 0.47579049630166126, + "nauc_mrr_at_1_std": 0.0856787171723342, + "nauc_mrr_at_20_diff1": 0.6630269532116994, + "nauc_mrr_at_20_max": 0.48862647994909303, + "nauc_mrr_at_20_std": 0.10717682940880707, + "nauc_mrr_at_3_diff1": 0.662292666771372, + "nauc_mrr_at_3_max": 0.48547341513990666, + "nauc_mrr_at_3_std": 0.10081169237632849, + "nauc_mrr_at_5_diff1": 0.6602863231828925, + "nauc_mrr_at_5_max": 0.4883925967971498, + "nauc_mrr_at_5_std": 0.10706113722330816, + "nauc_ndcg_at_1000_diff1": 0.6588020949056204, + "nauc_ndcg_at_1000_max": 0.4911070818710271, + "nauc_ndcg_at_1000_std": 0.11159626078786719, + "nauc_ndcg_at_100_diff1": 0.6578430110166047, + "nauc_ndcg_at_100_max": 0.49175784358270214, + "nauc_ndcg_at_100_std": 0.11271907006804692, + "nauc_ndcg_at_10_diff1": 0.6491199341823203, + "nauc_ndcg_at_10_max": 0.49780129062969325, + "nauc_ndcg_at_10_std": 0.1217823152584022, + "nauc_ndcg_at_1_diff1": 0.696381939657304, + "nauc_ndcg_at_1_max": 0.47579049630166126, + "nauc_ndcg_at_1_std": 0.0856787171723342, + "nauc_ndcg_at_20_diff1": 0.6527562952947522, + "nauc_ndcg_at_20_max": 0.4954501005931264, + "nauc_ndcg_at_20_std": 0.12020249054744403, + "nauc_ndcg_at_3_diff1": 0.6504859796432384, + "nauc_ndcg_at_3_max": 0.4892786954353797, + "nauc_ndcg_at_3_std": 0.10686134169700189, + "nauc_ndcg_at_5_diff1": 0.6458666500847972, + "nauc_ndcg_at_5_max": 0.4954671444369836, + "nauc_ndcg_at_5_std": 0.11996492669439957, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.5202707749766482, + "nauc_precision_at_100_max": 0.5908309990662891, + "nauc_precision_at_100_std": 0.29396825396825177, + "nauc_precision_at_10_diff1": 0.5617933093550302, + "nauc_precision_at_10_max": 0.5589071830149824, + "nauc_precision_at_10_std": 0.22443429828543696, + "nauc_precision_at_1_diff1": 0.696381939657304, + "nauc_precision_at_1_max": 0.47579049630166126, + "nauc_precision_at_1_std": 0.0856787171723342, + "nauc_precision_at_20_diff1": 0.545415499533147, + "nauc_precision_at_20_max": 0.5756660441954571, + "nauc_precision_at_20_std": 0.27880018674136353, + "nauc_precision_at_3_diff1": 0.6071203712031833, + "nauc_precision_at_3_max": 0.5034522530911553, + "nauc_precision_at_3_std": 0.1294339186732061, + "nauc_precision_at_5_diff1": 0.5786903531517906, + "nauc_precision_at_5_max": 0.5299681868096384, + "nauc_precision_at_5_std": 0.183416653593807, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5202707749766565, + "nauc_recall_at_100_max": 0.5908309990662858, + "nauc_recall_at_100_std": 0.2939682539682632, + "nauc_recall_at_10_diff1": 0.5617933093550299, + "nauc_recall_at_10_max": 0.5589071830149802, + "nauc_recall_at_10_std": 0.2244342982854361, + "nauc_recall_at_1_diff1": 0.696381939657304, + "nauc_recall_at_1_max": 0.47579049630166126, + "nauc_recall_at_1_std": 0.0856787171723342, + "nauc_recall_at_20_diff1": 0.5454154995331454, + "nauc_recall_at_20_max": 0.575666044195457, + "nauc_recall_at_20_std": 0.27880018674136237, + "nauc_recall_at_3_diff1": 0.6071203712031832, + "nauc_recall_at_3_max": 0.5034522530911553, + "nauc_recall_at_3_std": 0.1294339186732062, + "nauc_recall_at_5_diff1": 0.5786903531517902, + "nauc_recall_at_5_max": 0.5299681868096371, + "nauc_recall_at_5_std": 0.18341665359380663, + "ndcg_at_1": 0.582, + "ndcg_at_10": 0.73691, + "ndcg_at_100": 0.75858, + "ndcg_at_1000": 0.75994, + "ndcg_at_20": 0.74904, + "ndcg_at_3": 0.69014, + "ndcg_at_5": 0.71631, + "precision_at_1": 0.582, + "precision_at_10": 0.0892, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "precision_at_20": 0.047, + "precision_at_3": 0.2552, + "precision_at_5": 0.16576, + "recall_at_1": 0.582, + "recall_at_10": 0.892, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "recall_at_20": 0.94, + "recall_at_3": 0.7656, + "recall_at_5": 0.8288 + } + ] + }, + "task_name": "Flickr30kT2IRetrieval" +} \ No newline at end of file diff --git a/scripts/data/flickr30k/build_flickr_30k_i2t.py b/scripts/data/flickr30k/build_flickr_30k_i2t.py new file mode 100644 index 0000000000..2abd1f1682 --- /dev/null +++ b/scripts/data/flickr30k/build_flickr_30k_i2t.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import os + +from datasets import Dataset, DatasetDict, load_dataset +from tqdm import tqdm + +WRITE_TOK = os.environ["HF_TOKEN"] + +eval_split = "test" +data_raw = load_dataset("clip-benchmark/wds_flickr30k")[eval_split] + + +## i2t +queries_ = {"id": [], "modality": [], "image": []} +corpus_ = {"id": [], "modality": [], "text": []} +relevant_docs_ = {"query-id": [], "corpus-id": [], "score": []} + +for row in tqdm(data_raw, total=len(data_raw)): + image = row["jpg"] + texts = row["txt"].split("\n") + key = row["__key__"] + query_id = f"q_{key}" + queries_["id"].append(query_id) + queries_["image"].append(image) + queries_["modality"].append("image") + + for i, text in enumerate(texts): + doc_id = f"d_{key}_{i}" + corpus_["id"].append(doc_id) + corpus_["text"].append(text) + corpus_["modality"].append("text") + + relevant_docs_["query-id"].append(query_id) + relevant_docs_["corpus-id"].append(doc_id) + relevant_docs_["score"].append(1) + +corpus = Dataset.from_dict(corpus_) +queries = Dataset.from_dict(queries_) +relevant_docs = Dataset.from_dict(relevant_docs_) + +corpus = DatasetDict({"corpus": corpus}) +queries = DatasetDict({"test": queries}) +relevant_docs = DatasetDict({"test": relevant_docs}) + + +repo_name = "isaacchung/flickr30ki2t" +# create_repo(repo_name, repo_type="dataset", token=WRITE_TOK) + +corpus.push_to_hub(repo_name, "corpus", token=WRITE_TOK) +queries.push_to_hub(repo_name, "query", token=WRITE_TOK) +relevant_docs.push_to_hub(repo_name, "qrels", token=WRITE_TOK) diff --git a/scripts/data/flickr30k/build_flickr_30k_t2i.py b/scripts/data/flickr30k/build_flickr_30k_t2i.py new file mode 100644 index 0000000000..b000de6e4d --- /dev/null +++ b/scripts/data/flickr30k/build_flickr_30k_t2i.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import os + +from datasets import Dataset, DatasetDict, load_dataset +from huggingface_hub import create_repo +from tqdm import tqdm + +WRITE_TOK = os.environ["HF_TOKEN"] + +eval_split = "test" +data_raw = load_dataset("clip-benchmark/wds_flickr30k")[eval_split] + + +## t2i +queries_ = {"id": [], "modality": [], "text": []} +corpus_ = {"id": [], "modality": [], "image": []} +relevant_docs_ = {"query-id": [], "corpus-id": [], "score": []} + +for row in tqdm(data_raw, total=len(data_raw)): + image = row["jpg"] + texts = row["txt"].split("\n") + key = row["__key__"] + + doc_id = f"d_{key}" + corpus_["id"].append(doc_id) + corpus_["image"].append(image) + corpus_["modality"].append("image") + + for i, text in enumerate(texts): + query_id = f"q_{key}_{i}" + queries_["id"].append(query_id) + queries_["text"].append(text) + queries_["modality"].append("text") + + relevant_docs_["query-id"].append(query_id) + relevant_docs_["corpus-id"].append(doc_id) + relevant_docs_["score"].append(1) + +corpus = Dataset.from_dict(corpus_) +queries = Dataset.from_dict(queries_) +relevant_docs = Dataset.from_dict(relevant_docs_) + +corpus = DatasetDict({"corpus": corpus}) +queries = DatasetDict({"test": queries}) +relevant_docs = DatasetDict({"test": relevant_docs}) + + +repo_name = "isaacchung/flickr30kt2i" +create_repo(repo_name, repo_type="dataset", token=WRITE_TOK) + +corpus.push_to_hub(repo_name, "corpus", token=WRITE_TOK) +queries.push_to_hub(repo_name, "query", token=WRITE_TOK) +relevant_docs.push_to_hub(repo_name, "qrels", token=WRITE_TOK)