Skip to content

Commit

Permalink
[mieb] Fix Flickr30k I2T and T2I (#1505)
Browse files Browse the repository at this point in the history 
* remake flickr30k it2 and t2i

* add openai clip vit-b32 b16 and jina-clip results

* make lint
  • Loading branch information
@isaac-chung
isaac-chung authored Nov 27, 2024
1 parent fab0b82 commit 67a035d
Show file tree
Hide file tree
Showing 10 changed files with 1,226 additions and 18 deletions.
4 changes: 2 additions & 2 deletions mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kI2TRetrieval.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ class Flickr30kI2TRetrieval(AbsTaskAny2AnyRetrieval):
description="Retrieve captions based on images.",
reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31",
dataset={
"path": "JamieSJS/flickr30k",
"revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629",
"path": "isaacchung/flickr30ki2t",
"revision": "6984df6bd4380034e7766d9a992d8907df363efb",
},
type="Any2AnyRetrieval",
category="i2t",
Expand Down
18 changes: 2 additions & 16 deletions mteb/tasks/Image/Any2AnyRetrieval/eng/Flickr30kT2IRetrieval.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval):
description="Retrieve images based on captions.",
reference="https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31",
dataset={
"path": "JamieSJS/flickr30k",
"revision": "24acb2d0b72e18b03388eb20a6225983c0e3f629",
"path": "isaacchung/flickr30kt2i",
"revision": "e819702b287bfbe084e129a61f308a802b7c108e",
},
type="Any2AnyRetrieval",
category="t2i",
Expand Down Expand Up @@ -39,17 +39,3 @@ class Flickr30kT2IRetrieval(AbsTaskAny2AnyRetrieval):
"n_samples": {"default": 31014}, # qrels
},
)

def load_data(self, **kwargs):
super().load_data(**kwargs)
# swap corpus and query
for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]):
self.queries[split], self.corpus[split] = (
self.corpus[split],
self.queries[split],
)
self.relevant_docs[split] = {
cid: {qid: score}
for qid, cid_score in self.relevant_docs[split].items()
for cid, score in cid_score.items()
}
186 changes: 186 additions & 0 deletions .../jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Flickr30kI2TRetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
{
"dataset_revision": "6984df6bd4380034e7766d9a992d8907df363efb",
"evaluation_time": 12.655046463012695,
"kg_co2_emissions": null,
"mteb_version": "1.16.5",
"scores": {
"test": [
{
"cv_recall_at_1": 0.806,
"cv_recall_at_10": 0.981,
"cv_recall_at_100": 0.999,
"cv_recall_at_1000": 1.0,
"cv_recall_at_20": 0.994,
"cv_recall_at_3": 0.922,
"cv_recall_at_5": 0.961,
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.75656,
"map_at_1": 0.1612,
"map_at_10": 0.65737,
"map_at_100": 0.6941,
"map_at_1000": 0.69494,
"map_at_20": 0.68213,
"map_at_3": 0.42077,
"map_at_5": 0.5824,
"mrr_at_1": 0.806,
"mrr_at_10": 0.8702051587301588,
"mrr_at_100": 0.8713558453277496,
"mrr_at_1000": 0.8713621346359256,
"mrr_at_20": 0.8711786018505061,
"mrr_at_3": 0.8581666666666669,
"mrr_at_5": 0.8672666666666671,
"nauc_cv_recall_at_1000_diff1": NaN,
"nauc_cv_recall_at_1000_max": NaN,
"nauc_cv_recall_at_1000_std": NaN,
"nauc_cv_recall_at_100_diff1": 0.3580765639589976,
"nauc_cv_recall_at_100_max": 1.0,
"nauc_cv_recall_at_100_std": 1.0,
"nauc_cv_recall_at_10_diff1": 0.4775664651825631,
"nauc_cv_recall_at_10_max": 0.7300113027667245,
"nauc_cv_recall_at_10_std": -0.05830753353973558,
"nauc_cv_recall_at_1_diff1": 0.3443688532171371,
"nauc_cv_recall_at_1_max": 0.4866993124852272,
"nauc_cv_recall_at_1_std": -0.1314705005806373,
"nauc_cv_recall_at_20_diff1": 0.5864456893868585,
"nauc_cv_recall_at_20_max": 0.9782135076252602,
"nauc_cv_recall_at_20_std": 0.5981948334889493,
"nauc_cv_recall_at_3_diff1": 0.3899961694079347,
"nauc_cv_recall_at_3_max": 0.5140295434413065,
"nauc_cv_recall_at_3_std": -0.12323732911968292,
"nauc_cv_recall_at_5_diff1": 0.48242715889774873,
"nauc_cv_recall_at_5_max": 0.5554957025545274,
"nauc_cv_recall_at_5_std": -0.16390145801910372,
"nauc_map_at_1000_diff1": 0.07860412999292432,
"nauc_map_at_1000_max": 0.5193407172877343,
"nauc_map_at_1000_std": -0.03554548015806451,
"nauc_map_at_100_diff1": 0.07860332936377284,
"nauc_map_at_100_max": 0.5202209025325473,
"nauc_map_at_100_std": -0.03402804669053566,
"nauc_map_at_10_diff1": 0.06738608544555293,
"nauc_map_at_10_max": 0.5046712341845588,
"nauc_map_at_10_std": -0.052089661267638164,
"nauc_map_at_1_diff1": 0.344368853217137,
"nauc_map_at_1_max": 0.4866993124852267,
"nauc_map_at_1_std": -0.1314705005806365,
"nauc_map_at_20_diff1": 0.07540132773467392,
"nauc_map_at_20_max": 0.5194163232471378,
"nauc_map_at_20_std": -0.03561110594882118,
"nauc_map_at_3_diff1": 0.13827655728488158,
"nauc_map_at_3_max": 0.48609865047620054,
"nauc_map_at_3_std": -0.09536745328619929,
"nauc_map_at_5_diff1": 0.07059479448464502,
"nauc_map_at_5_max": 0.48086096767511555,
"nauc_map_at_5_std": -0.08160089788651494,
"nauc_mrr_at_1000_diff1": 0.36023929557475076,
"nauc_mrr_at_1000_max": 0.5035860304880058,
"nauc_mrr_at_1000_std": -0.1249299862258316,
"nauc_mrr_at_100_diff1": 0.3602391842803307,
"nauc_mrr_at_100_max": 0.5036115760084787,
"nauc_mrr_at_100_std": -0.12487209719821057,
"nauc_mrr_at_10_diff1": 0.3611514720783506,
"nauc_mrr_at_10_max": 0.5051263876997938,
"nauc_mrr_at_10_std": -0.1257752596646823,
"nauc_mrr_at_1_diff1": 0.3443688532171371,
"nauc_mrr_at_1_max": 0.4866993124852272,
"nauc_mrr_at_1_std": -0.1314705005806373,
"nauc_mrr_at_20_diff1": 0.3606869616253337,
"nauc_mrr_at_20_max": 0.5042795551778488,
"nauc_mrr_at_20_std": -0.1239507002161601,
"nauc_mrr_at_3_diff1": 0.35911327275172067,
"nauc_mrr_at_3_max": 0.49885073455301077,
"nauc_mrr_at_3_std": -0.12597845586993137,
"nauc_mrr_at_5_diff1": 0.36427948002844535,
"nauc_mrr_at_5_max": 0.5025432319868158,
"nauc_mrr_at_5_std": -0.12876002101412964,
"nauc_ndcg_at_1000_diff1": 0.1404445479237733,
"nauc_ndcg_at_1000_max": 0.5347818240932689,
"nauc_ndcg_at_1000_std": -0.03316587882098286,
"nauc_ndcg_at_100_diff1": 0.1358934054564792,
"nauc_ndcg_at_100_max": 0.5460446827666793,
"nauc_ndcg_at_100_std": -0.007831409654410723,
"nauc_ndcg_at_10_diff1": 0.10422124872393643,
"nauc_ndcg_at_10_max": 0.522718634663143,
"nauc_ndcg_at_10_std": -0.042784056691292496,
"nauc_ndcg_at_1_diff1": 0.3443688532171371,
"nauc_ndcg_at_1_max": 0.4866993124852272,
"nauc_ndcg_at_1_std": -0.1314705005806373,
"nauc_ndcg_at_20_diff1": 0.12477162836638274,
"nauc_ndcg_at_20_max": 0.5506038459831472,
"nauc_ndcg_at_20_std": -0.0058328462109728755,
"nauc_ndcg_at_3_diff1": 0.1727965348989873,
"nauc_ndcg_at_3_max": 0.49295039267335533,
"nauc_ndcg_at_3_std": -0.09392106684131309,
"nauc_ndcg_at_5_diff1": 0.1101745223344572,
"nauc_ndcg_at_5_max": 0.4913548608912362,
"nauc_ndcg_at_5_std": -0.08489198959710985,
"nauc_precision_at_1000_diff1": -0.08286647992530825,
"nauc_precision_at_1000_max": 0.854341736694646,
"nauc_precision_at_1000_std": 0.8109243697478463,
"nauc_precision_at_100_diff1": 0.02976321846328062,
"nauc_precision_at_100_max": 0.6817348214942566,
"nauc_precision_at_100_std": 0.4525517335603402,
"nauc_precision_at_10_diff1": 0.03152919182802702,
"nauc_precision_at_10_max": 0.5209015914034817,
"nauc_precision_at_10_std": 0.012785208286159547,
"nauc_precision_at_1_diff1": 0.3443688532171371,
"nauc_precision_at_1_max": 0.4866993124852272,
"nauc_precision_at_1_std": -0.1314705005806373,
"nauc_precision_at_20_diff1": 0.058403294374015056,
"nauc_precision_at_20_max": 0.5976698949057889,
"nauc_precision_at_20_std": 0.15342898310071018,
"nauc_precision_at_3_diff1": 0.13112541027685642,
"nauc_precision_at_3_max": 0.4910052478350366,
"nauc_precision_at_3_std": -0.0846106773972222,
"nauc_precision_at_5_diff1": 0.057600655958168046,
"nauc_precision_at_5_max": 0.4823384872903635,
"nauc_precision_at_5_std": -0.07416100661323119,
"nauc_recall_at_1000_diff1": -0.08286647992533601,
"nauc_recall_at_1000_max": 0.8543417366946594,
"nauc_recall_at_1000_std": 0.8109243697478747,
"nauc_recall_at_100_diff1": 0.029763218463283347,
"nauc_recall_at_100_max": 0.6817348214942569,
"nauc_recall_at_100_std": 0.4525517335603416,
"nauc_recall_at_10_diff1": 0.03152919182802702,
"nauc_recall_at_10_max": 0.5209015914034817,
"nauc_recall_at_10_std": 0.012785208286159547,
"nauc_recall_at_1_diff1": 0.344368853217137,
"nauc_recall_at_1_max": 0.4866993124852267,
"nauc_recall_at_1_std": -0.1314705005806365,
"nauc_recall_at_20_diff1": 0.058403294374015056,
"nauc_recall_at_20_max": 0.5976698949057889,
"nauc_recall_at_20_std": 0.15342898310071018,
"nauc_recall_at_3_diff1": 0.1311254102768568,
"nauc_recall_at_3_max": 0.49100524783503674,
"nauc_recall_at_3_std": -0.0846106773972213,
"nauc_recall_at_5_diff1": 0.057600655958168046,
"nauc_recall_at_5_max": 0.4823384872903635,
"nauc_recall_at_5_std": -0.07416100661323119,
"ndcg_at_1": 0.806,
"ndcg_at_10": 0.75656,
"ndcg_at_100": 0.82771,
"ndcg_at_1000": 0.83522,
"ndcg_at_20": 0.79537,
"ndcg_at_3": 0.7524,
"ndcg_at_5": 0.67901,
"precision_at_1": 0.806,
"precision_at_10": 0.3869,
"precision_at_100": 0.04818,
"precision_at_1000": 0.00499,
"precision_at_20": 0.2159,
"precision_at_3": 0.735,
"precision_at_5": 0.6344,
"recall_at_1": 0.1612,
"recall_at_10": 0.7738,
"recall_at_100": 0.9636,
"recall_at_1000": 0.9976,
"recall_at_20": 0.8636,
"recall_at_3": 0.441,
"recall_at_5": 0.6344
}
]
},
"task_name": "Flickr30kI2TRetrieval"
}
Loading

0 comments on commit 67a035d

Please sign in to comment.