From 7f8cec47d196fc7707e8b934bd7bfec6a63362d2 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Tue, 10 Dec 2024 16:15:44 +0000 Subject: [PATCH 1/2] mieb final runs --- scripts/run_mieb_missed_results.py | 107 ++++++++--------------------- 1 file changed, 29 insertions(+), 78 deletions(-) diff --git a/scripts/run_mieb_missed_results.py b/scripts/run_mieb_missed_results.py index f3b523178..ec28671e4 100644 --- a/scripts/run_mieb_missed_results.py +++ b/scripts/run_mieb_missed_results.py @@ -2,75 +2,6 @@ import mteb -# missing -model = mteb.get_model("royokong/e5-v") -tasks = mteb.get_tasks(tasks=["EDIST2ITRetrieval", "Winoground"]) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("nyu-visionx/moco-v3-vit-b") -tasks = mteb.get_tasks( - tasks=[ - "CIFAR10Clustering", - "ImageNetDog15Clustering", - "TinyImageNetClustering", - "UCF101", - "VOC2007", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("nyu-visionx/moco-v3-vit-l") -tasks = mteb.get_tasks( - tasks=[ - "SketchyI2IRetrieval", - "CIFAR10Clustering", - "ImageNetDog15Clustering", - "TinyImageNetClustering", - "UCF101", - "VOC2007", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("BAAI/bge-visualized-base") -tasks = mteb.get_tasks( - tasks=[ - "SciMMIRI2TRetrieval", - "SciMMIRT2IRetrieval", - "VisualNewsI2TRetrieval", - "WITT2IRetrieval", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("TIGER-Lab/VLM2Vec-Full") -tasks = mteb.get_tasks( - tasks=["CVBenchCount", "CVBenchDepth", "CVBenchDistance", "CVBenchRelation"] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - -model = mteb.get_model("TIGER-Lab/VLM2Vec-LoRA") -tasks = mteb.get_tasks( - task_types=[ - "Any2AnyRetrieval", - "Any2AnyMultiChoice", - "Any2TextMutipleChoice", - "ImageClustering", - "ImageClassification", - "ImageMultilabelClassification", - "ImageTextPairClassification", - "VisualSTS", - "ZeroShotClassification", - ] -) -evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-final") - # rerun for model_name in [ "openai/clip-vit-base-patch32", @@ -82,8 +13,8 @@ "kakaobrain/align-base", "jinaai/jina-clip-v1", "nomic-ai/nomic-embed-vision-v1.5", - "Salesforce/blip-image-captioning-large", - "Salesforce/blip-image-captioning-base", + # "Salesforce/blip-image-captioning-large", + # "Salesforce/blip-image-captioning-base", "Salesforce/blip2-opt-2.7b", "Salesforce/blip2-opt-6.7b-coco", "facebook/dinov2-small", @@ -125,12 +56,18 @@ model = mteb.get_model(model_name) tasks = mteb.get_tasks( tasks=[ - "ROxfordEasyI2IRetrieval", - "ROxfordHardI2IRetrieval", - "ROxfordMediumI2IRetrieval", - "RParisEasyI2IRetrieval", - "RParisHardI2IRetrieval", - "RParisMediumI2IRetrieval", + "ROxfordEasyI2IMultiChoice", + "ROxfordHardI2IMultiChoice", + "ROxfordMediumI2IMultiChoice", + "RParisEasyI2IMultiChoice", + "RParisHardI2IMultiChoice", + "RParisMediumI2IMultiChoice", + "BLINKIT2IRetrieval", + "BLINKIT2TRetrieval", + "BLINKIT2IMultiChoice", + "BLINKIT2TMultiChoice", + "Flickr30kI2TRetrieval", + "Flickr30kT2IRetrieval" ] ) # get i-only tasks for i-only models. @@ -138,4 +75,18 @@ tasks = [task for task in tasks if "t" not in task.metadata.category] evaluation = mteb.MTEB(tasks=tasks) - results = evaluation.run(model, output_folder="results-mieb-rerun") + results = evaluation.run(model, output_folder="results-mieb-rerun2") + +# # missing task +model_name = "TIGER-Lab/VLM2Vec-Full" +model = mteb.get_model(model_name) +tasks = mteb.get_tasks( + tasks=[ + "CVBenchCount", + "CVBenchDepth", + "CVBenchDistance", + "CVBenchRelation", + ] +) +evaluation = mteb.MTEB(tasks=tasks) +results = evaluation.run(model, output_folder="results-mieb-rerun2") \ No newline at end of file From 57977cdca12d04d4b263017ff7fa0d537d13448d Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Tue, 10 Dec 2024 16:16:23 +0000 Subject: [PATCH 2/2] lint --- scripts/run_mieb_missed_results.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/run_mieb_missed_results.py b/scripts/run_mieb_missed_results.py index ec28671e4..5c92289f6 100644 --- a/scripts/run_mieb_missed_results.py +++ b/scripts/run_mieb_missed_results.py @@ -67,7 +67,7 @@ "BLINKIT2IMultiChoice", "BLINKIT2TMultiChoice", "Flickr30kI2TRetrieval", - "Flickr30kT2IRetrieval" + "Flickr30kT2IRetrieval", ] ) # get i-only tasks for i-only models. @@ -76,7 +76,7 @@ evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-rerun2") - + # # missing task model_name = "TIGER-Lab/VLM2Vec-Full" model = mteb.get_model(model_name) @@ -89,4 +89,4 @@ ] ) evaluation = mteb.MTEB(tasks=tasks) -results = evaluation.run(model, output_folder="results-mieb-rerun2") \ No newline at end of file +results = evaluation.run(model, output_folder="results-mieb-rerun2")