Skip to content

Commit

Permalink
Addresults (#53)
Browse files Browse the repository at this point in the history 
* Add

* Add res

* fix: reduce sizes of files and added makefile command

* Add

* Add res

* formatted flores files

* Add

* Add res

* make preprush

* Add OAI T3Small MMTEB

* update ci dependencies to >3.9

* tmp

* reduce large file sizes

* Add missing TE3L results

* Make prepush

* Add res

* reduce size

* AddRes

* Add

* merge

* rmvdup

* addres

---------

Co-authored-by: Kenneth Enevoldsen <[email protected]>
  • Loading branch information
@Muennighoff @KennethEnevoldsen
Muennighoff and KennethEnevoldsen authored Dec 12, 2024
1 parent edb303f commit 1e4015b
Show file tree
Hide file tree
Showing 16 changed files with 7,432 additions and 0 deletions.
2,657 changes: 2,657 additions & 0 deletions ...n2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLRetrievalHardNegatives.json
View file

Large diffs are not rendered by default.

158 changes: 158 additions & 0 deletions ...gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/TopiOCQAHardNegatives.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "b4cc09fb8bb3a9e0ce0f94dc69c96397a2a47c18",
"evaluation_time": 1372.583566904068,
"kg_co2_emissions": 0.8386569017336079,
"mteb_version": "1.18.0",
"scores": {
"validation": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.21379,
"map_at_1": 0.101,
"map_at_10": 0.16999,
"map_at_100": 0.18447,
"map_at_1000": 0.1857,
"map_at_20": 0.17815,
"map_at_3": 0.1445,
"map_at_5": 0.15685,
"mrr_at_1": 0.101,
"mrr_at_10": 0.16999285714285714,
"mrr_at_100": 0.1844746515014581,
"mrr_at_1000": 0.18570487360712473,
"mrr_at_20": 0.17815270115161763,
"mrr_at_3": 0.1444999999999999,
"mrr_at_5": 0.15684999999999996,
"nauc_map_at_1000_diff1": 0.13945748323445184,
"nauc_map_at_1000_max": 0.180274903790705,
"nauc_map_at_1000_std": 0.06709372815594669,
"nauc_map_at_100_diff1": 0.13919785917177155,
"nauc_map_at_100_max": 0.18042107824349934,
"nauc_map_at_100_std": 0.06717059926787362,
"nauc_map_at_10_diff1": 0.14261175175203336,
"nauc_map_at_10_max": 0.17927760124979583,
"nauc_map_at_10_std": 0.06629638707545342,
"nauc_map_at_1_diff1": 0.15905716654183333,
"nauc_map_at_1_max": 0.2242468020908022,
"nauc_map_at_1_std": 0.09479778338713164,
"nauc_map_at_20_diff1": 0.139220636105314,
"nauc_map_at_20_max": 0.17848699783493913,
"nauc_map_at_20_std": 0.06518309842275453,
"nauc_map_at_3_diff1": 0.13727914503491365,
"nauc_map_at_3_max": 0.18229014733987836,
"nauc_map_at_3_std": 0.07237713804423689,
"nauc_map_at_5_diff1": 0.13722921086137477,
"nauc_map_at_5_max": 0.17343924186979326,
"nauc_map_at_5_std": 0.06662998675824422,
"nauc_mrr_at_1000_diff1": 0.13945748323445184,
"nauc_mrr_at_1000_max": 0.180274903790705,
"nauc_mrr_at_1000_std": 0.06709372815594669,
"nauc_mrr_at_100_diff1": 0.13919785917177155,
"nauc_mrr_at_100_max": 0.18042107824349934,
"nauc_mrr_at_100_std": 0.06717059926787362,
"nauc_mrr_at_10_diff1": 0.14261175175203336,
"nauc_mrr_at_10_max": 0.17927760124979583,
"nauc_mrr_at_10_std": 0.06629638707545342,
"nauc_mrr_at_1_diff1": 0.15905716654183333,
"nauc_mrr_at_1_max": 0.2242468020908022,
"nauc_mrr_at_1_std": 0.09479778338713164,
"nauc_mrr_at_20_diff1": 0.139220636105314,
"nauc_mrr_at_20_max": 0.17848699783493913,
"nauc_mrr_at_20_std": 0.06518309842275453,
"nauc_mrr_at_3_diff1": 0.13727914503491365,
"nauc_mrr_at_3_max": 0.18229014733987836,
"nauc_mrr_at_3_std": 0.07237713804423689,
"nauc_mrr_at_5_diff1": 0.13722921086137477,
"nauc_mrr_at_5_max": 0.17343924186979326,
"nauc_mrr_at_5_std": 0.06662998675824422,
"nauc_ndcg_at_1000_diff1": 0.13462695630731858,
"nauc_ndcg_at_1000_max": 0.17755443691247924,
"nauc_ndcg_at_1000_std": 0.06519254409553903,
"nauc_ndcg_at_100_diff1": 0.12945694978383163,
"nauc_ndcg_at_100_max": 0.18146541998404866,
"nauc_ndcg_at_100_std": 0.06437218294937233,
"nauc_ndcg_at_10_diff1": 0.14492910077258592,
"nauc_ndcg_at_10_max": 0.17155360648651682,
"nauc_ndcg_at_10_std": 0.056013456686342916,
"nauc_ndcg_at_1_diff1": 0.15905716654183333,
"nauc_ndcg_at_1_max": 0.2242468020908022,
"nauc_ndcg_at_1_std": 0.09479778338713164,
"nauc_ndcg_at_20_diff1": 0.13448532388570655,
"nauc_ndcg_at_20_max": 0.1687675913840615,
"nauc_ndcg_at_20_std": 0.05200988337607004,
"nauc_ndcg_at_3_diff1": 0.13268386072320917,
"nauc_ndcg_at_3_max": 0.17132134501845353,
"nauc_ndcg_at_3_std": 0.06657030830595347,
"nauc_ndcg_at_5_diff1": 0.13229564249040088,
"nauc_ndcg_at_5_max": 0.15705178443583584,
"nauc_ndcg_at_5_std": 0.0576524931823522,
"nauc_precision_at_1000_diff1": -0.07584994782226347,
"nauc_precision_at_1000_max": 0.23235568737298928,
"nauc_precision_at_1000_std": 0.34585598945459417,
"nauc_precision_at_100_diff1": 0.0789506575749579,
"nauc_precision_at_100_max": 0.21343758245726152,
"nauc_precision_at_100_std": 0.07469908679513332,
"nauc_precision_at_10_diff1": 0.1533995866069045,
"nauc_precision_at_10_max": 0.15985540439857587,
"nauc_precision_at_10_std": 0.03396892850244202,
"nauc_precision_at_1_diff1": 0.15905716654183333,
"nauc_precision_at_1_max": 0.2242468020908022,
"nauc_precision_at_1_std": 0.09479778338713164,
"nauc_precision_at_20_diff1": 0.12327729240061686,
"nauc_precision_at_20_max": 0.15111780475474534,
"nauc_precision_at_20_std": 0.02148096425427092,
"nauc_precision_at_3_diff1": 0.12252699360258856,
"nauc_precision_at_3_max": 0.14647466927409164,
"nauc_precision_at_3_std": 0.05342916048850624,
"nauc_precision_at_5_diff1": 0.12177288818501525,
"nauc_precision_at_5_max": 0.12089951556932252,
"nauc_precision_at_5_std": 0.037959685120059615,
"nauc_recall_at_1000_diff1": -0.07584994782226935,
"nauc_recall_at_1000_max": 0.23235568737298334,
"nauc_recall_at_1000_std": 0.34585598945460877,
"nauc_recall_at_100_diff1": 0.07895065757495787,
"nauc_recall_at_100_max": 0.21343758245726166,
"nauc_recall_at_100_std": 0.07469908679513311,
"nauc_recall_at_10_diff1": 0.15339958660690484,
"nauc_recall_at_10_max": 0.15985540439857632,
"nauc_recall_at_10_std": 0.033968928502442236,
"nauc_recall_at_1_diff1": 0.15905716654183333,
"nauc_recall_at_1_max": 0.2242468020908022,
"nauc_recall_at_1_std": 0.09479778338713164,
"nauc_recall_at_20_diff1": 0.12327729240061752,
"nauc_recall_at_20_max": 0.15111780475474582,
"nauc_recall_at_20_std": 0.02148096425427147,
"nauc_recall_at_3_diff1": 0.12252699360258854,
"nauc_recall_at_3_max": 0.1464746692740916,
"nauc_recall_at_3_std": 0.05342916048850615,
"nauc_recall_at_5_diff1": 0.12177288818501512,
"nauc_recall_at_5_max": 0.1208995155693223,
"nauc_recall_at_5_std": 0.037959685120059594,
"ndcg_at_1": 0.101,
"ndcg_at_10": 0.21379,
"ndcg_at_100": 0.29074,
"ndcg_at_1000": 0.32281,
"ndcg_at_20": 0.24325,
"ndcg_at_3": 0.15922,
"ndcg_at_5": 0.18147,
"precision_at_1": 0.101,
"precision_at_10": 0.0357,
"precision_at_100": 0.00733,
"precision_at_1000": 0.00098,
"precision_at_20": 0.02365,
"precision_at_3": 0.06733,
"precision_at_5": 0.0512,
"recall_at_1": 0.101,
"recall_at_10": 0.357,
"recall_at_100": 0.733,
"recall_at_1000": 0.983,
"recall_at_20": 0.473,
"recall_at_3": 0.202,
"recall_at_5": 0.256
}
]
},
"task_name": "TopiOCQAHardNegatives"
}
111 changes: 111 additions & 0 deletions ...wen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/WebLINXCandidatesReranking.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
{
"dataset_revision": "ed1c933c2b3617e5700d8a7ebe07f5975969a453",
"evaluation_time": 123372.13516354561,
"kg_co2_emissions": 11.952502877439517,
"mteb_version": "1.18.0",
"scores": {
"test_cat": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.0976013242375602,
"map": 0.11237258520677565,
"mrr": 0.0976013242375602,
"nAUC_map_diff1": 0.07114335558335581,
"nAUC_map_max": 0.18797165030793325,
"nAUC_map_std": 0.23018588741459572,
"nAUC_mrr_diff1": 0.07350550589163288,
"nAUC_mrr_max": 0.17927106844326243,
"nAUC_mrr_std": 0.21017440831643888
}
],
"test_geo": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.11024537628992469,
"map": 0.1283388969478226,
"mrr": 0.11024537628992469,
"nAUC_map_diff1": 0.17659125476015666,
"nAUC_map_max": 0.22654802479325403,
"nAUC_map_std": 0.14965408976996084,
"nAUC_mrr_diff1": 0.1833294351348487,
"nAUC_mrr_max": 0.2317049762005828,
"nAUC_mrr_std": 0.142763903607862
}
],
"test_iid": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.14476400203103956,
"map": 0.16087191768124767,
"mrr": 0.14476400203103956,
"nAUC_map_diff1": 0.09435070066744501,
"nAUC_map_max": 0.051616909122828326,
"nAUC_map_std": 0.03766806115164064,
"nAUC_mrr_diff1": 0.09795722670339345,
"nAUC_mrr_max": 0.05111781870317291,
"nAUC_mrr_std": 0.02954647008403948
}
],
"test_vis": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.13416690634980558,
"map": 0.1506176662817587,
"mrr": 0.13416690634980558,
"nAUC_map_diff1": 0.14001473527710837,
"nAUC_map_max": 0.16101203617005808,
"nAUC_map_std": 0.1636794967931306,
"nAUC_mrr_diff1": 0.14382866878229997,
"nAUC_mrr_max": 0.15824416820535323,
"nAUC_mrr_std": 0.14705510573289177
}
],
"test_web": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.10753450765378245,
"map": 0.12185726408995741,
"mrr": 0.10753450765378245,
"nAUC_map_diff1": 0.1827597942661111,
"nAUC_map_max": 0.1319497088529989,
"nAUC_map_std": 0.1263163552354218,
"nAUC_mrr_diff1": 0.17856124712920643,
"nAUC_mrr_max": 0.12874239350387714,
"nAUC_mrr_std": 0.11293242307448244
}
],
"validation": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.15576845649866403,
"map": 0.17124390969492517,
"mrr": 0.15576845649866403,
"nAUC_map_diff1": 0.2281755967894526,
"nAUC_map_max": 0.15065079958122887,
"nAUC_map_std": 0.007141893298152778,
"nAUC_mrr_diff1": 0.2328616941037803,
"nAUC_mrr_max": 0.15260088516639617,
"nAUC_mrr_std": 0.007862536306838423
}
]
},
"task_name": "WebLINXCandidatesReranking"
}
Loading

0 comments on commit 1e4015b

Please sign in to comment.