Skip to content

Commit

Permalink
Add Gemma-Embeddings-v0.8 Retrieval Results (#59)
Browse files Browse the repository at this point in the history 
* Add Gofer-Embeddings-v0.8 Retrieval Results

* Set revision to be external

* Fix revision ID

* Fix formatting

* Fix formatting

* Fix formatting

* model naming

* Update to improved model and results using mteb version 1.21.7

* Update results/google__Gemma-Embeddings-v0.8/d6813d20532a97ea8e30fc285397d5105316511f/model_meta.json

Co-authored-by: Kenneth Enevoldsen <[email protected]>

* Fix -1 evaluation time

* Fix revision hash / model meta data

---------

Co-authored-by: Kenneth Enevoldsen <[email protected]>
  • Loading branch information
@nicholasmonath @KennethEnevoldsen
nicholasmonath and KennethEnevoldsen authored Dec 11, 2024
1 parent 5f6f731 commit 2a8b9de
Show file tree
Hide file tree
Showing 27 changed files with 4,131 additions and 0 deletions.
158 changes: 158 additions & 0 deletions results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/ArguAna.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"task_name": "ArguAna",
"mteb_version": "1.21.7",
"scores": {
"test": [
{
"ndcg_at_1": 0.6835,
"ndcg_at_3": 0.80894,
"ndcg_at_5": 0.82664,
"ndcg_at_10": 0.83828,
"ndcg_at_20": 0.84084,
"ndcg_at_100": 0.8428,
"ndcg_at_1000": 0.8428,
"map_at_1": 0.6835,
"map_at_3": 0.77786,
"map_at_5": 0.78774,
"map_at_10": 0.79276,
"map_at_20": 0.79349,
"map_at_100": 0.7938,
"map_at_1000": 0.7938,
"recall_at_1": 0.6835,
"recall_at_3": 0.899,
"recall_at_5": 0.94168,
"recall_at_10": 0.97653,
"recall_at_20": 0.98649,
"recall_at_100": 0.99644,
"recall_at_1000": 0.99644,
"precision_at_1": 0.6835,
"precision_at_3": 0.29967,
"precision_at_5": 0.18834,
"precision_at_10": 0.09765,
"precision_at_20": 0.04932,
"precision_at_100": 0.00996,
"precision_at_1000": 0.001,
"mrr_at_1": 0.684922,
"mrr_at_3": 0.777975,
"mrr_at_5": 0.788039,
"mrr_at_10": 0.79313,
"mrr_at_20": 0.793787,
"mrr_at_100": 0.794095,
"mrr_at_1000": 0.794095,
"nauc_ndcg_at_1_max": -0.139649,
"nauc_ndcg_at_1_std": -0.375886,
"nauc_ndcg_at_1_diff1": 0.433638,
"nauc_ndcg_at_3_max": -0.086534,
"nauc_ndcg_at_3_std": -0.403432,
"nauc_ndcg_at_3_diff1": 0.405864,
"nauc_ndcg_at_5_max": -0.073493,
"nauc_ndcg_at_5_std": -0.389494,
"nauc_ndcg_at_5_diff1": 0.409875,
"nauc_ndcg_at_10_max": -0.085517,
"nauc_ndcg_at_10_std": -0.396341,
"nauc_ndcg_at_10_diff1": 0.420024,
"nauc_ndcg_at_20_max": -0.092515,
"nauc_ndcg_at_20_std": -0.393067,
"nauc_ndcg_at_20_diff1": 0.417239,
"nauc_ndcg_at_100_max": -0.100057,
"nauc_ndcg_at_100_std": -0.387815,
"nauc_ndcg_at_100_diff1": 0.41327,
"nauc_ndcg_at_1000_max": -0.100057,
"nauc_ndcg_at_1000_std": -0.387815,
"nauc_ndcg_at_1000_diff1": 0.41327,
"nauc_map_at_1_max": -0.139649,
"nauc_map_at_1_std": -0.375886,
"nauc_map_at_1_diff1": 0.433638,
"nauc_map_at_3_max": -0.107184,
"nauc_map_at_3_std": -0.398843,
"nauc_map_at_3_diff1": 0.409684,
"nauc_map_at_5_max": -0.10149,
"nauc_map_at_5_std": -0.39196,
"nauc_map_at_5_diff1": 0.41196,
"nauc_map_at_10_max": -0.106406,
"nauc_map_at_10_std": -0.394026,
"nauc_map_at_10_diff1": 0.415305,
"nauc_map_at_20_max": -0.107914,
"nauc_map_at_20_std": -0.393155,
"nauc_map_at_20_diff1": 0.414699,
"nauc_map_at_100_max": -0.108878,
"nauc_map_at_100_std": -0.392627,
"nauc_map_at_100_diff1": 0.414206,
"nauc_map_at_1000_max": -0.108878,
"nauc_map_at_1000_std": -0.392627,
"nauc_map_at_1000_diff1": 0.414206,
"nauc_recall_at_1_max": -0.139649,
"nauc_recall_at_1_std": -0.375886,
"nauc_recall_at_1_diff1": 0.433638,
"nauc_recall_at_3_max": 0.031895,
"nauc_recall_at_3_std": -0.425581,
"nauc_recall_at_3_diff1": 0.388713,
"nauc_recall_at_5_max": 0.191475,
"nauc_recall_at_5_std": -0.355495,
"nauc_recall_at_5_diff1": 0.39885,
"nauc_recall_at_10_max": 0.367349,
"nauc_recall_at_10_std": -0.443247,
"nauc_recall_at_10_diff1": 0.572029,
"nauc_recall_at_20_max": 0.408104,
"nauc_recall_at_20_std": -0.360184,
"nauc_recall_at_20_diff1": 0.569123,
"nauc_recall_at_100_max": 0.336931,
"nauc_recall_at_100_std": 0.82911,
"nauc_recall_at_100_diff1": 0.218345,
"nauc_recall_at_1000_max": 0.336931,
"nauc_recall_at_1000_std": 0.82911,
"nauc_recall_at_1000_diff1": 0.218345,
"nauc_precision_at_1_max": -0.139649,
"nauc_precision_at_1_std": -0.375886,
"nauc_precision_at_1_diff1": 0.433638,
"nauc_precision_at_3_max": 0.031895,
"nauc_precision_at_3_std": -0.425581,
"nauc_precision_at_3_diff1": 0.388713,
"nauc_precision_at_5_max": 0.191475,
"nauc_precision_at_5_std": -0.355495,
"nauc_precision_at_5_diff1": 0.39885,
"nauc_precision_at_10_max": 0.367349,
"nauc_precision_at_10_std": -0.443247,
"nauc_precision_at_10_diff1": 0.572029,
"nauc_precision_at_20_max": 0.408104,
"nauc_precision_at_20_std": -0.360184,
"nauc_precision_at_20_diff1": 0.569123,
"nauc_precision_at_100_max": 0.336931,
"nauc_precision_at_100_std": 0.82911,
"nauc_precision_at_100_diff1": 0.218345,
"nauc_precision_at_1000_max": 0.336931,
"nauc_precision_at_1000_std": 0.82911,
"nauc_precision_at_1000_diff1": 0.218345,
"nauc_mrr_at_1_max": -0.133793,
"nauc_mrr_at_1_std": -0.362766,
"nauc_mrr_at_1_diff1": 0.429598,
"nauc_mrr_at_3_max": -0.106856,
"nauc_mrr_at_3_std": -0.392887,
"nauc_mrr_at_3_diff1": 0.407134,
"nauc_mrr_at_5_max": -0.099574,
"nauc_mrr_at_5_std": -0.384355,
"nauc_mrr_at_5_diff1": 0.409777,
"nauc_mrr_at_10_max": -0.104527,
"nauc_mrr_at_10_std": -0.385681,
"nauc_mrr_at_10_diff1": 0.412832,
"nauc_mrr_at_20_max": -0.105995,
"nauc_mrr_at_20_std": -0.385301,
"nauc_mrr_at_20_diff1": 0.412397,
"nauc_mrr_at_100_max": -0.106958,
"nauc_mrr_at_100_std": -0.384759,
"nauc_mrr_at_100_diff1": 0.411898,
"nauc_mrr_at_1000_max": -0.106958,
"nauc_mrr_at_1000_std": -0.384759,
"nauc_mrr_at_1000_diff1": 0.411898,
"main_score": 0.83828,
"hf_subset": "default",
"languages": [
"eng-Latn"
]
}
]
},
"evaluation_time": null,
"kg_co2_emissions": null
}
158 changes: 158 additions & 0 deletions ...Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackAndroidRetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"task_name": "CQADupstackAndroidRetrieval",
"mteb_version": "1.21.7",
"scores": {
"test": [
{
"ndcg_at_1": 0.48069,
"ndcg_at_3": 0.55233,
"ndcg_at_5": 0.5781,
"ndcg_at_10": 0.60813,
"ndcg_at_20": 0.63176,
"ndcg_at_100": 0.65472,
"ndcg_at_1000": 0.66453,
"map_at_1": 0.39131,
"map_at_3": 0.49395,
"map_at_5": 0.51762,
"map_at_10": 0.5379,
"map_at_20": 0.54828,
"map_at_100": 0.55428,
"map_at_1000": 0.55517,
"recall_at_1": 0.39131,
"recall_at_3": 0.5836,
"recall_at_5": 0.65664,
"recall_at_10": 0.74367,
"recall_at_20": 0.82633,
"recall_at_100": 0.92675,
"recall_at_1000": 0.98313,
"precision_at_1": 0.48069,
"precision_at_3": 0.26943,
"precision_at_5": 0.19371,
"precision_at_10": 0.11888,
"precision_at_20": 0.0706,
"precision_at_100": 0.01775,
"precision_at_1000": 0.00213,
"mrr_at_1": 0.480687,
"mrr_at_3": 0.567,
"mrr_at_5": 0.583453,
"mrr_at_10": 0.59313,
"mrr_at_20": 0.597255,
"mrr_at_100": 0.599273,
"mrr_at_1000": 0.599466,
"nauc_ndcg_at_1_max": 0.279776,
"nauc_ndcg_at_1_std": -0.076593,
"nauc_ndcg_at_1_diff1": 0.558078,
"nauc_ndcg_at_3_max": 0.261882,
"nauc_ndcg_at_3_std": -0.077324,
"nauc_ndcg_at_3_diff1": 0.545451,
"nauc_ndcg_at_5_max": 0.26582,
"nauc_ndcg_at_5_std": -0.05957,
"nauc_ndcg_at_5_diff1": 0.542039,
"nauc_ndcg_at_10_max": 0.264581,
"nauc_ndcg_at_10_std": -0.068243,
"nauc_ndcg_at_10_diff1": 0.53496,
"nauc_ndcg_at_20_max": 0.270382,
"nauc_ndcg_at_20_std": -0.057978,
"nauc_ndcg_at_20_diff1": 0.533699,
"nauc_ndcg_at_100_max": 0.277093,
"nauc_ndcg_at_100_std": -0.046942,
"nauc_ndcg_at_100_diff1": 0.534789,
"nauc_ndcg_at_1000_max": 0.274909,
"nauc_ndcg_at_1000_std": -0.056378,
"nauc_ndcg_at_1000_diff1": 0.536635,
"nauc_map_at_1_max": 0.239159,
"nauc_map_at_1_std": -0.095362,
"nauc_map_at_1_diff1": 0.601201,
"nauc_map_at_3_max": 0.255082,
"nauc_map_at_3_std": -0.093217,
"nauc_map_at_3_diff1": 0.569299,
"nauc_map_at_5_max": 0.263045,
"nauc_map_at_5_std": -0.082091,
"nauc_map_at_5_diff1": 0.567506,
"nauc_map_at_10_max": 0.265626,
"nauc_map_at_10_std": -0.083517,
"nauc_map_at_10_diff1": 0.561056,
"nauc_map_at_20_max": 0.269138,
"nauc_map_at_20_std": -0.077477,
"nauc_map_at_20_diff1": 0.559743,
"nauc_map_at_100_max": 0.270629,
"nauc_map_at_100_std": -0.075127,
"nauc_map_at_100_diff1": 0.557582,
"nauc_map_at_1000_max": 0.270353,
"nauc_map_at_1000_std": -0.075523,
"nauc_map_at_1000_diff1": 0.55744,
"nauc_recall_at_1_max": 0.239159,
"nauc_recall_at_1_std": -0.095362,
"nauc_recall_at_1_diff1": 0.601201,
"nauc_recall_at_3_max": 0.22928,
"nauc_recall_at_3_std": -0.082319,
"nauc_recall_at_3_diff1": 0.516506,
"nauc_recall_at_5_max": 0.227638,
"nauc_recall_at_5_std": -0.039444,
"nauc_recall_at_5_diff1": 0.486232,
"nauc_recall_at_10_max": 0.212423,
"nauc_recall_at_10_std": -0.058951,
"nauc_recall_at_10_diff1": 0.449737,
"nauc_recall_at_20_max": 0.237039,
"nauc_recall_at_20_std": 0.015395,
"nauc_recall_at_20_diff1": 0.424174,
"nauc_recall_at_100_max": 0.34741,
"nauc_recall_at_100_std": 0.285448,
"nauc_recall_at_100_diff1": 0.420057,
"nauc_recall_at_1000_max": 0.506705,
"nauc_recall_at_1000_std": 0.545243,
"nauc_recall_at_1000_diff1": 0.475028,
"nauc_precision_at_1_max": 0.279776,
"nauc_precision_at_1_std": -0.076593,
"nauc_precision_at_1_diff1": 0.558078,
"nauc_precision_at_3_max": 0.19415,
"nauc_precision_at_3_std": -0.036236,
"nauc_precision_at_3_diff1": 0.234531,
"nauc_precision_at_5_max": 0.168166,
"nauc_precision_at_5_std": 0.028196,
"nauc_precision_at_5_diff1": 0.11287,
"nauc_precision_at_10_max": 0.096798,
"nauc_precision_at_10_std": 0.04027,
"nauc_precision_at_10_diff1": -0.038503,
"nauc_precision_at_20_max": 0.047349,
"nauc_precision_at_20_std": 0.067286,
"nauc_precision_at_20_diff1": -0.135178,
"nauc_precision_at_100_max": -0.045056,
"nauc_precision_at_100_std": 0.050698,
"nauc_precision_at_100_diff1": -0.244088,
"nauc_precision_at_1000_max": -0.121574,
"nauc_precision_at_1000_std": -0.015487,
"nauc_precision_at_1000_diff1": -0.292365,
"nauc_mrr_at_1_max": 0.279776,
"nauc_mrr_at_1_std": -0.076593,
"nauc_mrr_at_1_diff1": 0.558078,
"nauc_mrr_at_3_max": 0.273289,
"nauc_mrr_at_3_std": -0.062736,
"nauc_mrr_at_3_diff1": 0.537039,
"nauc_mrr_at_5_max": 0.275071,
"nauc_mrr_at_5_std": -0.052781,
"nauc_mrr_at_5_diff1": 0.531544,
"nauc_mrr_at_10_max": 0.271179,
"nauc_mrr_at_10_std": -0.057327,
"nauc_mrr_at_10_diff1": 0.530266,
"nauc_mrr_at_20_max": 0.272699,
"nauc_mrr_at_20_std": -0.055493,
"nauc_mrr_at_20_diff1": 0.530654,
"nauc_mrr_at_100_max": 0.273777,
"nauc_mrr_at_100_std": -0.054868,
"nauc_mrr_at_100_diff1": 0.530861,
"nauc_mrr_at_1000_max": 0.27372,
"nauc_mrr_at_1000_std": -0.055075,
"nauc_mrr_at_1000_diff1": 0.53095,
"main_score": 0.60813,
"hf_subset": "default",
"languages": [
"eng-Latn"
]
}
]
},
"evaluation_time": null,
"kg_co2_emissions": null
}
Loading

0 comments on commit 2a8b9de

Please sign in to comment.