Skip to content

Commit

Permalink
chore: add jina-v3 evaluation result on mmteb multilingual (#41)
Browse files Browse the repository at this point in the history 
* chore: add jina-v3 evaluation result on mmteb

* chore: apply make pre push

* chore: patch miracl retrieval results

* trigger CI

---------

Co-authored-by: Kenneth Enevoldsen <[email protected]>
Co-authored-by: Isaac Chung <[email protected]>
  • Loading branch information
@bwanglzu @KennethEnevoldsen @isaac-chung
3 people authored Nov 9, 2024
1 parent 7695129 commit 1616d1b
Show file tree
Hide file tree
Showing 135 changed files with 161,363 additions and 1 deletion.
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest] #, macos-latest, windows-latest]
python-version: ["3.8"] # , "3.9", "3.10"]
python-version: ["3.9"]

steps:
- uses: actions/checkout@v3
Expand Down
158 changes: 158 additions & 0 deletions ...lts/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/AILAStatutes.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e",
"evaluation_time": 1.649610996246338,
"kg_co2_emissions": null,
"mteb_version": "1.18.2",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.32773,
"map_at_1": 0.071,
"map_at_10": 0.19666,
"map_at_100": 0.26018,
"map_at_1000": 0.26018,
"map_at_20": 0.21765,
"map_at_3": 0.1275,
"map_at_5": 0.16017,
"mrr_at_1": 0.32,
"mrr_at_10": 0.483,
"mrr_at_100": 0.4904463118580766,
"mrr_at_1000": 0.4904463118580766,
"mrr_at_20": 0.48949393090569565,
"mrr_at_3": 0.43,
"mrr_at_5": 0.461,
"nauc_map_at_1000_diff1": -0.12797552251112276,
"nauc_map_at_1000_max": 0.163195009479752,
"nauc_map_at_1000_std": 0.1784295270856589,
"nauc_map_at_100_diff1": -0.12797552251112276,
"nauc_map_at_100_max": 0.163195009479752,
"nauc_map_at_100_std": 0.1784295270856589,
"nauc_map_at_10_diff1": -0.08995364022904571,
"nauc_map_at_10_max": 0.1925229843113862,
"nauc_map_at_10_std": 0.18380298751451712,
"nauc_map_at_1_diff1": -0.21742547253217148,
"nauc_map_at_1_max": 0.05927526768636921,
"nauc_map_at_1_std": 0.05854718387583288,
"nauc_map_at_20_diff1": -0.11929591170076337,
"nauc_map_at_20_max": 0.1779790778162725,
"nauc_map_at_20_std": 0.17363915696250012,
"nauc_map_at_3_diff1": -0.1922330072240532,
"nauc_map_at_3_max": 0.19438147683933096,
"nauc_map_at_3_std": 0.14954385008570212,
"nauc_map_at_5_diff1": -0.08263647472418753,
"nauc_map_at_5_max": 0.2497528708805151,
"nauc_map_at_5_std": 0.14502538628236758,
"nauc_mrr_at_1000_diff1": -0.12336330643042633,
"nauc_mrr_at_1000_max": 0.13854040920452687,
"nauc_mrr_at_1000_std": 0.07329470588671386,
"nauc_mrr_at_100_diff1": -0.12336330643042633,
"nauc_mrr_at_100_max": 0.13854040920452687,
"nauc_mrr_at_100_std": 0.07329470588671386,
"nauc_mrr_at_10_diff1": -0.11631992795422062,
"nauc_mrr_at_10_max": 0.1449668733600396,
"nauc_mrr_at_10_std": 0.07455379940623108,
"nauc_mrr_at_1_diff1": -0.22367012736808312,
"nauc_mrr_at_1_max": 0.04209033501016787,
"nauc_mrr_at_1_std": 0.052204859252916465,
"nauc_mrr_at_20_diff1": -0.12071283554889783,
"nauc_mrr_at_20_max": 0.13849101618586232,
"nauc_mrr_at_20_std": 0.0725279183503699,
"nauc_mrr_at_3_diff1": -0.105666344069899,
"nauc_mrr_at_3_max": 0.1439281381622888,
"nauc_mrr_at_3_std": 0.07165737497103443,
"nauc_mrr_at_5_diff1": -0.10458708791169632,
"nauc_mrr_at_5_max": 0.1561852295113067,
"nauc_mrr_at_5_std": 0.09093723120024294,
"nauc_ndcg_at_1000_diff1": -0.13788256526312664,
"nauc_ndcg_at_1000_max": 0.12199528260885305,
"nauc_ndcg_at_1000_std": 0.15799629538429735,
"nauc_ndcg_at_100_diff1": -0.13788256526312664,
"nauc_ndcg_at_100_max": 0.12199528260885305,
"nauc_ndcg_at_100_std": 0.15799629538429735,
"nauc_ndcg_at_10_diff1": -0.054268964094045816,
"nauc_ndcg_at_10_max": 0.20300723974175514,
"nauc_ndcg_at_10_std": 0.18179546754141582,
"nauc_ndcg_at_1_diff1": -0.22367012736808312,
"nauc_ndcg_at_1_max": 0.04209033501016787,
"nauc_ndcg_at_1_std": 0.052204859252916465,
"nauc_ndcg_at_20_diff1": -0.09238136761445143,
"nauc_ndcg_at_20_max": 0.17200635359871944,
"nauc_ndcg_at_20_std": 0.16809293840346623,
"nauc_ndcg_at_3_diff1": -0.14214311345152836,
"nauc_ndcg_at_3_max": 0.16960155702863947,
"nauc_ndcg_at_3_std": 0.1410356537415608,
"nauc_ndcg_at_5_diff1": -0.036383910483647294,
"nauc_ndcg_at_5_max": 0.27156565266137944,
"nauc_ndcg_at_5_std": 0.14355032468779033,
"nauc_precision_at_1000_diff1": -0.40377966008671456,
"nauc_precision_at_1000_max": -0.7248312325058065,
"nauc_precision_at_1000_std": 0.09538793655440012,
"nauc_precision_at_100_diff1": -0.40377966008671107,
"nauc_precision_at_100_max": -0.7248312325058035,
"nauc_precision_at_100_std": 0.09538793655440192,
"nauc_precision_at_10_diff1": -0.03272268254790227,
"nauc_precision_at_10_max": 0.054343604350077776,
"nauc_precision_at_10_std": 0.2512407215605042,
"nauc_precision_at_1_diff1": -0.22367012736808312,
"nauc_precision_at_1_max": 0.04209033501016787,
"nauc_precision_at_1_std": 0.052204859252916465,
"nauc_precision_at_20_diff1": -0.12200642439899881,
"nauc_precision_at_20_max": -0.08255695652670927,
"nauc_precision_at_20_std": 0.1903085311903426,
"nauc_precision_at_3_diff1": -0.10267114523352527,
"nauc_precision_at_3_max": 0.20153550863723615,
"nauc_precision_at_3_std": 0.19846449136276398,
"nauc_precision_at_5_diff1": 0.046751707190575705,
"nauc_precision_at_5_max": 0.27963349427643525,
"nauc_precision_at_5_std": 0.21412430076190722,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": NaN,
"nauc_recall_at_100_max": NaN,
"nauc_recall_at_100_std": NaN,
"nauc_recall_at_10_diff1": 0.07485826287523366,
"nauc_recall_at_10_max": 0.2455644517516293,
"nauc_recall_at_10_std": 0.2159406283650886,
"nauc_recall_at_1_diff1": -0.21742547253217148,
"nauc_recall_at_1_max": 0.05927526768636921,
"nauc_recall_at_1_std": 0.05854718387583288,
"nauc_recall_at_20_diff1": 0.023968970111766418,
"nauc_recall_at_20_max": 0.18284685351268357,
"nauc_recall_at_20_std": 0.17225210435399174,
"nauc_recall_at_3_diff1": -0.10092032231612168,
"nauc_recall_at_3_max": 0.29201596671116925,
"nauc_recall_at_3_std": 0.17071030178168448,
"nauc_recall_at_5_diff1": 0.09179148478207606,
"nauc_recall_at_5_max": 0.3822915149728462,
"nauc_recall_at_5_std": 0.1506803578849412,
"ndcg_at_1": 0.32,
"ndcg_at_10": 0.32773,
"ndcg_at_100": 0.52388,
"ndcg_at_1000": 0.52388,
"ndcg_at_20": 0.38143,
"ndcg_at_3": 0.26059,
"ndcg_at_5": 0.25606,
"precision_at_1": 0.32,
"precision_at_10": 0.168,
"precision_at_100": 0.0434,
"precision_at_1000": 0.00434,
"precision_at_20": 0.114,
"precision_at_3": 0.24,
"precision_at_5": 0.212,
"recall_at_1": 0.071,
"recall_at_10": 0.38867,
"recall_at_100": 1.0,
"recall_at_1000": 1.0,
"recall_at_20": 0.521,
"recall_at_3": 0.166,
"recall_at_5": 0.24867
}
]
},
"task_name": "AILAStatutes"
}
Loading

0 comments on commit 1616d1b

Please sign in to comment.