Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add combined file for CQADupstackRetrieval for Gemma-Embeddings-v0.8 #70

Merged
merged 23 commits into from
Dec 12, 2024
Merged

Add combined file for CQADupstackRetrieval for Gemma-Embeddings-v0.8 #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f2fe004
Add Gofer-Embeddings-v0.8 Retrieval Results
nicholasmonath Dec 2, 2024
f4e4fea
Set revision to be external
nicholasmonath Dec 2, 2024
2577635
Fix revision ID
nicholasmonath Dec 2, 2024
3df2119
Fix formatting
nicholasmonath Dec 2, 2024
cecfff9
Fix formatting
nicholasmonath Dec 2, 2024
eff7c50
Fix formatting
nicholasmonath Dec 3, 2024
b5a7196
Merge branch 'embeddings-benchmark:main' into main
nicholasmonath Dec 3, 2024
28c5baf
model naming
nicholasmonath Dec 9, 2024
1be252c
Update to improved model and results using mteb version 1.21.7
nicholasmonath Dec 9, 2024
0b5e079
Merge branch 'main' of github.com:nicholasmonath/results
nicholasmonath Dec 9, 2024
c4366a5
Merge branch 'embeddings-benchmark:main' into main
nicholasmonath Dec 9, 2024
48e738f
Merge branch 'main' of github.com:nicholasmonath/results
nicholasmonath Dec 9, 2024
618b029
Update results/google__Gemma-Embeddings-v0.8/d6813d20532a97ea8e30fc28…
nicholasmonath Dec 10, 2024
aca915a
Fix -1 evaluation time
nicholasmonath Dec 10, 2024
1ac505f
Merge branch 'main' of github.com:nicholasmonath/results
nicholasmonath Dec 10, 2024
be81705
Fix revision hash / model meta data
nicholasmonath Dec 10, 2024
8944407
Merge branch 'embeddings-benchmark:main' into main
nicholasmonath Dec 10, 2024
f1f47ad
Merge branch 'embeddings-benchmark:main' into main
nicholasmonath Dec 12, 2024
f14ed8e
Update paths to include new Gemma Embedding results. Fix malformed js…
nicholasmonath Dec 12, 2024
06c0120
Merge branch 'main' of github.com:nicholasmonath/results
nicholasmonath Dec 12, 2024
d2d6e9c
Add CQADupstackRetrieval combined results file.
nicholasmonath Dec 12, 2024
ba8969d
Merge branch 'embeddings-benchmark:main' into main
nicholasmonath Dec 12, 2024
f7581bb
Merge branch 'main' of github.com:nicholasmonath/results
nicholasmonath Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion paths.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15118,6 +15118,7 @@
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackGamingRetrieval.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/HotpotQA.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackProgrammersRetrieval.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackRetrieval.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackPhysicsRetrieval.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackWordpressRetrieval.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/ClimateFEVER.json",
Expand All @@ -15134,4 +15135,4 @@
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/MSMARCO.json",
"results/google__Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackMathematicaRetrieval.json"
]
}
}
158 changes: 158 additions & 0 deletions ..._Gemma-Embeddings-v0.8/c9c76df8fa92c9f561bee6c61b4f9c13750a661c/CQADupstackRetrieval.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "CQADupstackRetrieval_is_a_combined_dataset",
"task_name": "CQADupstackRetrieval",
"mteb_version": "1.21.7",
"scores": {
"test": [
{
"ndcg_at_1": 0.4020541666666666,
"ndcg_at_3": 0.4666908333333333,
"ndcg_at_5": 0.4936708333333333,
"ndcg_at_10": 0.5218766666666667,
"ndcg_at_20": 0.5432208333333334,
"ndcg_at_100": 0.5703583333333334,
"ndcg_at_1000": 0.5849916666666666,
"map_at_1": 0.33820999999999996,
"map_at_3": 0.42414833333333335,
"map_at_5": 0.44435916666666664,
"map_at_10": 0.45972166666666675,
"map_at_20": 0.4674225,
"map_at_100": 0.4728383333333333,
"map_at_1000": 0.47376666666666667,
"recall_at_1": 0.33820999999999996,
"recall_at_3": 0.5055849999999998,
"recall_at_5": 0.5759116666666667,
"recall_at_10": 0.6590283333333333,
"recall_at_20": 0.7360616666666667,
"recall_at_100": 0.8637716666666666,
"recall_at_1000": 0.9607316666666664,
"precision_at_1": 0.4020541666666666,
"precision_at_3": 0.2170416666666667,
"precision_at_5": 0.153365,
"precision_at_10": 0.09181833333333334,
"precision_at_20": 0.053378333333333326,
"precision_at_100": 0.013653333333333332,
"precision_at_1000": 0.001665,
"mrr_at_1": 0.40205325,
"mrr_at_3": 0.4771851666666666,
"mrr_at_5": 0.4928709166666667,
"mrr_at_10": 0.50312025,
"mrr_at_20": 0.5079848333333333,
"mrr_at_100": 0.5107778333333334,
"mrr_at_1000": 0.5111320833333334,
"nauc_ndcg_at_1_max": 0.27959433333333333,
"nauc_ndcg_at_1_std": -0.051863,
"nauc_ndcg_at_1_diff1": 0.512957,
"nauc_ndcg_at_3_max": 0.26203025,
"nauc_ndcg_at_3_std": -0.06255891666666667,
"nauc_ndcg_at_3_diff1": 0.4646831666666667,
"nauc_ndcg_at_5_max": 0.2749124166666667,
"nauc_ndcg_at_5_std": -0.05559333333333335,
"nauc_ndcg_at_5_diff1": 0.4599999166666666,
"nauc_ndcg_at_10_max": 0.28059758333333334,
"nauc_ndcg_at_10_std": -0.04480933333333333,
"nauc_ndcg_at_10_diff1": 0.4566293333333335,
"nauc_ndcg_at_20_max": 0.2844583333333333,
"nauc_ndcg_at_20_std": -0.03484508333333334,
"nauc_ndcg_at_20_diff1": 0.45688449999999997,
"nauc_ndcg_at_100_max": 0.28945174999999995,
"nauc_ndcg_at_100_std": -0.02521816666666667,
"nauc_ndcg_at_100_diff1": 0.4582004166666667,
"nauc_ndcg_at_1000_max": 0.28690625,
"nauc_ndcg_at_1000_std": -0.02926841666666666,
"nauc_ndcg_at_1000_diff1": 0.46183749999999996,
"nauc_map_at_1_max": 0.22903666666666667,
"nauc_map_at_1_std": -0.08212708333333334,
"nauc_map_at_1_diff1": 0.5246739166666666,
"nauc_map_at_3_max": 0.24647175000000007,
"nauc_map_at_3_std": -0.07754766666666667,
"nauc_map_at_3_diff1": 0.4807849166666667,
"nauc_map_at_5_max": 0.25908183333333334,
"nauc_map_at_5_std": -0.070416,
"nauc_map_at_5_diff1": 0.47696650000000007,
"nauc_map_at_10_max": 0.2648871666666666,
"nauc_map_at_10_std": -0.06285816666666666,
"nauc_map_at_10_diff1": 0.47451991666666665,
"nauc_map_at_20_max": 0.26760866666666666,
"nauc_map_at_20_std": -0.05832466666666664,
"nauc_map_at_20_diff1": 0.4743503333333334,
"nauc_map_at_100_max": 0.2692709166666667,
"nauc_map_at_100_std": -0.055308583333333335,
"nauc_map_at_100_diff1": 0.474343,
"nauc_map_at_1000_max": 0.26909025,
"nauc_map_at_1000_std": -0.05502908333333333,
"nauc_map_at_1000_diff1": 0.4744370833333333,
"nauc_recall_at_1_max": 0.22903666666666667,
"nauc_recall_at_1_std": -0.08212708333333334,
"nauc_recall_at_1_diff1": 0.5246739166666666,
"nauc_recall_at_3_max": 0.2338484166666667,
"nauc_recall_at_3_std": -0.07751666666666666,
"nauc_recall_at_3_diff1": 0.4226811666666666,
"nauc_recall_at_5_max": 0.26351625,
"nauc_recall_at_5_std": -0.056870583333333335,
"nauc_recall_at_5_diff1": 0.39880075,
"nauc_recall_at_10_max": 0.28123716666666665,
"nauc_recall_at_10_std": -0.019335916666666664,
"nauc_recall_at_10_diff1": 0.3767715833333334,
"nauc_recall_at_20_max": 0.30026233333333335,
"nauc_recall_at_20_std": 0.03786475,
"nauc_recall_at_20_diff1": 0.3637902499999999,
"nauc_recall_at_100_max": 0.38340541666666655,
"nauc_recall_at_100_std": 0.19742633333333334,
"nauc_recall_at_100_diff1": 0.33567858333333334,
"nauc_recall_at_1000_max": 0.4925595,
"nauc_recall_at_1000_std": 0.4351984166666667,
"nauc_recall_at_1000_diff1": 0.3328621666666666,
"nauc_precision_at_1_max": 0.27959433333333333,
"nauc_precision_at_1_std": -0.051863,
"nauc_precision_at_1_diff1": 0.512957,
"nauc_precision_at_3_max": 0.2537675833333333,
"nauc_precision_at_3_std": -0.0009907499999999981,
"nauc_precision_at_3_diff1": 0.27616416666666666,
"nauc_precision_at_5_max": 0.2626798333333334,
"nauc_precision_at_5_std": 0.03978466666666666,
"nauc_precision_at_5_diff1": 0.19944983333333333,
"nauc_precision_at_10_max": 0.23977883333333336,
"nauc_precision_at_10_std": 0.09176050000000001,
"nauc_precision_at_10_diff1": 0.11407491666666665,
"nauc_precision_at_20_max": 0.21376024999999998,
"nauc_precision_at_20_std": 0.13474358333333333,
"nauc_precision_at_20_diff1": 0.048363750000000004,
"nauc_precision_at_100_max": 0.14341025000000002,
"nauc_precision_at_100_std": 0.17450875,
"nauc_precision_at_100_diff1": -0.056095833333333345,
"nauc_precision_at_1000_max": 0.04704733333333333,
"nauc_precision_at_1000_std": 0.1305885,
"nauc_precision_at_1000_diff1": -0.12969583333333332,
"nauc_mrr_at_1_max": 0.27959433333333333,
"nauc_mrr_at_1_std": -0.051863,
"nauc_mrr_at_1_diff1": 0.512957,
"nauc_mrr_at_3_max": 0.28132708333333334,
"nauc_mrr_at_3_std": -0.048320916666666665,
"nauc_mrr_at_3_diff1": 0.47595641666666677,
"nauc_mrr_at_5_max": 0.28779724999999995,
"nauc_mrr_at_5_std": -0.04348183333333334,
"nauc_mrr_at_5_diff1": 0.47218791666666676,
"nauc_mrr_at_10_max": 0.2882275,
"nauc_mrr_at_10_std": -0.04052833333333333,
"nauc_mrr_at_10_diff1": 0.4715955833333334,
"nauc_mrr_at_20_max": 0.2885099166666667,
"nauc_mrr_at_20_std": -0.038913500000000004,
"nauc_mrr_at_20_diff1": 0.47183975,
"nauc_mrr_at_100_max": 0.28868125,
"nauc_mrr_at_100_std": -0.03853541666666666,
"nauc_mrr_at_100_diff1": 0.47237300000000015,
"nauc_mrr_at_1000_max": 0.28862125,
"nauc_mrr_at_1000_std": -0.03864533333333334,
"nauc_mrr_at_1000_diff1": 0.4724678333333334,
"main_score": 0.5218766666666667,
"hf_subset": "default",
"languages": [
"eng-Latn"
]
}
]
},
"evaluation_time": null,
"kg_co2_emissions": null
}
Loading