diff --git a/paths.json b/paths.json index e0ae355ac..639135d96 100644 --- a/paths.json +++ b/paths.json @@ -14696,6 +14696,18 @@ "results/voyageai__voyage-3/no_revision_available/LegalQuAD.json", "results/voyageai__voyage-3/no_revision_available/XPQARetrieval.json" ], + "voyageai__voyage-code-3": [ + "results/voyageai__voyage-code-3/no_revision_available/SyntheticText2SQL.json", + "results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackST.json", + "results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetCCRetrieval.json", + "results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanContest.json", + "results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackMT.json", + "results/voyageai__voyage-code-3/no_revision_available/AppsRetrieval.json", + "results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetRetrieval.json", + "results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanDL.json", + "results/voyageai__voyage-code-3/no_revision_available/StackOverflowQA.json", + "results/voyageai__voyage-code-3/no_revision_available/CosQA.json" + ], "voyageai__voyage-3-lite": [ "results/voyageai__voyage-3-lite/no_revision_available/LEMBWikimQARetrieval.json", "results/voyageai__voyage-3-lite/no_revision_available/LegalSummarization.json", diff --git a/results/voyageai__voyage-code-3/no_revision_available/AppsRetrieval.json b/results/voyageai__voyage-code-3/no_revision_available/AppsRetrieval.json new file mode 100644 index 000000000..33f1934a3 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "evaluation_time": 207.9835319519043, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ], + "main_score": 0.93621, + "map_at_1": 0.87968, + "map_at_10": 0.92047, + "map_at_100": 0.92115, + "map_at_1000": 0.92115, + "map_at_20": 0.92091, + "map_at_3": 0.91532, + "map_at_5": 0.91874, + "mrr_at_1": 0.8799468791500664, + "mrr_at_10": 0.9206234321971379, + "mrr_at_100": 0.9213012638701028, + "mrr_at_1000": 0.921308801595124, + "mrr_at_20": 0.9210639248723755, + "mrr_at_3": 0.9154050464807443, + "mrr_at_5": 0.9188977423638782, + "nauc_map_at_1000_diff1": 0.9015490174664648, + "nauc_map_at_1000_max": 0.6702866848649027, + "nauc_map_at_1000_std": -0.4343778333375324, + "nauc_map_at_100_diff1": 0.9015545196165746, + "nauc_map_at_100_max": 0.6703097567361977, + "nauc_map_at_100_std": -0.43437587975498626, + "nauc_map_at_10_diff1": 0.9014099743587611, + "nauc_map_at_10_max": 0.6716096459752251, + "nauc_map_at_10_std": -0.43443704106955056, + "nauc_map_at_1_diff1": 0.9099585708349436, + "nauc_map_at_1_max": 0.6357782533912141, + "nauc_map_at_1_std": -0.4236134139461213, + "nauc_map_at_20_diff1": 0.9015514084107631, + "nauc_map_at_20_max": 0.6707765073421194, + "nauc_map_at_20_std": -0.4341448959858248, + "nauc_map_at_3_diff1": 0.898995753062064, + "nauc_map_at_3_max": 0.6761366242819364, + "nauc_map_at_3_std": -0.4422691404903539, + "nauc_map_at_5_diff1": 0.9009003396740809, + "nauc_map_at_5_max": 0.674059817584288, + "nauc_map_at_5_std": -0.4362598115209535, + "nauc_mrr_at_1000_diff1": 0.901224401379764, + "nauc_mrr_at_1000_max": 0.670844465768374, + "nauc_mrr_at_1000_std": -0.43634995805553367, + "nauc_mrr_at_100_diff1": 0.9012299449341405, + "nauc_mrr_at_100_max": 0.6708675243260439, + "nauc_mrr_at_100_std": -0.43634780889033514, + "nauc_mrr_at_10_diff1": 0.901087963438495, + "nauc_mrr_at_10_max": 0.6721648611418671, + "nauc_mrr_at_10_std": -0.43639195425027527, + "nauc_mrr_at_1_diff1": 0.9094858533614968, + "nauc_mrr_at_1_max": 0.636569816124724, + "nauc_mrr_at_1_std": -0.4261438951155136, + "nauc_mrr_at_20_diff1": 0.901227820706247, + "nauc_mrr_at_20_max": 0.671333396707349, + "nauc_mrr_at_20_std": -0.4361103858801164, + "nauc_mrr_at_3_diff1": 0.8987704691693779, + "nauc_mrr_at_3_max": 0.6769227631885193, + "nauc_mrr_at_3_std": -0.44331998862855104, + "nauc_mrr_at_5_diff1": 0.9005844149558918, + "nauc_mrr_at_5_max": 0.6746072486731265, + "nauc_mrr_at_5_std": -0.4381755835911823, + "nauc_ndcg_at_1000_diff1": 0.9004496527123014, + "nauc_ndcg_at_1000_max": 0.6760247091688248, + "nauc_ndcg_at_1000_std": -0.4303623963675031, + "nauc_ndcg_at_100_diff1": 0.9007047193164399, + "nauc_ndcg_at_100_max": 0.6768039037445461, + "nauc_ndcg_at_100_std": -0.43036110157737034, + "nauc_ndcg_at_10_diff1": 0.900028263117178, + "nauc_ndcg_at_10_max": 0.6838084649569117, + "nauc_ndcg_at_10_std": -0.4296245497355038, + "nauc_ndcg_at_1_diff1": 0.9099585708349436, + "nauc_ndcg_at_1_max": 0.6357782533912141, + "nauc_ndcg_at_1_std": -0.4236134139461213, + "nauc_ndcg_at_20_diff1": 0.9006473833364463, + "nauc_ndcg_at_20_max": 0.6804393781250493, + "nauc_ndcg_at_20_std": -0.4275464319022175, + "nauc_ndcg_at_3_diff1": 0.8947150366489366, + "nauc_ndcg_at_3_max": 0.6928073683657389, + "nauc_ndcg_at_3_std": -0.4480650989406889, + "nauc_ndcg_at_5_diff1": 0.8985493663258894, + "nauc_ndcg_at_5_max": 0.6897238224506672, + "nauc_ndcg_at_5_std": -0.4359588098814929, + "nauc_precision_at_1000_diff1": 0.7152280764342951, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.9564334438407598, + "nauc_precision_at_100_diff1": 0.8905538400916919, + "nauc_precision_at_100_max": 0.9359380801456066, + "nauc_precision_at_100_std": 0.034218682638907, + "nauc_precision_at_10_diff1": 0.8882693413470759, + "nauc_precision_at_10_max": 0.8371281148941699, + "nauc_precision_at_10_std": -0.3365732721164693, + "nauc_precision_at_1_diff1": 0.9099585708349436, + "nauc_precision_at_1_max": 0.6357782533912141, + "nauc_precision_at_1_std": -0.4236134139461213, + "nauc_precision_at_20_diff1": 0.8960083706628117, + "nauc_precision_at_20_max": 0.8508806968533703, + "nauc_precision_at_20_std": -0.22269540063924853, + "nauc_precision_at_3_diff1": 0.8716551109941072, + "nauc_precision_at_3_max": 0.7832435110823177, + "nauc_precision_at_3_std": -0.47825568017450937, + "nauc_precision_at_5_diff1": 0.8821645334780426, + "nauc_precision_at_5_max": 0.8121614910200275, + "nauc_precision_at_5_std": -0.426577409583649, + "nauc_recall_at_1000_diff1": 0.7152280764343947, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.9564334438409235, + "nauc_recall_at_100_diff1": 0.8905538400917727, + "nauc_recall_at_100_max": 0.9359380801455807, + "nauc_recall_at_100_std": 0.03421868263895069, + "nauc_recall_at_10_diff1": 0.8882693413470616, + "nauc_recall_at_10_max": 0.8371281148941585, + "nauc_recall_at_10_std": -0.33657327211644983, + "nauc_recall_at_1_diff1": 0.9099585708349436, + "nauc_recall_at_1_max": 0.6357782533912141, + "nauc_recall_at_1_std": -0.4236134139461213, + "nauc_recall_at_20_diff1": 0.8960083706628053, + "nauc_recall_at_20_max": 0.8508806968533525, + "nauc_recall_at_20_std": -0.2226954006392219, + "nauc_recall_at_3_diff1": 0.8716551109941042, + "nauc_recall_at_3_max": 0.7832435110823132, + "nauc_recall_at_3_std": -0.4782556801745169, + "nauc_recall_at_5_diff1": 0.8821645334780379, + "nauc_recall_at_5_max": 0.8121614910200308, + "nauc_recall_at_5_std": -0.42657740958365065, + "ndcg_at_1": 0.87968, + "ndcg_at_10": 0.93621, + "ndcg_at_100": 0.93929, + "ndcg_at_1000": 0.93949, + "ndcg_at_20": 0.93778, + "ndcg_at_3": 0.92602, + "ndcg_at_5": 0.93217, + "precision_at_1": 0.87968, + "precision_at_10": 0.09838, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "precision_at_20": 0.0495, + "precision_at_3": 0.3189, + "precision_at_5": 0.19432, + "recall_at_1": 0.87968, + "recall_at_10": 0.9838, + "recall_at_100": 0.99761, + "recall_at_1000": 0.9992, + "recall_at_20": 0.98991, + "recall_at_3": 0.95671, + "recall_at_5": 0.97158 + } + ] + }, + "task_name": "AppsRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/COIRCodeSearchNetRetrieval.json b/results/voyageai__voyage-code-3/no_revision_available/COIRCodeSearchNetRetrieval.json new file mode 100644 index 000000000..2632692a4 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/COIRCodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "4adc7bc41202b5c13543c9c886a25f340634dab3", + "evaluation_time": 10422.876004219055, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "python", + "languages": [ + "python-Code" + ], + "main_score": 0.98367, + "map_at_1": 0.96494, + "map_at_10": 0.97881, + "map_at_100": 0.97891, + "map_at_1000": 0.97891, + "map_at_20": 0.97888, + "map_at_3": 0.97761, + "map_at_5": 0.97849, + "mrr_at_1": 0.9649416811905082, + "mrr_at_10": 0.9788111783570305, + "mrr_at_100": 0.978906549372564, + "mrr_at_1000": 0.9789071223051654, + "mrr_at_20": 0.9788807761475905, + "mrr_at_3": 0.9776109398042645, + "mrr_at_5": 0.9784924252580788, + "nauc_map_at_1000_diff1": 0.9815528913593723, + "nauc_map_at_1000_max": 0.8510151231357158, + "nauc_map_at_1000_std": -0.4270593910947264, + "nauc_map_at_100_diff1": 0.9815533924129993, + "nauc_map_at_100_max": 0.8510116247232071, + "nauc_map_at_100_std": -0.4270950485228204, + "nauc_map_at_10_diff1": 0.9815404628849377, + "nauc_map_at_10_max": 0.85129420000526, + "nauc_map_at_10_std": -0.4305569868322259, + "nauc_map_at_1_diff1": 0.9805311494766504, + "nauc_map_at_1_max": 0.8337279874457242, + "nauc_map_at_1_std": -0.3678783832567891, + "nauc_map_at_20_diff1": 0.9815680755256112, + "nauc_map_at_20_max": 0.851144859632837, + "nauc_map_at_20_std": -0.4276588304453336, + "nauc_map_at_3_diff1": 0.9819103616317174, + "nauc_map_at_3_max": 0.8564151924417325, + "nauc_map_at_3_std": -0.4386751765575472, + "nauc_map_at_5_diff1": 0.9816782356611096, + "nauc_map_at_5_max": 0.8528003038549159, + "nauc_map_at_5_std": -0.4374493490831982, + "nauc_mrr_at_1000_diff1": 0.9815528913593723, + "nauc_mrr_at_1000_max": 0.8510151231357158, + "nauc_mrr_at_1000_std": -0.4270593910947264, + "nauc_mrr_at_100_diff1": 0.9815533924129993, + "nauc_mrr_at_100_max": 0.8510116247232071, + "nauc_mrr_at_100_std": -0.4270950485228204, + "nauc_mrr_at_10_diff1": 0.9815404628849377, + "nauc_mrr_at_10_max": 0.85129420000526, + "nauc_mrr_at_10_std": -0.4305569868322259, + "nauc_mrr_at_1_diff1": 0.9805311494766504, + "nauc_mrr_at_1_max": 0.8337279874457242, + "nauc_mrr_at_1_std": -0.3678783832567891, + "nauc_mrr_at_20_diff1": 0.9815680755256112, + "nauc_mrr_at_20_max": 0.851144859632837, + "nauc_mrr_at_20_std": -0.4276588304453336, + "nauc_mrr_at_3_diff1": 0.9819103616317174, + "nauc_mrr_at_3_max": 0.8564151924417325, + "nauc_mrr_at_3_std": -0.4386751765575472, + "nauc_mrr_at_5_diff1": 0.9816782356611096, + "nauc_mrr_at_5_max": 0.8528003038549159, + "nauc_mrr_at_5_std": -0.4374493490831982, + "nauc_ndcg_at_1000_diff1": 0.9816554701617589, + "nauc_ndcg_at_1000_max": 0.852748828477838, + "nauc_ndcg_at_1000_std": -0.4353566701835024, + "nauc_ndcg_at_100_diff1": 0.9816666937775803, + "nauc_ndcg_at_100_max": 0.8526689650159106, + "nauc_ndcg_at_100_std": -0.4361547874575031, + "nauc_ndcg_at_10_diff1": 0.9815895834975364, + "nauc_ndcg_at_10_max": 0.8543419484878363, + "nauc_ndcg_at_10_std": -0.4534965339542094, + "nauc_ndcg_at_1_diff1": 0.9805311494766504, + "nauc_ndcg_at_1_max": 0.8337279874457242, + "nauc_ndcg_at_1_std": -0.3678783832567891, + "nauc_ndcg_at_20_diff1": 0.9817535928747724, + "nauc_ndcg_at_20_max": 0.8537092010493639, + "nauc_ndcg_at_20_std": -0.43988286523034953, + "nauc_ndcg_at_3_diff1": 0.9825821737148956, + "nauc_ndcg_at_3_max": 0.8661782450729814, + "nauc_ndcg_at_3_std": -0.47329156737175687, + "nauc_ndcg_at_5_diff1": 0.982061154768608, + "nauc_ndcg_at_5_max": 0.8587935683953549, + "nauc_ndcg_at_5_std": -0.47302585930633895, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 0.7222153459022752, + "nauc_precision_at_100_std": -1.7398459168247267, + "nauc_precision_at_10_diff1": 0.9794083448220364, + "nauc_precision_at_10_max": 0.9142141149896693, + "nauc_precision_at_10_std": -1.1027705518270434, + "nauc_precision_at_1_diff1": 0.9805311494766504, + "nauc_precision_at_1_max": 0.8337279874457242, + "nauc_precision_at_1_std": -0.3678783832567891, + "nauc_precision_at_20_diff1": 0.9899445460386596, + "nauc_precision_at_20_max": 0.93841013383867, + "nauc_precision_at_20_std": -0.8533131689356599, + "nauc_precision_at_3_diff1": 0.9877736667583922, + "nauc_precision_at_3_max": 0.9404942862307165, + "nauc_precision_at_3_std": -0.7409082528149944, + "nauc_precision_at_5_diff1": 0.9869061226878082, + "nauc_precision_at_5_max": 0.9354052780940045, + "nauc_precision_at_5_std": -0.9744201029419809, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_100_max": 0.7222153459013213, + "nauc_recall_at_100_std": -1.7398459168226446, + "nauc_recall_at_10_diff1": 0.9794083448220702, + "nauc_recall_at_10_max": 0.9142141149896996, + "nauc_recall_at_10_std": -1.1027705518269966, + "nauc_recall_at_1_diff1": 0.9805311494766504, + "nauc_recall_at_1_max": 0.8337279874457242, + "nauc_recall_at_1_std": -0.3678783832567891, + "nauc_recall_at_20_diff1": 0.9899445460384722, + "nauc_recall_at_20_max": 0.9384101338387074, + "nauc_recall_at_20_std": -0.8533131689356599, + "nauc_recall_at_3_diff1": 0.9877736667583843, + "nauc_recall_at_3_max": 0.940494286230709, + "nauc_recall_at_3_std": -0.7409082528149965, + "nauc_recall_at_5_diff1": 0.9869061226878156, + "nauc_recall_at_5_max": 0.9354052780940341, + "nauc_recall_at_5_std": -0.9744201029420337, + "ndcg_at_1": 0.96494, + "ndcg_at_10": 0.98367, + "ndcg_at_100": 0.98408, + "ndcg_at_1000": 0.98409, + "ndcg_at_20": 0.98392, + "ndcg_at_3": 0.9813, + "ndcg_at_5": 0.98289, + "precision_at_1": 0.96494, + "precision_at_10": 0.09981, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04996, + "precision_at_3": 0.33061, + "precision_at_5": 0.19914, + "recall_at_1": 0.96494, + "recall_at_10": 0.99812, + "recall_at_100": 0.99993, + "recall_at_1000": 1.0, + "recall_at_20": 0.99913, + "recall_at_3": 0.99182, + "recall_at_5": 0.99571 + }, + { + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ], + "main_score": 0.83479, + "map_at_1": 0.74354, + "map_at_10": 0.80713, + "map_at_100": 0.80957, + "map_at_1000": 0.80963, + "map_at_20": 0.80871, + "map_at_3": 0.79712, + "map_at_5": 0.8029, + "mrr_at_1": 0.7435429960498329, + "mrr_at_10": 0.8071294728769667, + "mrr_at_100": 0.8095719685924737, + "mrr_at_1000": 0.809626426876391, + "mrr_at_20": 0.8087121221349531, + "mrr_at_3": 0.7971234680441609, + "mrr_at_5": 0.8028967892231339, + "nauc_map_at_1000_diff1": 0.8288577068822386, + "nauc_map_at_1000_max": 0.6767862670522469, + "nauc_map_at_1000_std": -0.03522371246361021, + "nauc_map_at_100_diff1": 0.8288293265014841, + "nauc_map_at_100_max": 0.6768203242273656, + "nauc_map_at_100_std": -0.03522160367469023, + "nauc_map_at_10_diff1": 0.8280831264112191, + "nauc_map_at_10_max": 0.6776494610226808, + "nauc_map_at_10_std": -0.03318867336370514, + "nauc_map_at_1_diff1": 0.8506863588556931, + "nauc_map_at_1_max": 0.645628789135785, + "nauc_map_at_1_std": -0.06191657522827031, + "nauc_map_at_20_diff1": 0.8285819113195612, + "nauc_map_at_20_max": 0.6775920634659378, + "nauc_map_at_20_std": -0.033782092153909746, + "nauc_map_at_3_diff1": 0.8293140971183676, + "nauc_map_at_3_max": 0.6788657739487108, + "nauc_map_at_3_std": -0.03904228069624038, + "nauc_map_at_5_diff1": 0.8278238115943125, + "nauc_map_at_5_max": 0.6778270133394841, + "nauc_map_at_5_std": -0.036293201723127694, + "nauc_mrr_at_1000_diff1": 0.8288577068822386, + "nauc_mrr_at_1000_max": 0.6767862670522469, + "nauc_mrr_at_1000_std": -0.03522371246361021, + "nauc_mrr_at_100_diff1": 0.8288293265014841, + "nauc_mrr_at_100_max": 0.6768203242273656, + "nauc_mrr_at_100_std": -0.03522160367469023, + "nauc_mrr_at_10_diff1": 0.8280831264112191, + "nauc_mrr_at_10_max": 0.6776494610226808, + "nauc_mrr_at_10_std": -0.03318867336370514, + "nauc_mrr_at_1_diff1": 0.8506863588556931, + "nauc_mrr_at_1_max": 0.645628789135785, + "nauc_mrr_at_1_std": -0.06191657522827031, + "nauc_mrr_at_20_diff1": 0.8285819113195612, + "nauc_mrr_at_20_max": 0.6775920634659378, + "nauc_mrr_at_20_std": -0.033782092153909746, + "nauc_mrr_at_3_diff1": 0.8293140971183676, + "nauc_mrr_at_3_max": 0.6788657739487108, + "nauc_mrr_at_3_std": -0.03904228069624038, + "nauc_mrr_at_5_diff1": 0.8278238115943125, + "nauc_mrr_at_5_max": 0.6778270133394841, + "nauc_mrr_at_5_std": -0.036293201723127694, + "nauc_ndcg_at_1000_diff1": 0.823199098700142, + "nauc_ndcg_at_1000_max": 0.6812034247293801, + "nauc_ndcg_at_1000_std": -0.028508509319395733, + "nauc_ndcg_at_100_diff1": 0.821633159731435, + "nauc_ndcg_at_100_max": 0.6826748139014519, + "nauc_ndcg_at_100_std": -0.02714633476731751, + "nauc_ndcg_at_10_diff1": 0.8182903994006059, + "nauc_ndcg_at_10_max": 0.6889584312261682, + "nauc_ndcg_at_10_std": -0.014252080306379408, + "nauc_ndcg_at_1_diff1": 0.8506863588556931, + "nauc_ndcg_at_1_max": 0.645628789135785, + "nauc_ndcg_at_1_std": -0.06191657522827031, + "nauc_ndcg_at_20_diff1": 0.8198789282274658, + "nauc_ndcg_at_20_max": 0.688959043317764, + "nauc_ndcg_at_20_std": -0.016185846906781556, + "nauc_ndcg_at_3_diff1": 0.8210544479985309, + "nauc_ndcg_at_3_max": 0.6898284750281354, + "nauc_ndcg_at_3_std": -0.0302774137516056, + "nauc_ndcg_at_5_diff1": 0.8177317252542665, + "nauc_ndcg_at_5_max": 0.6884306708611059, + "nauc_ndcg_at_5_std": -0.02367601395365958, + "nauc_precision_at_1000_diff1": 0.712357013205843, + "nauc_precision_at_1000_max": 0.5954259563274978, + "nauc_precision_at_1000_std": -0.06453769813732958, + "nauc_precision_at_100_diff1": 0.706646143169478, + "nauc_precision_at_100_max": 0.7221908402194197, + "nauc_precision_at_100_std": 0.03639030914288442, + "nauc_precision_at_10_diff1": 0.7569911710504575, + "nauc_precision_at_10_max": 0.7597057792213979, + "nauc_precision_at_10_std": 0.11632934387302263, + "nauc_precision_at_1_diff1": 0.8506863588556931, + "nauc_precision_at_1_max": 0.645628789135785, + "nauc_precision_at_1_std": -0.06191657522827031, + "nauc_precision_at_20_diff1": 0.7491685968954767, + "nauc_precision_at_20_max": 0.7848940148220587, + "nauc_precision_at_20_std": 0.14477347988107261, + "nauc_precision_at_3_diff1": 0.7878186792721571, + "nauc_precision_at_3_max": 0.7331020934353045, + "nauc_precision_at_3_std": 0.004957741917380896, + "nauc_precision_at_5_diff1": 0.7696340948709872, + "nauc_precision_at_5_max": 0.736957286199771, + "nauc_precision_at_5_std": 0.037822933962717555, + "nauc_recall_at_1000_diff1": 0.7123570132058368, + "nauc_recall_at_1000_max": 0.5954259563275034, + "nauc_recall_at_1000_std": -0.06453769813732672, + "nauc_recall_at_100_diff1": 0.7066461431694706, + "nauc_recall_at_100_max": 0.7221908402194223, + "nauc_recall_at_100_std": 0.03639030914287266, + "nauc_recall_at_10_diff1": 0.7569911710504581, + "nauc_recall_at_10_max": 0.7597057792213957, + "nauc_recall_at_10_std": 0.11632934387302335, + "nauc_recall_at_1_diff1": 0.8506863588556931, + "nauc_recall_at_1_max": 0.645628789135785, + "nauc_recall_at_1_std": -0.06191657522827031, + "nauc_recall_at_20_diff1": 0.7491685968954777, + "nauc_recall_at_20_max": 0.784894014822059, + "nauc_recall_at_20_std": 0.14477347988107575, + "nauc_recall_at_3_diff1": 0.7878186792721584, + "nauc_recall_at_3_max": 0.7331020934353042, + "nauc_recall_at_3_std": 0.004957741917380415, + "nauc_recall_at_5_diff1": 0.7696340948709854, + "nauc_recall_at_5_max": 0.7369572861997701, + "nauc_recall_at_5_std": 0.037822933962717256, + "ndcg_at_1": 0.74354, + "ndcg_at_10": 0.83479, + "ndcg_at_100": 0.84656, + "ndcg_at_1000": 0.84851, + "ndcg_at_20": 0.84035, + "ndcg_at_3": 0.81422, + "ndcg_at_5": 0.82461, + "precision_at_1": 0.74354, + "precision_at_10": 0.09201, + "precision_at_100": 0.00975, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04708, + "precision_at_3": 0.28786, + "precision_at_5": 0.17776, + "recall_at_1": 0.74354, + "recall_at_10": 0.92009, + "recall_at_100": 0.97508, + "recall_at_1000": 0.99149, + "recall_at_20": 0.94166, + "recall_at_3": 0.86357, + "recall_at_5": 0.88879 + }, + { + "hf_subset": "go", + "languages": [ + "go-Code" + ], + "main_score": 0.91093, + "map_at_1": 0.83957, + "map_at_10": 0.89095, + "map_at_100": 0.89219, + "map_at_1000": 0.89221, + "map_at_20": 0.89181, + "map_at_3": 0.88472, + "map_at_5": 0.88866, + "mrr_at_1": 0.8395715341049003, + "mrr_at_10": 0.890947231309824, + "mrr_at_100": 0.8921908275588152, + "mrr_at_1000": 0.8922056604897018, + "mrr_at_20": 0.8918054785129605, + "mrr_at_3": 0.8847164081096583, + "mrr_at_5": 0.8886624805056191, + "nauc_map_at_1000_diff1": 0.8624973579936616, + "nauc_map_at_1000_max": 0.4376713186644932, + "nauc_map_at_1000_std": -0.4720109532688898, + "nauc_map_at_100_diff1": 0.8624862049034168, + "nauc_map_at_100_max": 0.43768810225851856, + "nauc_map_at_100_std": -0.47204265214349733, + "nauc_map_at_10_diff1": 0.8625350003349301, + "nauc_map_at_10_max": 0.43771945888748304, + "nauc_map_at_10_std": -0.4762265974147625, + "nauc_map_at_1_diff1": 0.8698798340919, + "nauc_map_at_1_max": 0.4302404335576899, + "nauc_map_at_1_std": -0.41469271817341957, + "nauc_map_at_20_diff1": 0.8625325068130977, + "nauc_map_at_20_max": 0.43807273768156374, + "nauc_map_at_20_std": -0.4731520043996017, + "nauc_map_at_3_diff1": 0.8610288537323795, + "nauc_map_at_3_max": 0.4379879378738717, + "nauc_map_at_3_std": -0.48546995258168885, + "nauc_map_at_5_diff1": 0.8618610042665852, + "nauc_map_at_5_max": 0.434814674201994, + "nauc_map_at_5_std": -0.48409761968549087, + "nauc_mrr_at_1000_diff1": 0.8624973579936616, + "nauc_mrr_at_1000_max": 0.4376713186644932, + "nauc_mrr_at_1000_std": -0.4720109532688898, + "nauc_mrr_at_100_diff1": 0.8624862049034168, + "nauc_mrr_at_100_max": 0.43768810225851856, + "nauc_mrr_at_100_std": -0.47204265214349733, + "nauc_mrr_at_10_diff1": 0.8625350003349301, + "nauc_mrr_at_10_max": 0.43771945888748304, + "nauc_mrr_at_10_std": -0.4762265974147625, + "nauc_mrr_at_1_diff1": 0.8698798340919, + "nauc_mrr_at_1_max": 0.4302404335576899, + "nauc_mrr_at_1_std": -0.41469271817341957, + "nauc_mrr_at_20_diff1": 0.8625325068130977, + "nauc_mrr_at_20_max": 0.43807273768156374, + "nauc_mrr_at_20_std": -0.4731520043996017, + "nauc_mrr_at_3_diff1": 0.8610288537323795, + "nauc_mrr_at_3_max": 0.4379879378738717, + "nauc_mrr_at_3_std": -0.48546995258168885, + "nauc_mrr_at_5_diff1": 0.8618610042665852, + "nauc_mrr_at_5_max": 0.434814674201994, + "nauc_mrr_at_5_std": -0.48409761968549087, + "nauc_ndcg_at_1000_diff1": 0.861495231100838, + "nauc_ndcg_at_1000_max": 0.4407212654503798, + "nauc_ndcg_at_1000_std": -0.47621639792761516, + "nauc_ndcg_at_100_diff1": 0.8609983539633851, + "nauc_ndcg_at_100_max": 0.4412175911440964, + "nauc_ndcg_at_100_std": -0.4760497182589526, + "nauc_ndcg_at_10_diff1": 0.8615470113116336, + "nauc_ndcg_at_10_max": 0.4422094281301947, + "nauc_ndcg_at_10_std": -0.49680350658225253, + "nauc_ndcg_at_1_diff1": 0.8698798340919, + "nauc_ndcg_at_1_max": 0.4302404335576899, + "nauc_ndcg_at_1_std": -0.41469271817341957, + "nauc_ndcg_at_20_diff1": 0.86153687239383, + "nauc_ndcg_at_20_max": 0.44413683910958995, + "nauc_ndcg_at_20_std": -0.4839097978043139, + "nauc_ndcg_at_3_diff1": 0.8581806042008513, + "nauc_ndcg_at_3_max": 0.4405606453052025, + "nauc_ndcg_at_3_std": -0.5177857306868798, + "nauc_ndcg_at_5_diff1": 0.8597978206190466, + "nauc_ndcg_at_5_max": 0.4341162519222535, + "nauc_ndcg_at_5_std": -0.5175858549128195, + "nauc_precision_at_1000_diff1": 0.8008890508034311, + "nauc_precision_at_1000_max": 0.9423440421215071, + "nauc_precision_at_1000_std": 0.5593973044712788, + "nauc_precision_at_100_diff1": 0.7593187211974605, + "nauc_precision_at_100_max": 0.6707999533740335, + "nauc_precision_at_100_std": -0.08479533178162366, + "nauc_precision_at_10_diff1": 0.8556280199155816, + "nauc_precision_at_10_max": 0.49465038213591783, + "nauc_precision_at_10_std": -0.687017504439551, + "nauc_precision_at_1_diff1": 0.8698798340919, + "nauc_precision_at_1_max": 0.4302404335576899, + "nauc_precision_at_1_std": -0.41469271817341957, + "nauc_precision_at_20_diff1": 0.8516456431737308, + "nauc_precision_at_20_max": 0.571052314590146, + "nauc_precision_at_20_std": -0.5720616563582712, + "nauc_precision_at_3_diff1": 0.8442151445378558, + "nauc_precision_at_3_max": 0.45323547924059393, + "nauc_precision_at_3_std": -0.6834786794101256, + "nauc_precision_at_5_diff1": 0.8472626746318639, + "nauc_precision_at_5_max": 0.4265450965313161, + "nauc_precision_at_5_std": -0.7463108495140407, + "nauc_recall_at_1000_diff1": 0.8008890508033205, + "nauc_recall_at_1000_max": 0.9423440421215782, + "nauc_recall_at_1000_std": 0.5593973044711981, + "nauc_recall_at_100_diff1": 0.7593187211975196, + "nauc_recall_at_100_max": 0.6707999533740575, + "nauc_recall_at_100_std": -0.08479533178153222, + "nauc_recall_at_10_diff1": 0.8556280199155815, + "nauc_recall_at_10_max": 0.4946503821359128, + "nauc_recall_at_10_std": -0.6870175044395225, + "nauc_recall_at_1_diff1": 0.8698798340919, + "nauc_recall_at_1_max": 0.4302404335576899, + "nauc_recall_at_1_std": -0.41469271817341957, + "nauc_recall_at_20_diff1": 0.8516456431737305, + "nauc_recall_at_20_max": 0.5710523145901444, + "nauc_recall_at_20_std": -0.5720616563582539, + "nauc_recall_at_3_diff1": 0.8442151445378547, + "nauc_recall_at_3_max": 0.45323547924059293, + "nauc_recall_at_3_std": -0.6834786794101289, + "nauc_recall_at_5_diff1": 0.8472626746318647, + "nauc_recall_at_5_max": 0.4265450965313161, + "nauc_recall_at_5_std": -0.7463108495140318, + "ndcg_at_1": 0.83957, + "ndcg_at_10": 0.91093, + "ndcg_at_100": 0.91644, + "ndcg_at_1000": 0.9168, + "ndcg_at_20": 0.91397, + "ndcg_at_3": 0.89843, + "ndcg_at_5": 0.90551, + "precision_at_1": 0.83957, + "precision_at_10": 0.09714, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "precision_at_20": 0.04916, + "precision_at_3": 0.31261, + "precision_at_5": 0.19099, + "recall_at_1": 0.83957, + "recall_at_10": 0.97144, + "recall_at_100": 0.99594, + "recall_at_1000": 0.99877, + "recall_at_20": 0.98326, + "recall_at_3": 0.93782, + "recall_at_5": 0.95494 + }, + { + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ], + "main_score": 0.86521, + "map_at_1": 0.77082, + "map_at_10": 0.83552, + "map_at_100": 0.83687, + "map_at_1000": 0.8369, + "map_at_20": 0.83642, + "map_at_3": 0.82117, + "map_at_5": 0.83164, + "mrr_at_1": 0.7708168120539255, + "mrr_at_10": 0.8355194919627917, + "mrr_at_100": 0.8368701866558312, + "mrr_at_1000": 0.836903032381438, + "mrr_at_20": 0.8364219478925247, + "mrr_at_3": 0.8211736716891364, + "mrr_at_5": 0.8316415543219676, + "nauc_map_at_1000_diff1": 0.7964679377977846, + "nauc_map_at_1000_max": 0.7074499431961593, + "nauc_map_at_1000_std": -0.14853409296691983, + "nauc_map_at_100_diff1": 0.7964499589721636, + "nauc_map_at_100_max": 0.7074759843854231, + "nauc_map_at_100_std": -0.14844258734825894, + "nauc_map_at_10_diff1": 0.7953290732095073, + "nauc_map_at_10_max": 0.7076525352436301, + "nauc_map_at_10_std": -0.14744825626716188, + "nauc_map_at_1_diff1": 0.8227728044611361, + "nauc_map_at_1_max": 0.6929382107125889, + "nauc_map_at_1_std": -0.1602966665531533, + "nauc_map_at_20_diff1": 0.7957275942354842, + "nauc_map_at_20_max": 0.7073838311952617, + "nauc_map_at_20_std": -0.1486925109860625, + "nauc_map_at_3_diff1": 0.7947572156907858, + "nauc_map_at_3_max": 0.7085276903670265, + "nauc_map_at_3_std": -0.157557920056126, + "nauc_map_at_5_diff1": 0.7942518354242956, + "nauc_map_at_5_max": 0.7069226154953026, + "nauc_map_at_5_std": -0.15473047773228993, + "nauc_mrr_at_1000_diff1": 0.7964679377977846, + "nauc_mrr_at_1000_max": 0.7074499431961593, + "nauc_mrr_at_1000_std": -0.14853409296691983, + "nauc_mrr_at_100_diff1": 0.7964499589721636, + "nauc_mrr_at_100_max": 0.7074759843854231, + "nauc_mrr_at_100_std": -0.14844258734825894, + "nauc_mrr_at_10_diff1": 0.7953290732095073, + "nauc_mrr_at_10_max": 0.7076525352436301, + "nauc_mrr_at_10_std": -0.14744825626716188, + "nauc_mrr_at_1_diff1": 0.8227728044611361, + "nauc_mrr_at_1_max": 0.6929382107125889, + "nauc_mrr_at_1_std": -0.1602966665531533, + "nauc_mrr_at_20_diff1": 0.7957275942354842, + "nauc_mrr_at_20_max": 0.7073838311952617, + "nauc_mrr_at_20_std": -0.1486925109860625, + "nauc_mrr_at_3_diff1": 0.7947572156907858, + "nauc_mrr_at_3_max": 0.7085276903670265, + "nauc_mrr_at_3_std": -0.157557920056126, + "nauc_mrr_at_5_diff1": 0.7942518354242956, + "nauc_mrr_at_5_max": 0.7069226154953026, + "nauc_mrr_at_5_std": -0.15473047773228993, + "nauc_ndcg_at_1000_diff1": 0.7915083604230807, + "nauc_ndcg_at_1000_max": 0.7111787439983639, + "nauc_ndcg_at_1000_std": -0.1428015599243118, + "nauc_ndcg_at_100_diff1": 0.7903711984795275, + "nauc_ndcg_at_100_max": 0.7120364489826714, + "nauc_ndcg_at_100_std": -0.13915390197654143, + "nauc_ndcg_at_10_diff1": 0.7838949093885134, + "nauc_ndcg_at_10_max": 0.7126426621928835, + "nauc_ndcg_at_10_std": -0.13497441432215215, + "nauc_ndcg_at_1_diff1": 0.8227728044611361, + "nauc_ndcg_at_1_max": 0.6929382107125889, + "nauc_ndcg_at_1_std": -0.1602966665531533, + "nauc_ndcg_at_20_diff1": 0.7851349613885007, + "nauc_ndcg_at_20_max": 0.7113292440385801, + "nauc_ndcg_at_20_std": -0.14035580557966088, + "nauc_ndcg_at_3_diff1": 0.783820255722769, + "nauc_ndcg_at_3_max": 0.7132668614112853, + "nauc_ndcg_at_3_std": -0.1604098840431829, + "nauc_ndcg_at_5_diff1": 0.7814070256474213, + "nauc_ndcg_at_5_max": 0.7104026107145508, + "nauc_ndcg_at_5_std": -0.15592128711464068, + "nauc_precision_at_1000_diff1": 0.772067992384371, + "nauc_precision_at_1000_max": 0.8555623308018635, + "nauc_precision_at_1000_std": -0.030163245425889006, + "nauc_precision_at_100_diff1": 0.7101994754486307, + "nauc_precision_at_100_max": 0.8230803790261785, + "nauc_precision_at_100_std": 0.12633506693103702, + "nauc_precision_at_10_diff1": 0.6704993913892164, + "nauc_precision_at_10_max": 0.7623886937696508, + "nauc_precision_at_10_std": 0.015250700897226325, + "nauc_precision_at_1_diff1": 0.8227728044611361, + "nauc_precision_at_1_max": 0.6929382107125889, + "nauc_precision_at_1_std": -0.1602966665531533, + "nauc_precision_at_20_diff1": 0.6366698683846803, + "nauc_precision_at_20_max": 0.7614090190912508, + "nauc_precision_at_20_std": -0.005871594787522816, + "nauc_precision_at_3_diff1": 0.7386799722582569, + "nauc_precision_at_3_max": 0.7322059804843699, + "nauc_precision_at_3_std": -0.17398868434567205, + "nauc_precision_at_5_diff1": 0.7002914493269555, + "nauc_precision_at_5_max": 0.7300069716131422, + "nauc_precision_at_5_std": -0.16614393845477327, + "nauc_recall_at_1000_diff1": 0.7720679923843755, + "nauc_recall_at_1000_max": 0.8555623308018376, + "nauc_recall_at_1000_std": -0.03016324542591852, + "nauc_recall_at_100_diff1": 0.7101994754486401, + "nauc_recall_at_100_max": 0.8230803790261768, + "nauc_recall_at_100_std": 0.12633506693104876, + "nauc_recall_at_10_diff1": 0.6704993913892174, + "nauc_recall_at_10_max": 0.7623886937696521, + "nauc_recall_at_10_std": 0.015250700897222571, + "nauc_recall_at_1_diff1": 0.8227728044611361, + "nauc_recall_at_1_max": 0.6929382107125889, + "nauc_recall_at_1_std": -0.1602966665531533, + "nauc_recall_at_20_diff1": 0.6366698683846833, + "nauc_recall_at_20_max": 0.7614090190912469, + "nauc_recall_at_20_std": -0.005871594787518352, + "nauc_recall_at_3_diff1": 0.7386799722582569, + "nauc_recall_at_3_max": 0.7322059804843682, + "nauc_recall_at_3_std": -0.17398868434566936, + "nauc_recall_at_5_diff1": 0.7002914493269549, + "nauc_recall_at_5_max": 0.7300069716131398, + "nauc_recall_at_5_std": -0.16614393845477526, + "ndcg_at_1": 0.77082, + "ndcg_at_10": 0.86521, + "ndcg_at_100": 0.87117, + "ndcg_at_1000": 0.87231, + "ndcg_at_20": 0.86858, + "ndcg_at_3": 0.837, + "ndcg_at_5": 0.85591, + "precision_at_1": 0.77082, + "precision_at_10": 0.09572, + "precision_at_100": 0.00983, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04853, + "precision_at_3": 0.29421, + "precision_at_5": 0.18573, + "recall_at_1": 0.77082, + "recall_at_10": 0.95718, + "recall_at_100": 0.98335, + "recall_at_1000": 0.99286, + "recall_at_20": 0.97066, + "recall_at_3": 0.88263, + "recall_at_5": 0.92863 + }, + { + "hf_subset": "java", + "languages": [ + "java-Code" + ], + "main_score": 0.87269, + "map_at_1": 0.78841, + "map_at_10": 0.84821, + "map_at_100": 0.84966, + "map_at_1000": 0.84972, + "map_at_20": 0.84912, + "map_at_3": 0.83949, + "map_at_5": 0.84514, + "mrr_at_1": 0.788407120036513, + "mrr_at_10": 0.8482137604775662, + "mrr_at_100": 0.8496600236904306, + "mrr_at_1000": 0.8497188010768014, + "mrr_at_20": 0.8491215021114384, + "mrr_at_3": 0.8394949033926656, + "mrr_at_5": 0.8451361630914327, + "nauc_map_at_1000_diff1": 0.8129745727169212, + "nauc_map_at_1000_max": 0.613738482977934, + "nauc_map_at_1000_std": -0.14611667076737653, + "nauc_map_at_100_diff1": 0.8129611575498952, + "nauc_map_at_100_max": 0.6138415642876098, + "nauc_map_at_100_std": -0.1460020471629818, + "nauc_map_at_10_diff1": 0.8124492887826994, + "nauc_map_at_10_max": 0.6147680472896054, + "nauc_map_at_10_std": -0.147216428942423, + "nauc_map_at_1_diff1": 0.8307021879054804, + "nauc_map_at_1_max": 0.5745256621613931, + "nauc_map_at_1_std": -0.1694155323782564, + "nauc_map_at_20_diff1": 0.8128505757844011, + "nauc_map_at_20_max": 0.6143691967628063, + "nauc_map_at_20_std": -0.14583239037636378, + "nauc_map_at_3_diff1": 0.810101519179173, + "nauc_map_at_3_max": 0.6145133191553263, + "nauc_map_at_3_std": -0.150980901015753, + "nauc_map_at_5_diff1": 0.8116287036289346, + "nauc_map_at_5_max": 0.6159538703827228, + "nauc_map_at_5_std": -0.14843988367319086, + "nauc_mrr_at_1000_diff1": 0.8129745727169212, + "nauc_mrr_at_1000_max": 0.613738482977934, + "nauc_mrr_at_1000_std": -0.14611667076737653, + "nauc_mrr_at_100_diff1": 0.8129611575498952, + "nauc_mrr_at_100_max": 0.6138415642876098, + "nauc_mrr_at_100_std": -0.1460020471629818, + "nauc_mrr_at_10_diff1": 0.8124492887826994, + "nauc_mrr_at_10_max": 0.6147680472896054, + "nauc_mrr_at_10_std": -0.147216428942423, + "nauc_mrr_at_1_diff1": 0.8307021879054804, + "nauc_mrr_at_1_max": 0.5745256621613931, + "nauc_mrr_at_1_std": -0.1694155323782564, + "nauc_mrr_at_20_diff1": 0.8128505757844011, + "nauc_mrr_at_20_max": 0.6143691967628063, + "nauc_mrr_at_20_std": -0.14583239037636378, + "nauc_mrr_at_3_diff1": 0.810101519179173, + "nauc_mrr_at_3_max": 0.6145133191553263, + "nauc_mrr_at_3_std": -0.150980901015753, + "nauc_mrr_at_5_diff1": 0.8116287036289346, + "nauc_mrr_at_5_max": 0.6159538703827228, + "nauc_mrr_at_5_std": -0.14843988367319086, + "nauc_ndcg_at_1000_diff1": 0.8093566924723379, + "nauc_ndcg_at_1000_max": 0.6239699094971305, + "nauc_ndcg_at_1000_std": -0.13127328464119836, + "nauc_ndcg_at_100_diff1": 0.8087553513050011, + "nauc_ndcg_at_100_max": 0.6272180161053774, + "nauc_ndcg_at_100_std": -0.12649445624854963, + "nauc_ndcg_at_10_diff1": 0.80608466242348, + "nauc_ndcg_at_10_max": 0.6324038696080356, + "nauc_ndcg_at_10_std": -0.1310528733578738, + "nauc_ndcg_at_1_diff1": 0.8307021879054804, + "nauc_ndcg_at_1_max": 0.5745256621613931, + "nauc_ndcg_at_1_std": -0.1694155323782564, + "nauc_ndcg_at_20_diff1": 0.8077174027485288, + "nauc_ndcg_at_20_max": 0.631297458573586, + "nauc_ndcg_at_20_std": -0.1243612330644508, + "nauc_ndcg_at_3_diff1": 0.801778653998912, + "nauc_ndcg_at_3_max": 0.630291128397288, + "nauc_ndcg_at_3_std": -0.14201414014064817, + "nauc_ndcg_at_5_diff1": 0.8041507067529043, + "nauc_ndcg_at_5_max": 0.6345670343701447, + "nauc_ndcg_at_5_std": -0.13566365227692753, + "nauc_precision_at_1000_diff1": 0.7389525267370539, + "nauc_precision_at_1000_max": 0.8346793489687887, + "nauc_precision_at_1000_std": 0.43369571434954207, + "nauc_precision_at_100_diff1": 0.7533082548811534, + "nauc_precision_at_100_max": 0.8508356966408422, + "nauc_precision_at_100_std": 0.3091532174961418, + "nauc_precision_at_10_diff1": 0.7596120115397336, + "nauc_precision_at_10_max": 0.7687957072290978, + "nauc_precision_at_10_std": 0.004829699864762251, + "nauc_precision_at_1_diff1": 0.8307021879054804, + "nauc_precision_at_1_max": 0.5745256621613931, + "nauc_precision_at_1_std": -0.1694155323782564, + "nauc_precision_at_20_diff1": 0.7637431221666497, + "nauc_precision_at_20_max": 0.8017866322470198, + "nauc_precision_at_20_std": 0.12914278225658554, + "nauc_precision_at_3_diff1": 0.7655984367763438, + "nauc_precision_at_3_max": 0.6986960822047167, + "nauc_precision_at_3_std": -0.10230045456623477, + "nauc_precision_at_5_diff1": 0.7632796311102129, + "nauc_precision_at_5_max": 0.7388101356144463, + "nauc_precision_at_5_std": -0.061510867874405094, + "nauc_recall_at_1000_diff1": 0.7389525267370504, + "nauc_recall_at_1000_max": 0.8346793489687493, + "nauc_recall_at_1000_std": 0.43369571434955073, + "nauc_recall_at_100_diff1": 0.7533082548811574, + "nauc_recall_at_100_max": 0.850835696640845, + "nauc_recall_at_100_std": 0.3091532174961395, + "nauc_recall_at_10_diff1": 0.7596120115397329, + "nauc_recall_at_10_max": 0.768795707229096, + "nauc_recall_at_10_std": 0.004829699864760997, + "nauc_recall_at_1_diff1": 0.8307021879054804, + "nauc_recall_at_1_max": 0.5745256621613931, + "nauc_recall_at_1_std": -0.1694155323782564, + "nauc_recall_at_20_diff1": 0.7637431221666506, + "nauc_recall_at_20_max": 0.8017866322470139, + "nauc_recall_at_20_std": 0.129142782256592, + "nauc_recall_at_3_diff1": 0.7655984367763459, + "nauc_recall_at_3_max": 0.6986960822047196, + "nauc_recall_at_3_std": -0.10230045456623338, + "nauc_recall_at_5_diff1": 0.7632796311102106, + "nauc_recall_at_5_max": 0.7388101356144475, + "nauc_recall_at_5_std": -0.06151086787440187, + "ndcg_at_1": 0.78841, + "ndcg_at_10": 0.87269, + "ndcg_at_100": 0.87977, + "ndcg_at_1000": 0.88133, + "ndcg_at_20": 0.87599, + "ndcg_at_3": 0.85521, + "ndcg_at_5": 0.86531, + "precision_at_1": 0.78841, + "precision_at_10": 0.09474, + "precision_at_100": 0.00981, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04802, + "precision_at_3": 0.30014, + "precision_at_5": 0.18496, + "recall_at_1": 0.78841, + "recall_at_10": 0.94742, + "recall_at_100": 0.98056, + "recall_at_1000": 0.99288, + "recall_at_20": 0.96047, + "recall_at_3": 0.90041, + "recall_at_5": 0.92478 + }, + { + "hf_subset": "php", + "languages": [ + "php-Code" + ], + "main_score": 0.89384, + "map_at_1": 0.81997, + "map_at_10": 0.87221, + "map_at_100": 0.87375, + "map_at_1000": 0.87378, + "map_at_20": 0.87328, + "map_at_3": 0.86429, + "map_at_5": 0.86916, + "mrr_at_1": 0.8199657485371771, + "mrr_at_10": 0.8722106125167395, + "mrr_at_100": 0.8737512926935147, + "mrr_at_1000": 0.8737841767615039, + "mrr_at_20": 0.8732764206073328, + "mrr_at_3": 0.8642904714333318, + "mrr_at_5": 0.869157033442753, + "nauc_map_at_1000_diff1": 0.8498375890205129, + "nauc_map_at_1000_max": 0.5216908309618805, + "nauc_map_at_1000_std": -0.35027524704460916, + "nauc_map_at_100_diff1": 0.8498366301880896, + "nauc_map_at_100_max": 0.5217720776838463, + "nauc_map_at_100_std": -0.3501620836480514, + "nauc_map_at_10_diff1": 0.8495191430457029, + "nauc_map_at_10_max": 0.5231158183948366, + "nauc_map_at_10_std": -0.3513714349074112, + "nauc_map_at_1_diff1": 0.8624870439500669, + "nauc_map_at_1_max": 0.49202395255573766, + "nauc_map_at_1_std": -0.33716612816706876, + "nauc_map_at_20_diff1": 0.84973933039348, + "nauc_map_at_20_max": 0.522266012979943, + "nauc_map_at_20_std": -0.3504159739087031, + "nauc_map_at_3_diff1": 0.8469068283876815, + "nauc_map_at_3_max": 0.5234374344958552, + "nauc_map_at_3_std": -0.35693665817322634, + "nauc_map_at_5_diff1": 0.8484059835122619, + "nauc_map_at_5_max": 0.5229585814594443, + "nauc_map_at_5_std": -0.35511386765161607, + "nauc_mrr_at_1000_diff1": 0.8498375890205129, + "nauc_mrr_at_1000_max": 0.5216908309618805, + "nauc_mrr_at_1000_std": -0.35027524704460916, + "nauc_mrr_at_100_diff1": 0.8498366301880896, + "nauc_mrr_at_100_max": 0.5217720776838463, + "nauc_mrr_at_100_std": -0.3501620836480514, + "nauc_mrr_at_10_diff1": 0.8495191430457029, + "nauc_mrr_at_10_max": 0.5231158183948366, + "nauc_mrr_at_10_std": -0.3513714349074112, + "nauc_mrr_at_1_diff1": 0.8624870439500669, + "nauc_mrr_at_1_max": 0.49202395255573766, + "nauc_mrr_at_1_std": -0.33716612816706876, + "nauc_mrr_at_20_diff1": 0.84973933039348, + "nauc_mrr_at_20_max": 0.522266012979943, + "nauc_mrr_at_20_std": -0.3504159739087031, + "nauc_mrr_at_3_diff1": 0.8469068283876815, + "nauc_mrr_at_3_max": 0.5234374344958552, + "nauc_mrr_at_3_std": -0.35693665817322634, + "nauc_mrr_at_5_diff1": 0.8484059835122619, + "nauc_mrr_at_5_max": 0.5229585814594443, + "nauc_mrr_at_5_std": -0.35511386765161607, + "nauc_ndcg_at_1000_diff1": 0.848458448603008, + "nauc_ndcg_at_1000_max": 0.5277266190305249, + "nauc_ndcg_at_1000_std": -0.34888069489815415, + "nauc_ndcg_at_100_diff1": 0.8484381394537597, + "nauc_ndcg_at_100_max": 0.5301563101161627, + "nauc_ndcg_at_100_std": -0.3450921162858631, + "nauc_ndcg_at_10_diff1": 0.8466548583706869, + "nauc_ndcg_at_10_max": 0.5369134601286215, + "nauc_ndcg_at_10_std": -0.3523415287527687, + "nauc_ndcg_at_1_diff1": 0.8624870439500669, + "nauc_ndcg_at_1_max": 0.49202395255573766, + "nauc_ndcg_at_1_std": -0.33716612816706876, + "nauc_ndcg_at_20_diff1": 0.8476038572414958, + "nauc_ndcg_at_20_max": 0.5336813159950785, + "nauc_ndcg_at_20_std": -0.3482141071146032, + "nauc_ndcg_at_3_diff1": 0.841222575178288, + "nauc_ndcg_at_3_max": 0.535458597204313, + "nauc_ndcg_at_3_std": -0.3657805029126134, + "nauc_ndcg_at_5_diff1": 0.8438081551698715, + "nauc_ndcg_at_5_max": 0.535309540463121, + "nauc_ndcg_at_5_std": -0.36263715268733626, + "nauc_precision_at_1000_diff1": 0.8552761499735974, + "nauc_precision_at_1000_max": 0.8557039121692811, + "nauc_precision_at_1000_std": 0.26076840198712997, + "nauc_precision_at_100_diff1": 0.8471683047722709, + "nauc_precision_at_100_max": 0.786718703783981, + "nauc_precision_at_100_std": 0.07427629853478684, + "nauc_precision_at_10_diff1": 0.8266742923426185, + "nauc_precision_at_10_max": 0.6587095967797794, + "nauc_precision_at_10_std": -0.3479989719383861, + "nauc_precision_at_1_diff1": 0.8624870439500669, + "nauc_precision_at_1_max": 0.49202395255573766, + "nauc_precision_at_1_std": -0.33716612816706876, + "nauc_precision_at_20_diff1": 0.8299219098545612, + "nauc_precision_at_20_max": 0.6830891426964217, + "nauc_precision_at_20_std": -0.28159133053121294, + "nauc_precision_at_3_diff1": 0.8161302076796814, + "nauc_precision_at_3_max": 0.5888366904824758, + "nauc_precision_at_3_std": -0.4057293703314752, + "nauc_precision_at_5_diff1": 0.8184587610537042, + "nauc_precision_at_5_max": 0.6058183720409386, + "nauc_precision_at_5_std": -0.40547760770888214, + "nauc_recall_at_1000_diff1": 0.8552761499733693, + "nauc_recall_at_1000_max": 0.8557039121690827, + "nauc_recall_at_1000_std": 0.2607684019872863, + "nauc_recall_at_100_diff1": 0.847168304772257, + "nauc_recall_at_100_max": 0.7867187037839709, + "nauc_recall_at_100_std": 0.07427629853478299, + "nauc_recall_at_10_diff1": 0.826674292342617, + "nauc_recall_at_10_max": 0.6587095967797796, + "nauc_recall_at_10_std": -0.347998971938389, + "nauc_recall_at_1_diff1": 0.8624870439500669, + "nauc_recall_at_1_max": 0.49202395255573766, + "nauc_recall_at_1_std": -0.33716612816706876, + "nauc_recall_at_20_diff1": 0.829921909854569, + "nauc_recall_at_20_max": 0.6830891426964256, + "nauc_recall_at_20_std": -0.28159133053120633, + "nauc_recall_at_3_diff1": 0.8161302076796801, + "nauc_recall_at_3_max": 0.5888366904824764, + "nauc_recall_at_3_std": -0.4057293703314801, + "nauc_recall_at_5_diff1": 0.8184587610537069, + "nauc_recall_at_5_max": 0.605818372040938, + "nauc_recall_at_5_std": -0.4054776077088785, + "ndcg_at_1": 0.81997, + "ndcg_at_10": 0.89384, + "ndcg_at_100": 0.90091, + "ndcg_at_1000": 0.9018, + "ndcg_at_20": 0.8977, + "ndcg_at_3": 0.87788, + "ndcg_at_5": 0.8866, + "precision_at_1": 0.81997, + "precision_at_10": 0.096, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "precision_at_20": 0.04876, + "precision_at_3": 0.30565, + "precision_at_5": 0.1876, + "recall_at_1": 0.81997, + "recall_at_10": 0.96004, + "recall_at_100": 0.99208, + "recall_at_1000": 0.99914, + "recall_at_20": 0.97524, + "recall_at_3": 0.91694, + "recall_at_5": 0.93799 + } + ] + }, + "task_name": "COIRCodeSearchNetRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackMT.json b/results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackMT.json new file mode 100644 index 000000000..a16a60bc5 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "evaluation_time": 1694.1078248023987, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.93581, + "map_at_1": 0.89177, + "map_at_10": 0.92293, + "map_at_100": 0.9239, + "map_at_1000": 0.92392, + "map_at_20": 0.92357, + "map_at_3": 0.91805, + "map_at_5": 0.92124, + "mrr_at_1": 0.891767718611132, + "mrr_at_10": 0.9229094710867727, + "mrr_at_100": 0.9238822901927145, + "mrr_at_1000": 0.9239035562398948, + "mrr_at_20": 0.9235530678666272, + "mrr_at_3": 0.9180412241721295, + "mrr_at_5": 0.92122718485602, + "nauc_map_at_1000_diff1": 0.9132273096224377, + "nauc_map_at_1000_max": 0.6791710992417915, + "nauc_map_at_1000_std": -0.7442383054373403, + "nauc_map_at_100_diff1": 0.9132246480919676, + "nauc_map_at_100_max": 0.6791988217232156, + "nauc_map_at_100_std": -0.7442982893903615, + "nauc_map_at_10_diff1": 0.9133091844723226, + "nauc_map_at_10_max": 0.6799200052857574, + "nauc_map_at_10_std": -0.7486264731079909, + "nauc_map_at_1_diff1": 0.9119451424067374, + "nauc_map_at_1_max": 0.6625392135499182, + "nauc_map_at_1_std": -0.6671862444546531, + "nauc_map_at_20_diff1": 0.91338096532563, + "nauc_map_at_20_max": 0.679501381727388, + "nauc_map_at_20_std": -0.7465414874093059, + "nauc_map_at_3_diff1": 0.9133378646831019, + "nauc_map_at_3_max": 0.6807178423980442, + "nauc_map_at_3_std": -0.748448215489277, + "nauc_map_at_5_diff1": 0.9131919404747947, + "nauc_map_at_5_max": 0.6798461966313379, + "nauc_map_at_5_std": -0.751772163451522, + "nauc_mrr_at_1000_diff1": 0.913199902377502, + "nauc_mrr_at_1000_max": 0.6792186725100839, + "nauc_mrr_at_1000_std": -0.7443167854141912, + "nauc_mrr_at_100_diff1": 0.9131972491331909, + "nauc_mrr_at_100_max": 0.6792463756203039, + "nauc_mrr_at_100_std": -0.7443767343342055, + "nauc_mrr_at_10_diff1": 0.9132821156688269, + "nauc_mrr_at_10_max": 0.679966801151313, + "nauc_mrr_at_10_std": -0.7487029985666573, + "nauc_mrr_at_1_diff1": 0.9119451424067374, + "nauc_mrr_at_1_max": 0.6625392135499182, + "nauc_mrr_at_1_std": -0.6671862444546531, + "nauc_mrr_at_20_diff1": 0.9133536515912956, + "nauc_mrr_at_20_max": 0.6795486643404076, + "nauc_mrr_at_20_std": -0.7466191099031442, + "nauc_mrr_at_3_diff1": 0.9133084003125199, + "nauc_mrr_at_3_max": 0.6807469130953063, + "nauc_mrr_at_3_std": -0.7486010698346194, + "nauc_mrr_at_5_diff1": 0.913165478177766, + "nauc_mrr_at_5_max": 0.6798920016571047, + "nauc_mrr_at_5_std": -0.7518463872642327, + "nauc_ndcg_at_1000_diff1": 0.9134667276431432, + "nauc_ndcg_at_1000_max": 0.6821491424591551, + "nauc_ndcg_at_1000_std": -0.7566992744128785, + "nauc_ndcg_at_100_diff1": 0.9132392901146443, + "nauc_ndcg_at_100_max": 0.6829729595248577, + "nauc_ndcg_at_100_std": -0.7571771214390896, + "nauc_ndcg_at_10_diff1": 0.9142077595372994, + "nauc_ndcg_at_10_max": 0.6868694743767313, + "nauc_ndcg_at_10_std": -0.7847824539632552, + "nauc_ndcg_at_1_diff1": 0.9119451424067374, + "nauc_ndcg_at_1_max": 0.6625392135499182, + "nauc_ndcg_at_1_std": -0.6671862444546531, + "nauc_ndcg_at_20_diff1": 0.9145155994342801, + "nauc_ndcg_at_20_max": 0.6853325586053745, + "nauc_ndcg_at_20_std": -0.7761552755050272, + "nauc_ndcg_at_3_diff1": 0.9140048288131363, + "nauc_ndcg_at_3_max": 0.6877879241382503, + "nauc_ndcg_at_3_std": -0.78145772024284, + "nauc_ndcg_at_5_diff1": 0.9138284059773454, + "nauc_ndcg_at_5_max": 0.6865210990623614, + "nauc_ndcg_at_5_std": -0.7912029324998576, + "nauc_precision_at_1000_diff1": 0.9346388819144805, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.6381917689216168, + "nauc_precision_at_100_diff1": 0.8814968521318447, + "nauc_precision_at_100_max": 0.7985443666227313, + "nauc_precision_at_100_std": -0.7160524535547103, + "nauc_precision_at_10_diff1": 0.9226837092071085, + "nauc_precision_at_10_max": 0.746573995994554, + "nauc_precision_at_10_std": -1.1001042456501373, + "nauc_precision_at_1_diff1": 0.9119451424067374, + "nauc_precision_at_1_max": 0.6625392135499182, + "nauc_precision_at_1_std": -0.6671862444546531, + "nauc_precision_at_20_diff1": 0.9320186301743918, + "nauc_precision_at_20_max": 0.7580862507034404, + "nauc_precision_at_20_std": -1.1447170564758076, + "nauc_precision_at_3_diff1": 0.917095571991953, + "nauc_precision_at_3_max": 0.7198340341782471, + "nauc_precision_at_3_std": -0.9318361438847609, + "nauc_precision_at_5_diff1": 0.9177735427940461, + "nauc_precision_at_5_max": 0.7260584615957594, + "nauc_precision_at_5_std": -1.0321989758142311, + "nauc_recall_at_1000_diff1": 0.9346388819147347, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.6381917689212638, + "nauc_recall_at_100_diff1": 0.8814968521317887, + "nauc_recall_at_100_max": 0.7985443666226893, + "nauc_recall_at_100_std": -0.7160524535547005, + "nauc_recall_at_10_diff1": 0.922683709207107, + "nauc_recall_at_10_max": 0.7465739959945585, + "nauc_recall_at_10_std": -1.1001042456501415, + "nauc_recall_at_1_diff1": 0.9119451424067374, + "nauc_recall_at_1_max": 0.6625392135499182, + "nauc_recall_at_1_std": -0.6671862444546531, + "nauc_recall_at_20_diff1": 0.9320186301743836, + "nauc_recall_at_20_max": 0.758086250703445, + "nauc_recall_at_20_std": -1.1447170564758093, + "nauc_recall_at_3_diff1": 0.9170955719919578, + "nauc_recall_at_3_max": 0.719834034178246, + "nauc_recall_at_3_std": -0.931836143884767, + "nauc_recall_at_5_diff1": 0.9177735427940513, + "nauc_recall_at_5_max": 0.7260584615957615, + "nauc_recall_at_5_std": -1.0321989758142347, + "ndcg_at_1": 0.89177, + "ndcg_at_10": 0.93581, + "ndcg_at_100": 0.94041, + "ndcg_at_1000": 0.94087, + "ndcg_at_20": 0.93813, + "ndcg_at_3": 0.92608, + "ndcg_at_5": 0.93183, + "precision_at_1": 0.89177, + "precision_at_10": 0.09752, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "precision_at_20": 0.04922, + "precision_at_3": 0.31639, + "precision_at_5": 0.19262, + "recall_at_1": 0.89177, + "recall_at_10": 0.97522, + "recall_at_100": 0.99638, + "recall_at_1000": 0.99985, + "recall_at_20": 0.98433, + "recall_at_3": 0.94916, + "recall_at_5": 0.96309 + } + ] + }, + "task_name": "CodeFeedbackMT" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackST.json b/results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackST.json new file mode 100644 index 000000000..3310e1d9e --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "evaluation_time": 3304.973326921463, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.90672, + "map_at_1": 0.8017, + "map_at_10": 0.87945, + "map_at_100": 0.88004, + "map_at_1000": 0.88004, + "map_at_20": 0.8799, + "map_at_3": 0.87248, + "map_at_5": 0.87796, + "mrr_at_1": 0.8008369002747078, + "mrr_at_10": 0.8789215440216771, + "mrr_at_100": 0.8795039935983602, + "mrr_at_1000": 0.8795113333851274, + "mrr_at_20": 0.8793646217104669, + "mrr_at_3": 0.871787090866489, + "mrr_at_5": 0.8773579079622562, + "nauc_map_at_1000_diff1": 0.8459599547053739, + "nauc_map_at_1000_max": 0.11563061625907411, + "nauc_map_at_1000_std": -0.6004444346870224, + "nauc_map_at_100_diff1": 0.8459537761332843, + "nauc_map_at_100_max": 0.11565642697551821, + "nauc_map_at_100_std": -0.6004854322246466, + "nauc_map_at_10_diff1": 0.8454000249494548, + "nauc_map_at_10_max": 0.1155440805584509, + "nauc_map_at_10_std": -0.6043783170183082, + "nauc_map_at_1_diff1": 0.86117489531929, + "nauc_map_at_1_max": 0.115654030601593, + "nauc_map_at_1_std": -0.535110885081521, + "nauc_map_at_20_diff1": 0.8458313379439, + "nauc_map_at_20_max": 0.11570044402077079, + "nauc_map_at_20_std": -0.6015198207708651, + "nauc_map_at_3_diff1": 0.843288071965302, + "nauc_map_at_3_max": 0.11306870549359703, + "nauc_map_at_3_std": -0.6166972599149546, + "nauc_map_at_5_diff1": 0.8444481140471144, + "nauc_map_at_5_max": 0.11431107322656027, + "nauc_map_at_5_std": -0.6107717915800889, + "nauc_mrr_at_1000_diff1": 0.8468763078545195, + "nauc_mrr_at_1000_max": 0.11654873092581682, + "nauc_mrr_at_1000_std": -0.5933698672573721, + "nauc_mrr_at_100_diff1": 0.8468700936878848, + "nauc_mrr_at_100_max": 0.11657439639783314, + "nauc_mrr_at_100_std": -0.5934111569519106, + "nauc_mrr_at_10_diff1": 0.8463101834390789, + "nauc_mrr_at_10_max": 0.11644216926632295, + "nauc_mrr_at_10_std": -0.5972967384330525, + "nauc_mrr_at_1_diff1": 0.8620926531200154, + "nauc_mrr_at_1_max": 0.11798672092672409, + "nauc_mrr_at_1_std": -0.5265061182155047, + "nauc_mrr_at_20_diff1": 0.8467473718323151, + "nauc_mrr_at_20_max": 0.1166210308966798, + "nauc_mrr_at_20_std": -0.5944638680106128, + "nauc_mrr_at_3_diff1": 0.8443542937601634, + "nauc_mrr_at_3_max": 0.11386252739390273, + "nauc_mrr_at_3_std": -0.6084726012330283, + "nauc_mrr_at_5_diff1": 0.8454163679900751, + "nauc_mrr_at_5_max": 0.11507249429826652, + "nauc_mrr_at_5_std": -0.6037227397638133, + "nauc_ndcg_at_1000_diff1": 0.8438515659117648, + "nauc_ndcg_at_1000_max": 0.11696388104993816, + "nauc_ndcg_at_1000_std": -0.6099717188632636, + "nauc_ndcg_at_100_diff1": 0.8436252666208128, + "nauc_ndcg_at_100_max": 0.11779193272629583, + "nauc_ndcg_at_100_std": -0.6107767732450027, + "nauc_ndcg_at_10_diff1": 0.8404954066703365, + "nauc_ndcg_at_10_max": 0.11779974489837082, + "nauc_ndcg_at_10_std": -0.6321669009592777, + "nauc_ndcg_at_1_diff1": 0.86117489531929, + "nauc_ndcg_at_1_max": 0.115654030601593, + "nauc_ndcg_at_1_std": -0.535110885081521, + "nauc_ndcg_at_20_diff1": 0.8424823772848972, + "nauc_ndcg_at_20_max": 0.11863360518928208, + "nauc_ndcg_at_20_std": -0.6193361896101012, + "nauc_ndcg_at_3_diff1": 0.8359193584331861, + "nauc_ndcg_at_3_max": 0.11101961347273388, + "nauc_ndcg_at_3_std": -0.6594436298501897, + "nauc_ndcg_at_5_diff1": 0.8378223501020723, + "nauc_ndcg_at_5_max": 0.11381166299845784, + "nauc_ndcg_at_5_std": -0.6504889907197641, + "nauc_precision_at_1000_diff1": 0.7606066445979025, + "nauc_precision_at_1000_max": 0.8880070662604956, + "nauc_precision_at_1000_std": 0.523340591830439, + "nauc_precision_at_100_diff1": 0.7204652343451334, + "nauc_precision_at_100_max": 0.6389605474602823, + "nauc_precision_at_100_std": -0.6943370560244633, + "nauc_precision_at_10_diff1": 0.7289952199107627, + "nauc_precision_at_10_max": 0.19311114535110577, + "nauc_precision_at_10_std": -1.2884033294430404, + "nauc_precision_at_1_diff1": 0.86117489531929, + "nauc_precision_at_1_max": 0.115654030601593, + "nauc_precision_at_1_std": -0.535110885081521, + "nauc_precision_at_20_diff1": 0.7267081355507392, + "nauc_precision_at_20_max": 0.3181000902775133, + "nauc_precision_at_20_std": -1.250241647763245, + "nauc_precision_at_3_diff1": 0.7825481403127574, + "nauc_precision_at_3_max": 0.09522751941864359, + "nauc_precision_at_3_std": -0.9778790351049261, + "nauc_precision_at_5_diff1": 0.7476998378870415, + "nauc_precision_at_5_max": 0.10813166077171799, + "nauc_precision_at_5_std": -1.2115471825739508, + "nauc_recall_at_1000_diff1": 0.7606066445979108, + "nauc_recall_at_1000_max": 0.8880070662604261, + "nauc_recall_at_1000_std": 0.5233405918305375, + "nauc_recall_at_100_diff1": 0.7204652343451712, + "nauc_recall_at_100_max": 0.6389605474602079, + "nauc_recall_at_100_std": -0.6943370560243692, + "nauc_recall_at_10_diff1": 0.7289952199107772, + "nauc_recall_at_10_max": 0.19311114535110474, + "nauc_recall_at_10_std": -1.2884033294430468, + "nauc_recall_at_1_diff1": 0.86117489531929, + "nauc_recall_at_1_max": 0.115654030601593, + "nauc_recall_at_1_std": -0.535110885081521, + "nauc_recall_at_20_diff1": 0.7267081355507441, + "nauc_recall_at_20_max": 0.3181000902775227, + "nauc_recall_at_20_std": -1.2502416477631773, + "nauc_recall_at_3_diff1": 0.7825481403127578, + "nauc_recall_at_3_max": 0.09522751941864487, + "nauc_recall_at_3_std": -0.9778790351049261, + "nauc_recall_at_5_diff1": 0.7476998378870401, + "nauc_recall_at_5_max": 0.10813166077172304, + "nauc_recall_at_5_std": -1.2115471825739255, + "ndcg_at_1": 0.8017, + "ndcg_at_10": 0.90672, + "ndcg_at_100": 0.90923, + "ndcg_at_1000": 0.90939, + "ndcg_at_20": 0.90833, + "ndcg_at_3": 0.89347, + "ndcg_at_5": 0.90321, + "precision_at_1": 0.8017, + "precision_at_10": 0.09875, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.04969, + "precision_at_3": 0.31786, + "precision_at_5": 0.19538, + "recall_at_1": 0.8017, + "recall_at_10": 0.98754, + "recall_at_100": 0.99853, + "recall_at_1000": 0.99971, + "recall_at_20": 0.99384, + "recall_at_3": 0.95359, + "recall_at_5": 0.97691 + } + ] + }, + "task_name": "CodeFeedbackST" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetCCRetrieval.json b/results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetCCRetrieval.json new file mode 100644 index 000000000..09db5802a --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetCCRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "6e1effa2c03723c5fde48ee912b5ee08d4f211e8", + "evaluation_time": 10742.119951248169, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "python", + "languages": [ + "python-Code" + ], + "main_score": 0.94433, + "map_at_1": 0.89194, + "map_at_10": 0.93028, + "map_at_100": 0.93082, + "map_at_1000": 0.93083, + "map_at_20": 0.93063, + "map_at_3": 0.92635, + "map_at_5": 0.92914, + "mrr_at_1": 0.8919426196541091, + "mrr_at_10": 0.9302709840248427, + "mrr_at_100": 0.930811600342414, + "mrr_at_1000": 0.9308217593910117, + "mrr_at_20": 0.9306190896089125, + "mrr_at_3": 0.9263529516914724, + "mrr_at_5": 0.9291147160030434, + "nauc_map_at_1000_diff1": 0.8883311452663212, + "nauc_map_at_1000_max": 0.6095017820522245, + "nauc_map_at_1000_std": -0.2789769485046986, + "nauc_map_at_100_diff1": 0.8883290610426658, + "nauc_map_at_100_max": 0.6095500551380402, + "nauc_map_at_100_std": -0.2789016844487898, + "nauc_map_at_10_diff1": 0.8885572581806649, + "nauc_map_at_10_max": 0.6115711675036097, + "nauc_map_at_10_std": -0.27863737251156817, + "nauc_map_at_1_diff1": 0.8875783103896557, + "nauc_map_at_1_max": 0.5605082536501996, + "nauc_map_at_1_std": -0.2676227127297406, + "nauc_map_at_20_diff1": 0.8883684101691744, + "nauc_map_at_20_max": 0.6100577046621442, + "nauc_map_at_20_std": -0.2793102109901855, + "nauc_map_at_3_diff1": 0.888117321224026, + "nauc_map_at_3_max": 0.6193756889954432, + "nauc_map_at_3_std": -0.28688755234894575, + "nauc_map_at_5_diff1": 0.8880829639130586, + "nauc_map_at_5_max": 0.6145327028871431, + "nauc_map_at_5_std": -0.2834998014072406, + "nauc_mrr_at_1000_diff1": 0.8884020090540667, + "nauc_mrr_at_1000_max": 0.6088994511891193, + "nauc_mrr_at_1000_std": -0.2798205495040352, + "nauc_mrr_at_100_diff1": 0.888399914575814, + "nauc_mrr_at_100_max": 0.6089478080634029, + "nauc_mrr_at_100_std": -0.27974516894246, + "nauc_mrr_at_10_diff1": 0.8886278756786536, + "nauc_mrr_at_10_max": 0.6109737437911841, + "nauc_mrr_at_10_std": -0.2794782127554896, + "nauc_mrr_at_1_diff1": 0.8875783103896557, + "nauc_mrr_at_1_max": 0.5600537716973826, + "nauc_mrr_at_1_std": -0.2685570824246162, + "nauc_mrr_at_20_diff1": 0.8884394060587956, + "nauc_mrr_at_20_max": 0.6094574319151849, + "nauc_mrr_at_20_std": -0.28015522604336573, + "nauc_mrr_at_3_diff1": 0.888197212493924, + "nauc_mrr_at_3_max": 0.6186494040675816, + "nauc_mrr_at_3_std": -0.28785412568295915, + "nauc_mrr_at_5_diff1": 0.8882086414470385, + "nauc_mrr_at_5_max": 0.6139568301316898, + "nauc_mrr_at_5_std": -0.284704407411122, + "nauc_ndcg_at_1000_diff1": 0.8887772673122608, + "nauc_ndcg_at_1000_max": 0.6172947423485932, + "nauc_ndcg_at_1000_std": -0.2773716470116156, + "nauc_ndcg_at_100_diff1": 0.8888494233648349, + "nauc_ndcg_at_100_max": 0.6189646036981659, + "nauc_ndcg_at_100_std": -0.27485345609558276, + "nauc_ndcg_at_10_diff1": 0.8900232938008411, + "nauc_ndcg_at_10_max": 0.6294119991860201, + "nauc_ndcg_at_10_std": -0.2746019414887392, + "nauc_ndcg_at_1_diff1": 0.8875783103896557, + "nauc_ndcg_at_1_max": 0.5605082536501996, + "nauc_ndcg_at_1_std": -0.2676227127297406, + "nauc_ndcg_at_20_diff1": 0.8891820321119602, + "nauc_ndcg_at_20_max": 0.6229658464928647, + "nauc_ndcg_at_20_std": -0.2776624493043811, + "nauc_ndcg_at_3_diff1": 0.888645684782722, + "nauc_ndcg_at_3_max": 0.6456115657789144, + "nauc_ndcg_at_3_std": -0.29624561516864517, + "nauc_ndcg_at_5_diff1": 0.8886373245096877, + "nauc_ndcg_at_5_max": 0.6372960904786197, + "nauc_ndcg_at_5_std": -0.289714090724707, + "nauc_precision_at_1000_diff1": 0.9346395492502747, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.7916615094256558, + "nauc_precision_at_100_diff1": 0.9118620004808257, + "nauc_precision_at_100_max": 0.9322441208688358, + "nauc_precision_at_100_std": 0.2454102760572213, + "nauc_precision_at_10_diff1": 0.9139898569182663, + "nauc_precision_at_10_max": 0.8636227958399685, + "nauc_precision_at_10_std": -0.18102422408193578, + "nauc_precision_at_1_diff1": 0.8875783103896557, + "nauc_precision_at_1_max": 0.5605082536501996, + "nauc_precision_at_1_std": -0.2676227127297406, + "nauc_precision_at_20_diff1": 0.9067031369822484, + "nauc_precision_at_20_max": 0.8374877222150064, + "nauc_precision_at_20_std": -0.20408972788884217, + "nauc_precision_at_3_diff1": 0.8920609963322217, + "nauc_precision_at_3_max": 0.8051313561745297, + "nauc_precision_at_3_std": -0.3536879832053998, + "nauc_precision_at_5_diff1": 0.8940146615001582, + "nauc_precision_at_5_max": 0.8394859838264774, + "nauc_precision_at_5_std": -0.34371668675188605, + "nauc_recall_at_1000_diff1": 0.9346395492500885, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.791661509425585, + "nauc_recall_at_100_diff1": 0.9118620004808853, + "nauc_recall_at_100_max": 0.9322441208689622, + "nauc_recall_at_100_std": 0.24541027605730553, + "nauc_recall_at_10_diff1": 0.9139898569182566, + "nauc_recall_at_10_max": 0.8636227958399565, + "nauc_recall_at_10_std": -0.18102422408192856, + "nauc_recall_at_1_diff1": 0.8875783103896557, + "nauc_recall_at_1_max": 0.5605082536501996, + "nauc_recall_at_1_std": -0.2676227127297406, + "nauc_recall_at_20_diff1": 0.9067031369822419, + "nauc_recall_at_20_max": 0.837487722215012, + "nauc_recall_at_20_std": -0.20408972788880642, + "nauc_recall_at_3_diff1": 0.8920609963322205, + "nauc_recall_at_3_max": 0.8051313561745231, + "nauc_recall_at_3_std": -0.35368798320539585, + "nauc_recall_at_5_diff1": 0.8940146615001574, + "nauc_recall_at_5_max": 0.8394859838264771, + "nauc_recall_at_5_std": -0.3437166867518835, + "ndcg_at_1": 0.89194, + "ndcg_at_10": 0.94433, + "ndcg_at_100": 0.94679, + "ndcg_at_1000": 0.94707, + "ndcg_at_20": 0.94556, + "ndcg_at_3": 0.93661, + "ndcg_at_5": 0.94161, + "precision_at_1": 0.89194, + "precision_at_10": 0.09864, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04956, + "precision_at_3": 0.322, + "precision_at_5": 0.19562, + "recall_at_1": 0.89194, + "recall_at_10": 0.98639, + "recall_at_100": 0.99745, + "recall_at_1000": 0.99973, + "recall_at_20": 0.99122, + "recall_at_3": 0.96601, + "recall_at_5": 0.97808 + }, + { + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ], + "main_score": 0.91532, + "map_at_1": 0.84047, + "map_at_10": 0.89504, + "map_at_100": 0.89581, + "map_at_1000": 0.89584, + "map_at_20": 0.89548, + "map_at_3": 0.88914, + "map_at_5": 0.89271, + "mrr_at_1": 0.8398663020358553, + "mrr_at_10": 0.8946699271220692, + "mrr_at_100": 0.8954429619092761, + "mrr_at_1000": 0.8954715966900696, + "mrr_at_20": 0.8951131350599995, + "mrr_at_3": 0.8887876025524161, + "mrr_at_5": 0.8923427529626254, + "nauc_map_at_1000_diff1": 0.8133895244348351, + "nauc_map_at_1000_max": 0.5100582756640207, + "nauc_map_at_1000_std": -0.22252578589937455, + "nauc_map_at_100_diff1": 0.8133433499029653, + "nauc_map_at_100_max": 0.5101323653352282, + "nauc_map_at_100_std": -0.2224639503900029, + "nauc_map_at_10_diff1": 0.8130647802788602, + "nauc_map_at_10_max": 0.5121114523806548, + "nauc_map_at_10_std": -0.22303196568890327, + "nauc_map_at_1_diff1": 0.8257372620311462, + "nauc_map_at_1_max": 0.4454905481211588, + "nauc_map_at_1_std": -0.21962071475865755, + "nauc_map_at_20_diff1": 0.8131928022528241, + "nauc_map_at_20_max": 0.51132386041714, + "nauc_map_at_20_std": -0.2220703220818948, + "nauc_map_at_3_diff1": 0.8110849311491785, + "nauc_map_at_3_max": 0.518208019735269, + "nauc_map_at_3_std": -0.2296789103735518, + "nauc_map_at_5_diff1": 0.8114944383400071, + "nauc_map_at_5_max": 0.5154484151827443, + "nauc_map_at_5_std": -0.2239835418757183, + "nauc_mrr_at_1000_diff1": 0.8142887177105008, + "nauc_mrr_at_1000_max": 0.5078456409105162, + "nauc_mrr_at_1000_std": -0.22810803121918408, + "nauc_mrr_at_100_diff1": 0.8142424440922578, + "nauc_mrr_at_100_max": 0.5079201071047587, + "nauc_mrr_at_100_std": -0.22804483531306144, + "nauc_mrr_at_10_diff1": 0.8139579265182798, + "nauc_mrr_at_10_max": 0.5099096194207801, + "nauc_mrr_at_10_std": -0.22856845472961557, + "nauc_mrr_at_1_diff1": 0.8268635369279959, + "nauc_mrr_at_1_max": 0.442393421717323, + "nauc_mrr_at_1_std": -0.2277517261029381, + "nauc_mrr_at_20_diff1": 0.8140894511054444, + "nauc_mrr_at_20_max": 0.5091149353347425, + "nauc_mrr_at_20_std": -0.2276342604181709, + "nauc_mrr_at_3_diff1": 0.811947777094554, + "nauc_mrr_at_3_max": 0.5160584449653166, + "nauc_mrr_at_3_std": -0.2348440779958821, + "nauc_mrr_at_5_diff1": 0.8123725416306373, + "nauc_mrr_at_5_max": 0.5132853667650834, + "nauc_mrr_at_5_std": -0.22939278713896133, + "nauc_ndcg_at_1000_diff1": 0.8112746150854298, + "nauc_ndcg_at_1000_max": 0.521786963598477, + "nauc_ndcg_at_1000_std": -0.21667355350607292, + "nauc_ndcg_at_100_diff1": 0.8100850024967238, + "nauc_ndcg_at_100_max": 0.5237690080977986, + "nauc_ndcg_at_100_std": -0.21437187821641884, + "nauc_ndcg_at_10_diff1": 0.8087079398487964, + "nauc_ndcg_at_10_max": 0.5371901285846592, + "nauc_ndcg_at_10_std": -0.2157272074203413, + "nauc_ndcg_at_1_diff1": 0.8257372620311462, + "nauc_ndcg_at_1_max": 0.4454905481211588, + "nauc_ndcg_at_1_std": -0.21962071475865755, + "nauc_ndcg_at_20_diff1": 0.8089584525085091, + "nauc_ndcg_at_20_max": 0.534014158746573, + "nauc_ndcg_at_20_std": -0.21129352577941285, + "nauc_ndcg_at_3_diff1": 0.8046103065917166, + "nauc_ndcg_at_3_max": 0.5491263156860439, + "nauc_ndcg_at_3_std": -0.230656484747823, + "nauc_ndcg_at_5_diff1": 0.8050385469743832, + "nauc_ndcg_at_5_max": 0.5453609881176663, + "nauc_ndcg_at_5_std": -0.2182614533682788, + "nauc_precision_at_1000_diff1": 0.814825203906492, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.694979677737838, + "nauc_precision_at_100_max": 0.7981453242811645, + "nauc_precision_at_100_std": 0.22735882735419854, + "nauc_precision_at_10_diff1": 0.7602578489916326, + "nauc_precision_at_10_max": 0.8220220582823439, + "nauc_precision_at_10_std": -0.11133888291415302, + "nauc_precision_at_1_diff1": 0.8257372620311462, + "nauc_precision_at_1_max": 0.4454905481211588, + "nauc_precision_at_1_std": -0.21962071475865755, + "nauc_precision_at_20_diff1": 0.7457535506743962, + "nauc_precision_at_20_max": 0.8622474553094298, + "nauc_precision_at_20_std": 0.0037124480616714155, + "nauc_precision_at_3_diff1": 0.7690725827318549, + "nauc_precision_at_3_max": 0.7180786851596808, + "nauc_precision_at_3_std": -0.2338779609088237, + "nauc_precision_at_5_diff1": 0.7576703985132757, + "nauc_precision_at_5_max": 0.7630757619721807, + "nauc_precision_at_5_std": -0.16772096432698, + "nauc_recall_at_1000_diff1": 0.8148252039064056, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.6949796777378361, + "nauc_recall_at_100_max": 0.7981453242811749, + "nauc_recall_at_100_std": 0.22735882735423568, + "nauc_recall_at_10_diff1": 0.7602578489916281, + "nauc_recall_at_10_max": 0.8220220582823432, + "nauc_recall_at_10_std": -0.11133888291414291, + "nauc_recall_at_1_diff1": 0.8257372620311462, + "nauc_recall_at_1_max": 0.4454905481211588, + "nauc_recall_at_1_std": -0.21962071475865755, + "nauc_recall_at_20_diff1": 0.7457535506744009, + "nauc_recall_at_20_max": 0.8622474553094256, + "nauc_recall_at_20_std": 0.0037124480616788406, + "nauc_recall_at_3_diff1": 0.7690725827318567, + "nauc_recall_at_3_max": 0.7180786851596824, + "nauc_recall_at_3_std": -0.2338779609088207, + "nauc_recall_at_5_diff1": 0.7576703985132793, + "nauc_recall_at_5_max": 0.7630757619721836, + "nauc_recall_at_5_std": -0.16772096432697478, + "ndcg_at_1": 0.84047, + "ndcg_at_10": 0.91532, + "ndcg_at_100": 0.91914, + "ndcg_at_1000": 0.91979, + "ndcg_at_20": 0.91688, + "ndcg_at_3": 0.90353, + "ndcg_at_5": 0.90993, + "precision_at_1": 0.84047, + "precision_at_10": 0.09763, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04912, + "precision_at_3": 0.3149, + "precision_at_5": 0.19204, + "recall_at_1": 0.84047, + "recall_at_10": 0.9763, + "recall_at_100": 0.99423, + "recall_at_1000": 0.99909, + "recall_at_20": 0.98238, + "recall_at_3": 0.9447, + "recall_at_5": 0.96019 + }, + { + "hf_subset": "go", + "languages": [ + "go-Code" + ], + "main_score": 0.9176, + "map_at_1": 0.8429, + "map_at_10": 0.89689, + "map_at_100": 0.89773, + "map_at_1000": 0.89774, + "map_at_20": 0.89747, + "map_at_3": 0.89026, + "map_at_5": 0.89484, + "mrr_at_1": 0.8427727160797833, + "mrr_at_10": 0.8967515722532932, + "mrr_at_100": 0.8975855757010807, + "mrr_at_1000": 0.897601837108608, + "mrr_at_20": 0.8973302201153147, + "mrr_at_3": 0.8901337929902301, + "mrr_at_5": 0.8947262579003495, + "nauc_map_at_1000_diff1": 0.8517428924238607, + "nauc_map_at_1000_max": 0.48685961107484227, + "nauc_map_at_1000_std": -0.307091565112973, + "nauc_map_at_100_diff1": 0.8517188089036237, + "nauc_map_at_100_max": 0.48691353942226695, + "nauc_map_at_100_std": -0.30704177527491316, + "nauc_map_at_10_diff1": 0.8516218860131448, + "nauc_map_at_10_max": 0.488187190573415, + "nauc_map_at_10_std": -0.3076900084805538, + "nauc_map_at_1_diff1": 0.8600618812234513, + "nauc_map_at_1_max": 0.45784592926163903, + "nauc_map_at_1_std": -0.29077490123642913, + "nauc_map_at_20_diff1": 0.8517186125725058, + "nauc_map_at_20_max": 0.48740585411792897, + "nauc_map_at_20_std": -0.3071428055810858, + "nauc_map_at_3_diff1": 0.8504460665149781, + "nauc_map_at_3_max": 0.49069208587865115, + "nauc_map_at_3_std": -0.31417225141010363, + "nauc_map_at_5_diff1": 0.8511442620241467, + "nauc_map_at_5_max": 0.4883495405895641, + "nauc_map_at_5_std": -0.3128790738450253, + "nauc_mrr_at_1000_diff1": 0.8521632400557363, + "nauc_mrr_at_1000_max": 0.4884146877454467, + "nauc_mrr_at_1000_std": -0.307223552994054, + "nauc_mrr_at_100_diff1": 0.852139127988098, + "nauc_mrr_at_100_max": 0.48846837041959856, + "nauc_mrr_at_100_std": -0.3071739085936448, + "nauc_mrr_at_10_diff1": 0.8520230661077468, + "nauc_mrr_at_10_max": 0.48957929042209614, + "nauc_mrr_at_10_std": -0.3078597710748554, + "nauc_mrr_at_1_diff1": 0.860269378207981, + "nauc_mrr_at_1_max": 0.4594255127280741, + "nauc_mrr_at_1_std": -0.29011204207077934, + "nauc_mrr_at_20_diff1": 0.8521586630373561, + "nauc_mrr_at_20_max": 0.48881837320576693, + "nauc_mrr_at_20_std": -0.3073183934707418, + "nauc_mrr_at_3_diff1": 0.8508479102826572, + "nauc_mrr_at_3_max": 0.49230084820615577, + "nauc_mrr_at_3_std": -0.314650469520573, + "nauc_mrr_at_5_diff1": 0.851584637044273, + "nauc_mrr_at_5_max": 0.4900177049406493, + "nauc_mrr_at_5_std": -0.31307066011998225, + "nauc_ndcg_at_1000_diff1": 0.8509478158429044, + "nauc_ndcg_at_1000_max": 0.49255609353429747, + "nauc_ndcg_at_1000_std": -0.3070761801312305, + "nauc_ndcg_at_100_diff1": 0.8503251451319845, + "nauc_ndcg_at_100_max": 0.49421043942452814, + "nauc_ndcg_at_100_std": -0.30551562494744167, + "nauc_ndcg_at_10_diff1": 0.8500106483349344, + "nauc_ndcg_at_10_max": 0.5011523191064857, + "nauc_ndcg_at_10_std": -0.3087312388259668, + "nauc_ndcg_at_1_diff1": 0.8600618812234513, + "nauc_ndcg_at_1_max": 0.45784592926163903, + "nauc_ndcg_at_1_std": -0.29077490123642913, + "nauc_ndcg_at_20_diff1": 0.8505361380063081, + "nauc_ndcg_at_20_max": 0.4977246579429325, + "nauc_ndcg_at_20_std": -0.3067259568687167, + "nauc_ndcg_at_3_diff1": 0.8477305397981563, + "nauc_ndcg_at_3_max": 0.5052441698065767, + "nauc_ndcg_at_3_std": -0.3246848973997472, + "nauc_ndcg_at_5_diff1": 0.8488420002224352, + "nauc_ndcg_at_5_max": 0.5011133138392584, + "nauc_ndcg_at_5_std": -0.323270235813713, + "nauc_precision_at_1000_diff1": 0.918303894683355, + "nauc_precision_at_1000_max": 0.8846880842429544, + "nauc_precision_at_1000_std": 0.21769969322436478, + "nauc_precision_at_100_diff1": 0.7647557733881312, + "nauc_precision_at_100_max": 0.8428732698302337, + "nauc_precision_at_100_std": 0.05283523585432388, + "nauc_precision_at_10_diff1": 0.8318181354333526, + "nauc_precision_at_10_max": 0.6800168082360677, + "nauc_precision_at_10_std": -0.29843581018958476, + "nauc_precision_at_1_diff1": 0.8600618812234513, + "nauc_precision_at_1_max": 0.45784592926163903, + "nauc_precision_at_1_std": -0.29077490123642913, + "nauc_precision_at_20_diff1": 0.8347691149529181, + "nauc_precision_at_20_max": 0.7101630819186266, + "nauc_precision_at_20_std": -0.24608779769104114, + "nauc_precision_at_3_diff1": 0.8338151337361877, + "nauc_precision_at_3_max": 0.5848742413861129, + "nauc_precision_at_3_std": -0.3823005616508197, + "nauc_precision_at_5_diff1": 0.8316792264231816, + "nauc_precision_at_5_max": 0.603244549312709, + "nauc_precision_at_5_std": -0.40872997708952075, + "nauc_recall_at_1000_diff1": 0.9183038946832791, + "nauc_recall_at_1000_max": 0.8846880842431686, + "nauc_recall_at_1000_std": 0.21769969322480404, + "nauc_recall_at_100_diff1": 0.7647557733881604, + "nauc_recall_at_100_max": 0.8428732698303842, + "nauc_recall_at_100_std": 0.05283523585434394, + "nauc_recall_at_10_diff1": 0.8318181354333507, + "nauc_recall_at_10_max": 0.680016808236062, + "nauc_recall_at_10_std": -0.2984358101895757, + "nauc_recall_at_1_diff1": 0.8600618812234513, + "nauc_recall_at_1_max": 0.45784592926163903, + "nauc_recall_at_1_std": -0.29077490123642913, + "nauc_recall_at_20_diff1": 0.8347691149529236, + "nauc_recall_at_20_max": 0.7101630819186342, + "nauc_recall_at_20_std": -0.24608779769102324, + "nauc_recall_at_3_diff1": 0.8338151337361859, + "nauc_recall_at_3_max": 0.5848742413861102, + "nauc_recall_at_3_std": -0.3823005616508199, + "nauc_recall_at_5_diff1": 0.8316792264231778, + "nauc_recall_at_5_max": 0.6032445493127084, + "nauc_recall_at_5_std": -0.408729977089511, + "ndcg_at_1": 0.8429, + "ndcg_at_10": 0.9176, + "ndcg_at_100": 0.92128, + "ndcg_at_1000": 0.92166, + "ndcg_at_20": 0.91965, + "ndcg_at_3": 0.90441, + "ndcg_at_5": 0.91264, + "precision_at_1": 0.8429, + "precision_at_10": 0.09802, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04941, + "precision_at_3": 0.31499, + "precision_at_5": 0.19298, + "recall_at_1": 0.8429, + "recall_at_10": 0.98018, + "recall_at_100": 0.99655, + "recall_at_1000": 0.99938, + "recall_at_20": 0.98818, + "recall_at_3": 0.94496, + "recall_at_5": 0.96491 + }, + { + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ], + "main_score": 0.89257, + "map_at_1": 0.80967, + "map_at_10": 0.86978, + "map_at_100": 0.87113, + "map_at_1000": 0.87117, + "map_at_20": 0.87054, + "map_at_3": 0.86294, + "map_at_5": 0.86738, + "mrr_at_1": 0.809674861221253, + "mrr_at_10": 0.8695775587528166, + "mrr_at_100": 0.8709311803235962, + "mrr_at_1000": 0.8709630839341809, + "mrr_at_20": 0.8703383454810835, + "mrr_at_3": 0.8625429553264607, + "mrr_at_5": 0.8671821305841928, + "nauc_map_at_1000_diff1": 0.8217808945938484, + "nauc_map_at_1000_max": 0.6075779539267633, + "nauc_map_at_1000_std": -0.23755896580375824, + "nauc_map_at_100_diff1": 0.821794560801495, + "nauc_map_at_100_max": 0.6076366486911127, + "nauc_map_at_100_std": -0.23741319565592223, + "nauc_map_at_10_diff1": 0.8218782847323822, + "nauc_map_at_10_max": 0.6091576878869062, + "nauc_map_at_10_std": -0.2386367782034753, + "nauc_map_at_1_diff1": 0.8372039170136367, + "nauc_map_at_1_max": 0.5725380612858921, + "nauc_map_at_1_std": -0.2593579147591399, + "nauc_map_at_20_diff1": 0.8217796635750986, + "nauc_map_at_20_max": 0.6083301592914026, + "nauc_map_at_20_std": -0.23683955007198032, + "nauc_map_at_3_diff1": 0.8183929922286214, + "nauc_map_at_3_max": 0.6130090311509839, + "nauc_map_at_3_std": -0.2367250024837735, + "nauc_map_at_5_diff1": 0.8188267528957118, + "nauc_map_at_5_max": 0.6106118360132867, + "nauc_map_at_5_std": -0.2356420203795342, + "nauc_mrr_at_1000_diff1": 0.8224831373360206, + "nauc_mrr_at_1000_max": 0.6054745821681178, + "nauc_mrr_at_1000_std": -0.2462799535642754, + "nauc_mrr_at_100_diff1": 0.8224966010195053, + "nauc_mrr_at_100_max": 0.6055337339986757, + "nauc_mrr_at_100_std": -0.24613214185234447, + "nauc_mrr_at_10_diff1": 0.8225711255868444, + "nauc_mrr_at_10_max": 0.6070674951461077, + "nauc_mrr_at_10_std": -0.2472313428563711, + "nauc_mrr_at_1_diff1": 0.8372039170136367, + "nauc_mrr_at_1_max": 0.5698766026206556, + "nauc_mrr_at_1_std": -0.26606885807082387, + "nauc_mrr_at_20_diff1": 0.8224783610762223, + "nauc_mrr_at_20_max": 0.6062362881960727, + "nauc_mrr_at_20_std": -0.24551756212224143, + "nauc_mrr_at_3_diff1": 0.8193274024955834, + "nauc_mrr_at_3_max": 0.6109367721118326, + "nauc_mrr_at_3_std": -0.24713070452305638, + "nauc_mrr_at_5_diff1": 0.8195108632175991, + "nauc_mrr_at_5_max": 0.6085590424882734, + "nauc_mrr_at_5_std": -0.24407854661602732, + "nauc_ndcg_at_1000_diff1": 0.8213638313064818, + "nauc_ndcg_at_1000_max": 0.6136229533164094, + "nauc_ndcg_at_1000_std": -0.2326361978261703, + "nauc_ndcg_at_100_diff1": 0.8217708460929015, + "nauc_ndcg_at_100_max": 0.6155039283837963, + "nauc_ndcg_at_100_std": -0.2282475534090763, + "nauc_ndcg_at_10_diff1": 0.8219581572094847, + "nauc_ndcg_at_10_max": 0.622507541968532, + "nauc_ndcg_at_10_std": -0.23522643567705576, + "nauc_ndcg_at_1_diff1": 0.8372039170136367, + "nauc_ndcg_at_1_max": 0.5725380612858921, + "nauc_ndcg_at_1_std": -0.2593579147591399, + "nauc_ndcg_at_20_diff1": 0.8214559730614958, + "nauc_ndcg_at_20_max": 0.6193079238970315, + "nauc_ndcg_at_20_std": -0.22719105026052122, + "nauc_ndcg_at_3_diff1": 0.8136414038291007, + "nauc_ndcg_at_3_max": 0.6297389940233761, + "nauc_ndcg_at_3_std": -0.229586797851577, + "nauc_ndcg_at_5_diff1": 0.8141077106697202, + "nauc_ndcg_at_5_max": 0.6260599959664633, + "nauc_ndcg_at_5_std": -0.22684305831996263, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.5542527628232039, + "nauc_precision_at_100_diff1": 0.877466635471206, + "nauc_precision_at_100_max": 0.8308111330779416, + "nauc_precision_at_100_std": 0.2539534841322347, + "nauc_precision_at_10_diff1": 0.8335598753622322, + "nauc_precision_at_10_max": 0.7370169446342424, + "nauc_precision_at_10_std": -0.2161554980391565, + "nauc_precision_at_1_diff1": 0.8372039170136367, + "nauc_precision_at_1_max": 0.5725380612858921, + "nauc_precision_at_1_std": -0.2593579147591399, + "nauc_precision_at_20_diff1": 0.8303143013817835, + "nauc_precision_at_20_max": 0.7349113429650389, + "nauc_precision_at_20_std": -0.09826961687536868, + "nauc_precision_at_3_diff1": 0.7916945277065827, + "nauc_precision_at_3_max": 0.7131564419236822, + "nauc_precision_at_3_std": -0.19527455537664723, + "nauc_precision_at_5_diff1": 0.785956097040408, + "nauc_precision_at_5_max": 0.7248547478245245, + "nauc_precision_at_5_std": -0.1707438252522886, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.5542527628229722, + "nauc_recall_at_100_diff1": 0.8774666354712278, + "nauc_recall_at_100_max": 0.8308111330779382, + "nauc_recall_at_100_std": 0.25395348413221663, + "nauc_recall_at_10_diff1": 0.8335598753622315, + "nauc_recall_at_10_max": 0.7370169446342429, + "nauc_recall_at_10_std": -0.21615549803915743, + "nauc_recall_at_1_diff1": 0.8372039170136367, + "nauc_recall_at_1_max": 0.5725380612858921, + "nauc_recall_at_1_std": -0.2593579147591399, + "nauc_recall_at_20_diff1": 0.8303143013817795, + "nauc_recall_at_20_max": 0.7349113429650307, + "nauc_recall_at_20_std": -0.09826961687536835, + "nauc_recall_at_3_diff1": 0.7916945277065781, + "nauc_recall_at_3_max": 0.7131564419236814, + "nauc_recall_at_3_std": -0.1952745553766413, + "nauc_recall_at_5_diff1": 0.7859560970404043, + "nauc_recall_at_5_max": 0.724854747824523, + "nauc_recall_at_5_std": -0.17074382525229173, + "ndcg_at_1": 0.80967, + "ndcg_at_10": 0.89257, + "ndcg_at_100": 0.89918, + "ndcg_at_1000": 0.90007, + "ndcg_at_20": 0.89524, + "ndcg_at_3": 0.87909, + "ndcg_at_5": 0.88701, + "precision_at_1": 0.80967, + "precision_at_10": 0.09611, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "precision_at_20": 0.04857, + "precision_at_3": 0.30849, + "precision_at_5": 0.1889, + "recall_at_1": 0.80967, + "recall_at_10": 0.96114, + "recall_at_100": 0.99207, + "recall_at_1000": 0.99921, + "recall_at_20": 0.97145, + "recall_at_3": 0.92546, + "recall_at_5": 0.94449 + }, + { + "hf_subset": "java", + "languages": [ + "java-Code" + ], + "main_score": 0.90187, + "map_at_1": 0.83049, + "map_at_10": 0.882, + "map_at_100": 0.88294, + "map_at_1000": 0.88297, + "map_at_20": 0.88268, + "map_at_3": 0.87581, + "map_at_5": 0.87997, + "mrr_at_1": 0.8303057964399817, + "mrr_at_10": 0.8819186353987806, + "mrr_at_100": 0.882852857857312, + "mrr_at_1000": 0.8828810205243453, + "mrr_at_20": 0.8825951626896247, + "mrr_at_3": 0.875749277346721, + "mrr_at_5": 0.879875247223489, + "nauc_map_at_1000_diff1": 0.8410730394433992, + "nauc_map_at_1000_max": 0.6152233212127969, + "nauc_map_at_1000_std": -0.1344169980569628, + "nauc_map_at_100_diff1": 0.8410552079587292, + "nauc_map_at_100_max": 0.6152588029468992, + "nauc_map_at_100_std": -0.13438768702376994, + "nauc_map_at_10_diff1": 0.8406622800697098, + "nauc_map_at_10_max": 0.6162852938331853, + "nauc_map_at_10_std": -0.13540123911279858, + "nauc_map_at_1_diff1": 0.8458928110942394, + "nauc_map_at_1_max": 0.5615285789003811, + "nauc_map_at_1_std": -0.17221403594342735, + "nauc_map_at_20_diff1": 0.8409390965699994, + "nauc_map_at_20_max": 0.6156998064856088, + "nauc_map_at_20_std": -0.13445035300847308, + "nauc_map_at_3_diff1": 0.8388744798135446, + "nauc_map_at_3_max": 0.6188368527363093, + "nauc_map_at_3_std": -0.14711176741863827, + "nauc_map_at_5_diff1": 0.8398674004934382, + "nauc_map_at_5_max": 0.6174583336524622, + "nauc_map_at_5_std": -0.13965194914228382, + "nauc_mrr_at_1000_diff1": 0.8407413270132194, + "nauc_mrr_at_1000_max": 0.6170365043563886, + "nauc_mrr_at_1000_std": -0.13350251134092783, + "nauc_mrr_at_100_diff1": 0.8407235854433451, + "nauc_mrr_at_100_max": 0.61707169972354, + "nauc_mrr_at_100_std": -0.13347338815823864, + "nauc_mrr_at_10_diff1": 0.8403062092695845, + "nauc_mrr_at_10_max": 0.6180570291957679, + "nauc_mrr_at_10_std": -0.13438890580569843, + "nauc_mrr_at_1_diff1": 0.8461937189362341, + "nauc_mrr_at_1_max": 0.5641943088748145, + "nauc_mrr_at_1_std": -0.17177446751086198, + "nauc_mrr_at_20_diff1": 0.8406068844396993, + "nauc_mrr_at_20_max": 0.6175092687172148, + "nauc_mrr_at_20_std": -0.1335357387841223, + "nauc_mrr_at_3_diff1": 0.8385345674418377, + "nauc_mrr_at_3_max": 0.6204381203396053, + "nauc_mrr_at_3_std": -0.146395621624969, + "nauc_mrr_at_5_diff1": 0.8395764939870174, + "nauc_mrr_at_5_max": 0.6192516669938368, + "nauc_mrr_at_5_std": -0.13886532596685805, + "nauc_ndcg_at_1000_diff1": 0.8442408974340632, + "nauc_ndcg_at_1000_max": 0.6332980075889167, + "nauc_ndcg_at_1000_std": -0.09859850962580813, + "nauc_ndcg_at_100_diff1": 0.8437653593475181, + "nauc_ndcg_at_100_max": 0.6339891417852467, + "nauc_ndcg_at_100_std": -0.09822808442396663, + "nauc_ndcg_at_10_diff1": 0.8414419181720488, + "nauc_ndcg_at_10_max": 0.6395060171395813, + "nauc_ndcg_at_10_std": -0.10490521537540003, + "nauc_ndcg_at_1_diff1": 0.8458928110942394, + "nauc_ndcg_at_1_max": 0.5615285789003811, + "nauc_ndcg_at_1_std": -0.17221403594342735, + "nauc_ndcg_at_20_diff1": 0.8426722898082026, + "nauc_ndcg_at_20_max": 0.6374976644807174, + "nauc_ndcg_at_20_std": -0.10004576105283305, + "nauc_ndcg_at_3_diff1": 0.8372365997941275, + "nauc_ndcg_at_3_max": 0.6431950978942049, + "nauc_ndcg_at_3_std": -0.1341750319915828, + "nauc_ndcg_at_5_diff1": 0.8391163247137122, + "nauc_ndcg_at_5_max": 0.6419792463538528, + "nauc_ndcg_at_5_std": -0.1176492800063572, + "nauc_precision_at_1000_diff1": 0.9713217959743038, + "nauc_precision_at_1000_max": 0.9846911971679215, + "nauc_precision_at_1000_std": 0.9023479950525061, + "nauc_precision_at_100_diff1": 0.9163251470011756, + "nauc_precision_at_100_max": 0.9017514956130656, + "nauc_precision_at_100_std": 0.610806660186732, + "nauc_precision_at_10_diff1": 0.8528726061856946, + "nauc_precision_at_10_max": 0.8323195279280301, + "nauc_precision_at_10_std": 0.17658421447762465, + "nauc_precision_at_1_diff1": 0.8458928110942394, + "nauc_precision_at_1_max": 0.5615285789003811, + "nauc_precision_at_1_std": -0.17221403594342735, + "nauc_precision_at_20_diff1": 0.8721693331269776, + "nauc_precision_at_20_max": 0.8697422806229869, + "nauc_precision_at_20_std": 0.32947726980858016, + "nauc_precision_at_3_diff1": 0.8300721034174204, + "nauc_precision_at_3_max": 0.7603478845540039, + "nauc_precision_at_3_std": -0.07075307314296672, + "nauc_precision_at_5_diff1": 0.8359241003022705, + "nauc_precision_at_5_max": 0.7935871830248997, + "nauc_precision_at_5_std": 0.02590360587580312, + "nauc_recall_at_1000_diff1": 0.9713217959743086, + "nauc_recall_at_1000_max": 0.9846911971679086, + "nauc_recall_at_1000_std": 0.9023479950524963, + "nauc_recall_at_100_diff1": 0.916325147001174, + "nauc_recall_at_100_max": 0.9017514956130668, + "nauc_recall_at_100_std": 0.6108066601867284, + "nauc_recall_at_10_diff1": 0.8528726061856851, + "nauc_recall_at_10_max": 0.8323195279280241, + "nauc_recall_at_10_std": 0.1765842144776203, + "nauc_recall_at_1_diff1": 0.8458928110942394, + "nauc_recall_at_1_max": 0.5615285789003811, + "nauc_recall_at_1_std": -0.17221403594342735, + "nauc_recall_at_20_diff1": 0.8721693331269785, + "nauc_recall_at_20_max": 0.8697422806229821, + "nauc_recall_at_20_std": 0.3294772698085802, + "nauc_recall_at_3_diff1": 0.8300721034174199, + "nauc_recall_at_3_max": 0.7603478845540012, + "nauc_recall_at_3_std": -0.07075307314297068, + "nauc_recall_at_5_diff1": 0.8359241003022677, + "nauc_recall_at_5_max": 0.793587183024897, + "nauc_recall_at_5_std": 0.025903605875805535, + "ndcg_at_1": 0.83049, + "ndcg_at_10": 0.90187, + "ndcg_at_100": 0.90614, + "ndcg_at_1000": 0.90686, + "ndcg_at_20": 0.90429, + "ndcg_at_3": 0.88957, + "ndcg_at_5": 0.897, + "precision_at_1": 0.83049, + "precision_at_10": 0.09619, + "precision_at_100": 0.00981, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04857, + "precision_at_3": 0.30969, + "precision_at_5": 0.18939, + "recall_at_1": 0.83049, + "recall_at_10": 0.96194, + "recall_at_100": 0.98129, + "recall_at_1000": 0.98695, + "recall_at_20": 0.97134, + "recall_at_3": 0.92907, + "recall_at_5": 0.94696 + }, + { + "hf_subset": "php", + "languages": [ + "php-Code" + ], + "main_score": 0.83388, + "map_at_1": 0.73255, + "map_at_10": 0.80387, + "map_at_100": 0.80641, + "map_at_1000": 0.80648, + "map_at_20": 0.80558, + "map_at_3": 0.79256, + "map_at_5": 0.79978, + "mrr_at_1": 0.7324818039103753, + "mrr_at_10": 0.8037402789953872, + "mrr_at_100": 0.8062749361355517, + "mrr_at_1000": 0.8063492699285514, + "mrr_at_20": 0.8054509374987581, + "mrr_at_3": 0.7925170068027252, + "mrr_at_5": 0.7996063460349235, + "nauc_map_at_1000_diff1": 0.7755679733254977, + "nauc_map_at_1000_max": 0.47304164739705884, + "nauc_map_at_1000_std": -0.26244493881093645, + "nauc_map_at_100_diff1": 0.7755521791705124, + "nauc_map_at_100_max": 0.4730912923045687, + "nauc_map_at_100_std": -0.26243222353374807, + "nauc_map_at_10_diff1": 0.7746319200229681, + "nauc_map_at_10_max": 0.4742459632571874, + "nauc_map_at_10_std": -0.26359798504203014, + "nauc_map_at_1_diff1": 0.79527051162636, + "nauc_map_at_1_max": 0.44896317245524625, + "nauc_map_at_1_std": -0.25451766234360096, + "nauc_map_at_20_diff1": 0.7752788532848108, + "nauc_map_at_20_max": 0.4733830573278101, + "nauc_map_at_20_std": -0.2625748232162862, + "nauc_map_at_3_diff1": 0.7741062263238054, + "nauc_map_at_3_max": 0.47499244399918156, + "nauc_map_at_3_std": -0.2652386675834199, + "nauc_map_at_5_diff1": 0.7735129682203654, + "nauc_map_at_5_max": 0.47476812315255235, + "nauc_map_at_5_std": -0.265173636683384, + "nauc_mrr_at_1000_diff1": 0.7757769206330354, + "nauc_mrr_at_1000_max": 0.47343673401512926, + "nauc_mrr_at_1000_std": -0.261670476224202, + "nauc_mrr_at_100_diff1": 0.7757602878422465, + "nauc_mrr_at_100_max": 0.47348765166704015, + "nauc_mrr_at_100_std": -0.2616596878042115, + "nauc_mrr_at_10_diff1": 0.7748926507168112, + "nauc_mrr_at_10_max": 0.4746252680868696, + "nauc_mrr_at_10_std": -0.2628220146139332, + "nauc_mrr_at_1_diff1": 0.7953993768139632, + "nauc_mrr_at_1_max": 0.4495304522643992, + "nauc_mrr_at_1_std": -0.2532633885677888, + "nauc_mrr_at_20_diff1": 0.7754890591729169, + "nauc_mrr_at_20_max": 0.47376316323902873, + "nauc_mrr_at_20_std": -0.26183017296393896, + "nauc_mrr_at_3_diff1": 0.774534326297116, + "nauc_mrr_at_3_max": 0.4754929766473431, + "nauc_mrr_at_3_std": -0.26432202942463784, + "nauc_mrr_at_5_diff1": 0.7738739995097214, + "nauc_mrr_at_5_max": 0.47501103906829356, + "nauc_mrr_at_5_std": -0.2646341620056175, + "nauc_ndcg_at_1000_diff1": 0.7720842500889081, + "nauc_ndcg_at_1000_max": 0.47708138957354207, + "nauc_ndcg_at_1000_std": -0.26023069702012447, + "nauc_ndcg_at_100_diff1": 0.7716015320575459, + "nauc_ndcg_at_100_max": 0.4788272658215937, + "nauc_ndcg_at_100_std": -0.25893271327002004, + "nauc_ndcg_at_10_diff1": 0.7668338764268011, + "nauc_ndcg_at_10_max": 0.484217483255242, + "nauc_ndcg_at_10_std": -0.2636940839993425, + "nauc_ndcg_at_1_diff1": 0.79527051162636, + "nauc_ndcg_at_1_max": 0.44896317245524625, + "nauc_ndcg_at_1_std": -0.25451766234360096, + "nauc_ndcg_at_20_diff1": 0.769371111655833, + "nauc_ndcg_at_20_max": 0.4810138804970404, + "nauc_ndcg_at_20_std": -0.2597419280800229, + "nauc_ndcg_at_3_diff1": 0.7661248461087025, + "nauc_ndcg_at_3_max": 0.4851663194946113, + "nauc_ndcg_at_3_std": -0.2678917728025779, + "nauc_ndcg_at_5_diff1": 0.7642970850899898, + "nauc_ndcg_at_5_max": 0.4852719429311673, + "nauc_ndcg_at_5_std": -0.2678716430555766, + "nauc_precision_at_1000_diff1": 0.7003887145024319, + "nauc_precision_at_1000_max": 0.42004167676466475, + "nauc_precision_at_1000_std": -0.021567781974502518, + "nauc_precision_at_100_diff1": 0.7297434027023919, + "nauc_precision_at_100_max": 0.564734539374456, + "nauc_precision_at_100_std": -0.1225156037378497, + "nauc_precision_at_10_diff1": 0.7150550227914846, + "nauc_precision_at_10_max": 0.5509637660292315, + "nauc_precision_at_10_std": -0.25929290375863356, + "nauc_precision_at_1_diff1": 0.79527051162636, + "nauc_precision_at_1_max": 0.44896317245524625, + "nauc_precision_at_1_std": -0.25451766234360096, + "nauc_precision_at_20_diff1": 0.7199203963125369, + "nauc_precision_at_20_max": 0.5466712448432459, + "nauc_precision_at_20_std": -0.2147885265378636, + "nauc_precision_at_3_diff1": 0.7333785677931962, + "nauc_precision_at_3_max": 0.5270785427475565, + "nauc_precision_at_3_std": -0.2781147380600097, + "nauc_precision_at_5_diff1": 0.7176130552913552, + "nauc_precision_at_5_max": 0.5378865964066962, + "nauc_precision_at_5_std": -0.28044309543143614, + "nauc_recall_at_1000_diff1": 0.7003887145024235, + "nauc_recall_at_1000_max": 0.4200416767647104, + "nauc_recall_at_1000_std": -0.0215677819744854, + "nauc_recall_at_100_diff1": 0.72974340270239, + "nauc_recall_at_100_max": 0.5647345393744531, + "nauc_recall_at_100_std": -0.1225156037378462, + "nauc_recall_at_10_diff1": 0.7150550227914844, + "nauc_recall_at_10_max": 0.55096376602923, + "nauc_recall_at_10_std": -0.25929290375863673, + "nauc_recall_at_1_diff1": 0.79527051162636, + "nauc_recall_at_1_max": 0.44896317245524625, + "nauc_recall_at_1_std": -0.25451766234360096, + "nauc_recall_at_20_diff1": 0.7199203963125299, + "nauc_recall_at_20_max": 0.5466712448432437, + "nauc_recall_at_20_std": -0.21478852653786606, + "nauc_recall_at_3_diff1": 0.7333785677931951, + "nauc_recall_at_3_max": 0.5270785427475563, + "nauc_recall_at_3_std": -0.2781147380600106, + "nauc_recall_at_5_diff1": 0.7176130552913536, + "nauc_recall_at_5_max": 0.5378865964066935, + "nauc_recall_at_5_std": -0.28044309543143714, + "ndcg_at_1": 0.73255, + "ndcg_at_10": 0.83388, + "ndcg_at_100": 0.84571, + "ndcg_at_1000": 0.84768, + "ndcg_at_20": 0.83998, + "ndcg_at_3": 0.81118, + "ndcg_at_5": 0.82409, + "precision_at_1": 0.73255, + "precision_at_10": 0.09259, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "precision_at_20": 0.04749, + "precision_at_3": 0.28826, + "precision_at_5": 0.17918, + "recall_at_1": 0.73255, + "recall_at_10": 0.92586, + "recall_at_100": 0.98016, + "recall_at_1000": 0.99572, + "recall_at_20": 0.94976, + "recall_at_3": 0.86478, + "recall_at_5": 0.89589 + } + ] + }, + "task_name": "CodeSearchNetCCRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetRetrieval.json b/results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetRetrieval.json new file mode 100644 index 000000000..905f67fe3 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "fdc6a9e39575768c27eb8a2a5f702bf846eb4759", + "evaluation_time": 147.30202841758728, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "python", + "languages": [ + "python-Code" + ], + "main_score": 0.96688, + "map_at_1": 0.927, + "map_at_10": 0.95643, + "map_at_100": 0.95649, + "map_at_1000": 0.95649, + "map_at_20": 0.95643, + "map_at_3": 0.95417, + "map_at_5": 0.95562, + "mrr_at_1": 0.927, + "mrr_at_10": 0.9564289682539684, + "mrr_at_100": 0.9564912393162395, + "mrr_at_1000": 0.9564912393162395, + "mrr_at_20": 0.9564289682539684, + "mrr_at_3": 0.9541666666666668, + "mrr_at_5": 0.9556166666666668, + "nauc_map_at_1000_diff1": 0.932086804630001, + "nauc_map_at_1000_max": 0.7136336949003405, + "nauc_map_at_1000_std": 0.07542855305507389, + "nauc_map_at_100_diff1": 0.932086804630001, + "nauc_map_at_100_max": 0.7136336949003405, + "nauc_map_at_100_std": 0.07542855305507389, + "nauc_map_at_10_diff1": 0.9321838651458632, + "nauc_map_at_10_max": 0.7140429653101196, + "nauc_map_at_10_std": 0.07674993664044225, + "nauc_map_at_1_diff1": 0.9297225739610897, + "nauc_map_at_1_max": 0.6906680480411342, + "nauc_map_at_1_std": 0.08970620211554939, + "nauc_map_at_20_diff1": 0.9321838651458632, + "nauc_map_at_20_max": 0.7140429653101196, + "nauc_map_at_20_std": 0.07674993664044225, + "nauc_map_at_3_diff1": 0.9322689075630252, + "nauc_map_at_3_max": 0.7183278159748736, + "nauc_map_at_3_std": 0.058383838383841875, + "nauc_map_at_5_diff1": 0.9307924446534143, + "nauc_map_at_5_max": 0.7148845769375428, + "nauc_map_at_5_std": 0.06123107648366922, + "nauc_mrr_at_1000_diff1": 0.932086804630001, + "nauc_mrr_at_1000_max": 0.7136336949003405, + "nauc_mrr_at_1000_std": 0.07542855305507389, + "nauc_mrr_at_100_diff1": 0.932086804630001, + "nauc_mrr_at_100_max": 0.7136336949003405, + "nauc_mrr_at_100_std": 0.07542855305507389, + "nauc_mrr_at_10_diff1": 0.9321838651458632, + "nauc_mrr_at_10_max": 0.7140429653101196, + "nauc_mrr_at_10_std": 0.07674993664044225, + "nauc_mrr_at_1_diff1": 0.9297225739610897, + "nauc_mrr_at_1_max": 0.6906680480411342, + "nauc_mrr_at_1_std": 0.08970620211554939, + "nauc_mrr_at_20_diff1": 0.9321838651458632, + "nauc_mrr_at_20_max": 0.7140429653101196, + "nauc_mrr_at_20_std": 0.07674993664044225, + "nauc_mrr_at_3_diff1": 0.9322689075630252, + "nauc_mrr_at_3_max": 0.7183278159748736, + "nauc_mrr_at_3_std": 0.058383838383841875, + "nauc_mrr_at_5_diff1": 0.9307924446534143, + "nauc_mrr_at_5_max": 0.7148845769375428, + "nauc_mrr_at_5_std": 0.06123107648366922, + "nauc_ndcg_at_1000_diff1": 0.9323304542841752, + "nauc_ndcg_at_1000_max": 0.716308350922939, + "nauc_ndcg_at_1000_std": 0.07571755253398231, + "nauc_ndcg_at_100_diff1": 0.9323304542841752, + "nauc_ndcg_at_100_max": 0.716308350922939, + "nauc_ndcg_at_100_std": 0.07571755253398231, + "nauc_ndcg_at_10_diff1": 0.9331367660323026, + "nauc_ndcg_at_10_max": 0.7196886589636926, + "nauc_ndcg_at_10_std": 0.08673077551486252, + "nauc_ndcg_at_1_diff1": 0.9297225739610897, + "nauc_ndcg_at_1_max": 0.6906680480411342, + "nauc_ndcg_at_1_std": 0.08970620211554939, + "nauc_ndcg_at_20_diff1": 0.9331367660323026, + "nauc_ndcg_at_20_max": 0.7196886589636926, + "nauc_ndcg_at_20_std": 0.08673077551486252, + "nauc_ndcg_at_3_diff1": 0.9323894522522178, + "nauc_ndcg_at_3_max": 0.7290249629532499, + "nauc_ndcg_at_3_std": 0.036178051090847846, + "nauc_ndcg_at_5_diff1": 0.9290972948390606, + "nauc_ndcg_at_5_max": 0.7219526067100511, + "nauc_ndcg_at_5_std": 0.040415822339757206, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.9297225739610897, + "nauc_precision_at_1_max": 0.6906680480411342, + "nauc_precision_at_1_std": 0.08970620211554939, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.9323062558356721, + "nauc_precision_at_3_max": 0.8246298519407813, + "nauc_precision_at_3_std": -0.17583700146723766, + "nauc_precision_at_5_diff1": 0.8978758169934664, + "nauc_precision_at_5_max": 0.8209033613445358, + "nauc_precision_at_5_std": -0.29580999066296026, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 1.0, + "nauc_recall_at_1_diff1": 0.9297225739610897, + "nauc_recall_at_1_max": 0.6906680480411342, + "nauc_recall_at_1_std": 0.08970620211554939, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_3_diff1": 0.9323062558356721, + "nauc_recall_at_3_max": 0.824629851940786, + "nauc_recall_at_3_std": -0.17583700146724698, + "nauc_recall_at_5_diff1": 0.8978758169934649, + "nauc_recall_at_5_max": 0.8209033613445361, + "nauc_recall_at_5_std": -0.2958099906629326, + "ndcg_at_1": 0.927, + "ndcg_at_10": 0.96688, + "ndcg_at_100": 0.96728, + "ndcg_at_1000": 0.96728, + "ndcg_at_20": 0.96688, + "ndcg_at_3": 0.96239, + "ndcg_at_5": 0.96493, + "precision_at_1": 0.927, + "precision_at_10": 0.0998, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.0499, + "precision_at_3": 0.32867, + "precision_at_5": 0.1984, + "recall_at_1": 0.927, + "recall_at_10": 0.998, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.998, + "recall_at_3": 0.986, + "recall_at_5": 0.992 + }, + { + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ], + "main_score": 0.89978, + "map_at_1": 0.85, + "map_at_10": 0.88427, + "map_at_100": 0.88627, + "map_at_1000": 0.88631, + "map_at_20": 0.88588, + "map_at_3": 0.87767, + "map_at_5": 0.88192, + "mrr_at_1": 0.85, + "mrr_at_10": 0.8842742063492065, + "mrr_at_100": 0.8862702226900807, + "mrr_at_1000": 0.8863072320739237, + "mrr_at_20": 0.8858809151632684, + "mrr_at_3": 0.8776666666666667, + "mrr_at_5": 0.881916666666667, + "nauc_map_at_1000_diff1": 0.9072581837961151, + "nauc_map_at_1000_max": 0.6296996529181109, + "nauc_map_at_1000_std": 0.054686257905510446, + "nauc_map_at_100_diff1": 0.9072513183990429, + "nauc_map_at_100_max": 0.6298186646957673, + "nauc_map_at_100_std": 0.054993593856406016, + "nauc_map_at_10_diff1": 0.907954839398183, + "nauc_map_at_10_max": 0.6322150406130067, + "nauc_map_at_10_std": 0.05413600198301859, + "nauc_map_at_1_diff1": 0.9152066384640422, + "nauc_map_at_1_max": 0.5763976570126915, + "nauc_map_at_1_std": 0.014959323136999483, + "nauc_map_at_20_diff1": 0.9071434972102586, + "nauc_map_at_20_max": 0.6307611119431469, + "nauc_map_at_20_std": 0.05640274998494253, + "nauc_map_at_3_diff1": 0.9033141255622977, + "nauc_map_at_3_max": 0.6234164710561914, + "nauc_map_at_3_std": 0.035068574882828926, + "nauc_map_at_5_diff1": 0.9079552786535547, + "nauc_map_at_5_max": 0.6311994420347811, + "nauc_map_at_5_std": 0.04953420352568366, + "nauc_mrr_at_1000_diff1": 0.9072581837961151, + "nauc_mrr_at_1000_max": 0.6296996529181109, + "nauc_mrr_at_1000_std": 0.054686257905510446, + "nauc_mrr_at_100_diff1": 0.9072513183990429, + "nauc_mrr_at_100_max": 0.6298186646957673, + "nauc_mrr_at_100_std": 0.054993593856406016, + "nauc_mrr_at_10_diff1": 0.907954839398183, + "nauc_mrr_at_10_max": 0.6322150406130067, + "nauc_mrr_at_10_std": 0.05413600198301859, + "nauc_mrr_at_1_diff1": 0.9152066384640422, + "nauc_mrr_at_1_max": 0.5763976570126915, + "nauc_mrr_at_1_std": 0.014959323136999483, + "nauc_mrr_at_20_diff1": 0.9071434972102586, + "nauc_mrr_at_20_max": 0.6307611119431469, + "nauc_mrr_at_20_std": 0.05640274998494253, + "nauc_mrr_at_3_diff1": 0.9033141255622977, + "nauc_mrr_at_3_max": 0.6234164710561914, + "nauc_mrr_at_3_std": 0.035068574882828926, + "nauc_mrr_at_5_diff1": 0.9079552786535547, + "nauc_mrr_at_5_max": 0.6311994420347811, + "nauc_mrr_at_5_std": 0.04953420352568366, + "nauc_ndcg_at_1000_diff1": 0.906732163208453, + "nauc_ndcg_at_1000_max": 0.6427542517763368, + "nauc_ndcg_at_1000_std": 0.07423716824052952, + "nauc_ndcg_at_100_diff1": 0.9065475476530616, + "nauc_ndcg_at_100_max": 0.6489222263529967, + "nauc_ndcg_at_100_std": 0.09036733154889919, + "nauc_ndcg_at_10_diff1": 0.9088391240350151, + "nauc_ndcg_at_10_max": 0.6610798661293683, + "nauc_ndcg_at_10_std": 0.08662637793277093, + "nauc_ndcg_at_1_diff1": 0.9152066384640422, + "nauc_ndcg_at_1_max": 0.5763976570126915, + "nauc_ndcg_at_1_std": 0.014959323136999483, + "nauc_ndcg_at_20_diff1": 0.9055301735430408, + "nauc_ndcg_at_20_max": 0.6565827895142017, + "nauc_ndcg_at_20_std": 0.09990699968426656, + "nauc_ndcg_at_3_diff1": 0.8996356996211253, + "nauc_ndcg_at_3_max": 0.6403584890384898, + "nauc_ndcg_at_3_std": 0.042959293509216484, + "nauc_ndcg_at_5_diff1": 0.9089636835182858, + "nauc_ndcg_at_5_max": 0.6569109133325729, + "nauc_ndcg_at_5_std": 0.07251888023539761, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8989595838335227, + "nauc_precision_at_100_max": 0.981325863678799, + "nauc_precision_at_100_std": 0.9588168600773691, + "nauc_precision_at_10_diff1": 0.9180313150901407, + "nauc_precision_at_10_max": 0.8473120017237689, + "nauc_precision_at_10_std": 0.30963154492565964, + "nauc_precision_at_1_diff1": 0.9152066384640422, + "nauc_precision_at_1_max": 0.5763976570126915, + "nauc_precision_at_1_std": 0.014959323136999483, + "nauc_precision_at_20_diff1": 0.8851379632312696, + "nauc_precision_at_20_max": 0.9389870890885065, + "nauc_precision_at_20_std": 0.6606458675424168, + "nauc_precision_at_3_diff1": 0.885919911035577, + "nauc_precision_at_3_max": 0.7047755431760744, + "nauc_precision_at_3_std": 0.07324877516549845, + "nauc_precision_at_5_diff1": 0.9168000533546777, + "nauc_precision_at_5_max": 0.7808323329331722, + "nauc_precision_at_5_std": 0.1878484727224213, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.898959583833538, + "nauc_recall_at_100_max": 0.9813258636788038, + "nauc_recall_at_100_std": 0.9588168600773624, + "nauc_recall_at_10_diff1": 0.9180313150901372, + "nauc_recall_at_10_max": 0.8473120017237645, + "nauc_recall_at_10_std": 0.30963154492566203, + "nauc_recall_at_1_diff1": 0.9152066384640422, + "nauc_recall_at_1_max": 0.5763976570126915, + "nauc_recall_at_1_std": 0.014959323136999483, + "nauc_recall_at_20_diff1": 0.8851379632312701, + "nauc_recall_at_20_max": 0.9389870890885045, + "nauc_recall_at_20_std": 0.660645867542416, + "nauc_recall_at_3_diff1": 0.8859199110355753, + "nauc_recall_at_3_max": 0.7047755431760725, + "nauc_recall_at_3_std": 0.07324877516549637, + "nauc_recall_at_5_diff1": 0.9168000533546758, + "nauc_recall_at_5_max": 0.7808323329331743, + "nauc_recall_at_5_std": 0.18784847272242206, + "ndcg_at_1": 0.85, + "ndcg_at_10": 0.89978, + "ndcg_at_100": 0.90842, + "ndcg_at_1000": 0.91004, + "ndcg_at_20": 0.90561, + "ndcg_at_3": 0.88626, + "ndcg_at_5": 0.89401, + "precision_at_1": 0.85, + "precision_at_10": 0.0948, + "precision_at_100": 0.00986, + "precision_at_1000": 0.001, + "precision_at_20": 0.04855, + "precision_at_3": 0.30367, + "precision_at_5": 0.186, + "recall_at_1": 0.85, + "recall_at_10": 0.948, + "recall_at_100": 0.986, + "recall_at_1000": 1.0, + "recall_at_20": 0.971, + "recall_at_3": 0.911, + "recall_at_5": 0.93 + }, + { + "hf_subset": "go", + "languages": [ + "go-Code" + ], + "main_score": 0.97505, + "map_at_1": 0.944, + "map_at_10": 0.96808, + "map_at_100": 0.96823, + "map_at_1000": 0.96824, + "map_at_20": 0.96813, + "map_at_3": 0.96733, + "map_at_5": 0.96808, + "mrr_at_1": 0.944, + "mrr_at_10": 0.9680833333333334, + "mrr_at_100": 0.9682307275541795, + "mrr_at_1000": 0.9682390608875128, + "mrr_at_20": 0.9681333333333333, + "mrr_at_3": 0.9673333333333334, + "mrr_at_5": 0.9680833333333334, + "nauc_map_at_1000_diff1": 0.9727333921447281, + "nauc_map_at_1000_max": 0.40531775829415156, + "nauc_map_at_1000_std": -0.09684978530016271, + "nauc_map_at_100_diff1": 0.9727405443932574, + "nauc_map_at_100_max": 0.40547374818520093, + "nauc_map_at_100_std": -0.09663493618536893, + "nauc_map_at_10_diff1": 0.9728664311677696, + "nauc_map_at_10_max": 0.40811154749106154, + "nauc_map_at_10_std": -0.10076963770712538, + "nauc_map_at_1_diff1": 0.9690709617180209, + "nauc_map_at_1_max": 0.3333833533413366, + "nauc_map_at_1_std": -0.06367547018807825, + "nauc_map_at_20_diff1": 0.9728238575765058, + "nauc_map_at_20_max": 0.40718285222038564, + "nauc_map_at_20_std": -0.09912064546879207, + "nauc_map_at_3_diff1": 0.9734893957583056, + "nauc_map_at_3_max": 0.4167881438289635, + "nauc_map_at_3_std": -0.09509041711922596, + "nauc_map_at_5_diff1": 0.9728664311677696, + "nauc_map_at_5_max": 0.40811154749106154, + "nauc_map_at_5_std": -0.10076963770712538, + "nauc_mrr_at_1000_diff1": 0.9727333921447281, + "nauc_mrr_at_1000_max": 0.40531775829415156, + "nauc_mrr_at_1000_std": -0.09684978530016271, + "nauc_mrr_at_100_diff1": 0.9727405443932574, + "nauc_mrr_at_100_max": 0.40547374818520093, + "nauc_mrr_at_100_std": -0.09663493618536893, + "nauc_mrr_at_10_diff1": 0.9728664311677696, + "nauc_mrr_at_10_max": 0.40811154749106154, + "nauc_mrr_at_10_std": -0.10076963770712538, + "nauc_mrr_at_1_diff1": 0.9690709617180209, + "nauc_mrr_at_1_max": 0.3333833533413366, + "nauc_mrr_at_1_std": -0.06367547018807825, + "nauc_mrr_at_20_diff1": 0.9728238575765058, + "nauc_mrr_at_20_max": 0.40718285222038564, + "nauc_mrr_at_20_std": -0.09912064546879207, + "nauc_mrr_at_3_diff1": 0.9734893957583056, + "nauc_mrr_at_3_max": 0.4167881438289635, + "nauc_mrr_at_3_std": -0.09509041711922596, + "nauc_mrr_at_5_diff1": 0.9728664311677696, + "nauc_mrr_at_5_max": 0.40811154749106154, + "nauc_mrr_at_5_std": -0.10076963770712538, + "nauc_ndcg_at_1000_diff1": 0.9733376561125955, + "nauc_ndcg_at_1000_max": 0.41757379851748067, + "nauc_ndcg_at_1000_std": -0.10511570027083764, + "nauc_ndcg_at_100_diff1": 0.9734974235237086, + "nauc_ndcg_at_100_max": 0.42106384150737575, + "nauc_ndcg_at_100_std": -0.10015808519562162, + "nauc_ndcg_at_10_diff1": 0.9743761675418023, + "nauc_ndcg_at_10_max": 0.4392682037582593, + "nauc_ndcg_at_10_std": -0.1311593389665566, + "nauc_ndcg_at_1_diff1": 0.9690709617180209, + "nauc_ndcg_at_1_max": 0.3333833533413366, + "nauc_ndcg_at_1_std": -0.06367547018807825, + "nauc_ndcg_at_20_diff1": 0.9741401672160926, + "nauc_ndcg_at_20_max": 0.4341037582459886, + "nauc_ndcg_at_20_std": -0.12175971519065602, + "nauc_ndcg_at_3_diff1": 0.9756379014982639, + "nauc_ndcg_at_3_max": 0.4563427566353211, + "nauc_ndcg_at_3_std": -0.11748268623706999, + "nauc_ndcg_at_5_diff1": 0.9743761675418023, + "nauc_ndcg_at_5_max": 0.4392682037582593, + "nauc_ndcg_at_5_std": -0.1311593389665566, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.7222222222222335, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 0.9738562091503177, + "nauc_precision_at_10_std": -0.6985060690943635, + "nauc_precision_at_1_diff1": 0.9690709617180209, + "nauc_precision_at_1_max": 0.3333833533413366, + "nauc_precision_at_1_std": -0.06367547018807825, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 0.9673202614378965, + "nauc_precision_at_20_std": -0.5852007469655789, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 0.9034197012138252, + "nauc_precision_at_3_std": -0.3815943043884111, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 0.9738562091503177, + "nauc_precision_at_5_std": -0.6985060690943635, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.7222222222222368, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_10_max": 0.9738562091503128, + "nauc_recall_at_10_std": -0.6985060690942723, + "nauc_recall_at_1_diff1": 0.9690709617180209, + "nauc_recall_at_1_max": 0.3333833533413366, + "nauc_recall_at_1_std": -0.06367547018807825, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_20_max": 0.967320261437891, + "nauc_recall_at_20_std": -0.5852007469654495, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_3_max": 0.9034197012138186, + "nauc_recall_at_3_std": -0.38159430438842934, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_5_max": 0.9738562091503128, + "nauc_recall_at_5_std": -0.6985060690942723, + "ndcg_at_1": 0.944, + "ndcg_at_10": 0.97505, + "ndcg_at_100": 0.97588, + "ndcg_at_1000": 0.97602, + "ndcg_at_20": 0.97528, + "ndcg_at_3": 0.97376, + "ndcg_at_5": 0.97505, + "precision_at_1": 0.944, + "precision_at_10": 0.0995, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.0498, + "precision_at_3": 0.33067, + "precision_at_5": 0.199, + "recall_at_1": 0.944, + "recall_at_10": 0.995, + "recall_at_100": 0.999, + "recall_at_1000": 1.0, + "recall_at_20": 0.996, + "recall_at_3": 0.992, + "recall_at_5": 0.995 + }, + { + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ], + "main_score": 0.92832, + "map_at_1": 0.877, + "map_at_10": 0.91496, + "map_at_100": 0.91582, + "map_at_1000": 0.91587, + "map_at_20": 0.9155, + "map_at_3": 0.91167, + "map_at_5": 0.91397, + "mrr_at_1": 0.877, + "mrr_at_10": 0.9149646825396827, + "mrr_at_100": 0.9158154350757961, + "mrr_at_1000": 0.9158711452954685, + "mrr_at_20": 0.9154990637415562, + "mrr_at_3": 0.9116666666666668, + "mrr_at_5": 0.9139666666666668, + "nauc_map_at_1000_diff1": 0.9001079432013319, + "nauc_map_at_1000_max": 0.7625110199586136, + "nauc_map_at_1000_std": 0.2195406993804367, + "nauc_map_at_100_diff1": 0.9001415529036015, + "nauc_map_at_100_max": 0.7626710387746687, + "nauc_map_at_100_std": 0.2199148984749778, + "nauc_map_at_10_diff1": 0.8988101575524061, + "nauc_map_at_10_max": 0.7640985435650426, + "nauc_map_at_10_std": 0.22554999001780365, + "nauc_map_at_1_diff1": 0.9045505123995242, + "nauc_map_at_1_max": 0.7027343537229698, + "nauc_map_at_1_std": 0.10378716550076664, + "nauc_map_at_20_diff1": 0.9003751820546869, + "nauc_map_at_20_max": 0.7635756916603282, + "nauc_map_at_20_std": 0.2215447723869247, + "nauc_map_at_3_diff1": 0.8990841801530556, + "nauc_map_at_3_max": 0.7690028137713493, + "nauc_map_at_3_std": 0.2184373711848299, + "nauc_map_at_5_diff1": 0.8993455699311205, + "nauc_map_at_5_max": 0.7652511956217162, + "nauc_map_at_5_std": 0.22235121325826523, + "nauc_mrr_at_1000_diff1": 0.9001079432013319, + "nauc_mrr_at_1000_max": 0.7625110199586136, + "nauc_mrr_at_1000_std": 0.2195406993804367, + "nauc_mrr_at_100_diff1": 0.9001415529036015, + "nauc_mrr_at_100_max": 0.7626710387746687, + "nauc_mrr_at_100_std": 0.2199148984749778, + "nauc_mrr_at_10_diff1": 0.8988101575524061, + "nauc_mrr_at_10_max": 0.7640985435650426, + "nauc_mrr_at_10_std": 0.22554999001780365, + "nauc_mrr_at_1_diff1": 0.9045505123995242, + "nauc_mrr_at_1_max": 0.7027343537229698, + "nauc_mrr_at_1_std": 0.10378716550076664, + "nauc_mrr_at_20_diff1": 0.9003751820546869, + "nauc_mrr_at_20_max": 0.7635756916603282, + "nauc_mrr_at_20_std": 0.2215447723869247, + "nauc_mrr_at_3_diff1": 0.8990841801530556, + "nauc_mrr_at_3_max": 0.7690028137713493, + "nauc_mrr_at_3_std": 0.2184373711848299, + "nauc_mrr_at_5_diff1": 0.8993455699311205, + "nauc_mrr_at_5_max": 0.7652511956217162, + "nauc_mrr_at_5_std": 0.22235121325826523, + "nauc_ndcg_at_1000_diff1": 0.9003883767897191, + "nauc_ndcg_at_1000_max": 0.774286773752732, + "nauc_ndcg_at_1000_std": 0.24905596869880495, + "nauc_ndcg_at_100_diff1": 0.9015705261849951, + "nauc_ndcg_at_100_max": 0.7797459201928656, + "nauc_ndcg_at_100_std": 0.2636432448776628, + "nauc_ndcg_at_10_diff1": 0.8960631662977974, + "nauc_ndcg_at_10_max": 0.7882415924492551, + "nauc_ndcg_at_10_std": 0.29259897960486475, + "nauc_ndcg_at_1_diff1": 0.9045505123995242, + "nauc_ndcg_at_1_max": 0.7027343537229698, + "nauc_ndcg_at_1_std": 0.10378716550076664, + "nauc_ndcg_at_20_diff1": 0.9034712330725388, + "nauc_ndcg_at_20_max": 0.7865263241067265, + "nauc_ndcg_at_20_std": 0.27634547611895843, + "nauc_ndcg_at_3_diff1": 0.8969525439370882, + "nauc_ndcg_at_3_max": 0.7968227200895334, + "nauc_ndcg_at_3_std": 0.2716978363425553, + "nauc_ndcg_at_5_diff1": 0.8974542043203508, + "nauc_ndcg_at_5_max": 0.790789328591088, + "nauc_ndcg_at_5_std": 0.2835584817342069, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.9484665661136168, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.8696760755584259, + "nauc_precision_at_10_diff1": 0.87527719421102, + "nauc_precision_at_10_max": 0.961922268907559, + "nauc_precision_at_10_std": 0.8032942343604061, + "nauc_precision_at_1_diff1": 0.9045505123995242, + "nauc_precision_at_1_max": 0.7027343537229698, + "nauc_precision_at_1_std": 0.10378716550076664, + "nauc_precision_at_20_diff1": 0.955746187363837, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.7869397759103581, + "nauc_precision_at_3_diff1": 0.8863450141961569, + "nauc_precision_at_3_max": 0.9364984088873672, + "nauc_precision_at_3_std": 0.542083500066697, + "nauc_precision_at_5_diff1": 0.8862314156431772, + "nauc_precision_at_5_max": 0.945892887069356, + "nauc_precision_at_5_std": 0.6680552562905465, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.948466566113629, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.869676075558426, + "nauc_recall_at_10_diff1": 0.8752771942110217, + "nauc_recall_at_10_max": 0.9619222689075643, + "nauc_recall_at_10_std": 0.8032942343604144, + "nauc_recall_at_1_diff1": 0.9045505123995242, + "nauc_recall_at_1_max": 0.7027343537229698, + "nauc_recall_at_1_std": 0.10378716550076664, + "nauc_recall_at_20_diff1": 0.9557461873638331, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.7869397759103617, + "nauc_recall_at_3_diff1": 0.8863450141961552, + "nauc_recall_at_3_max": 0.9364984088873658, + "nauc_recall_at_3_std": 0.5420835000666945, + "nauc_recall_at_5_diff1": 0.8862314156431814, + "nauc_recall_at_5_max": 0.9458928870693577, + "nauc_recall_at_5_std": 0.6680552562905518, + "ndcg_at_1": 0.877, + "ndcg_at_10": 0.92832, + "ndcg_at_100": 0.93242, + "ndcg_at_1000": 0.93403, + "ndcg_at_20": 0.93031, + "ndcg_at_3": 0.92186, + "ndcg_at_5": 0.92599, + "precision_at_1": 0.877, + "precision_at_10": 0.0968, + "precision_at_100": 0.00987, + "precision_at_1000": 0.001, + "precision_at_20": 0.0488, + "precision_at_3": 0.317, + "precision_at_5": 0.1922, + "recall_at_1": 0.877, + "recall_at_10": 0.968, + "recall_at_100": 0.987, + "recall_at_1000": 1.0, + "recall_at_20": 0.976, + "recall_at_3": 0.951, + "recall_at_5": 0.961 + }, + { + "hf_subset": "java", + "languages": [ + "java-Code" + ], + "main_score": 0.94507, + "map_at_1": 0.914, + "map_at_10": 0.93677, + "map_at_100": 0.93746, + "map_at_1000": 0.93753, + "map_at_20": 0.93706, + "map_at_3": 0.93433, + "map_at_5": 0.93593, + "mrr_at_1": 0.914, + "mrr_at_10": 0.9367706349206348, + "mrr_at_100": 0.9374605961033865, + "mrr_at_1000": 0.9375307690063596, + "mrr_at_20": 0.9370575802250722, + "mrr_at_3": 0.9343333333333333, + "mrr_at_5": 0.9359333333333333, + "nauc_map_at_1000_diff1": 0.9196988867062096, + "nauc_map_at_1000_max": 0.8087530316508049, + "nauc_map_at_1000_std": 0.5438722696006999, + "nauc_map_at_100_diff1": 0.9197889892956285, + "nauc_map_at_100_max": 0.8089676220389074, + "nauc_map_at_100_std": 0.544132005408842, + "nauc_map_at_10_diff1": 0.9200847920033857, + "nauc_map_at_10_max": 0.8110521744502199, + "nauc_map_at_10_std": 0.546879092966103, + "nauc_map_at_1_diff1": 0.8960328317373455, + "nauc_map_at_1_max": 0.7247899159663876, + "nauc_map_at_1_std": 0.4426584587323289, + "nauc_map_at_20_diff1": 0.9202464449562623, + "nauc_map_at_20_max": 0.8101907888930855, + "nauc_map_at_20_std": 0.5457103107368239, + "nauc_map_at_3_diff1": 0.9209358870451744, + "nauc_map_at_3_max": 0.8170076829378119, + "nauc_map_at_3_std": 0.5527046690080435, + "nauc_map_at_5_diff1": 0.9211292217199042, + "nauc_map_at_5_max": 0.813521575817287, + "nauc_map_at_5_std": 0.5499813695856418, + "nauc_mrr_at_1000_diff1": 0.9196988867062096, + "nauc_mrr_at_1000_max": 0.8087530316508049, + "nauc_mrr_at_1000_std": 0.5438722696006999, + "nauc_mrr_at_100_diff1": 0.9197889892956285, + "nauc_mrr_at_100_max": 0.8089676220389074, + "nauc_mrr_at_100_std": 0.544132005408842, + "nauc_mrr_at_10_diff1": 0.9200847920033857, + "nauc_mrr_at_10_max": 0.8110521744502199, + "nauc_mrr_at_10_std": 0.546879092966103, + "nauc_mrr_at_1_diff1": 0.8960328317373455, + "nauc_mrr_at_1_max": 0.7247899159663876, + "nauc_mrr_at_1_std": 0.4426584587323289, + "nauc_mrr_at_20_diff1": 0.9202464449562623, + "nauc_mrr_at_20_max": 0.8101907888930855, + "nauc_mrr_at_20_std": 0.5457103107368239, + "nauc_mrr_at_3_diff1": 0.9209358870451744, + "nauc_mrr_at_3_max": 0.8170076829378119, + "nauc_mrr_at_3_std": 0.5527046690080435, + "nauc_mrr_at_5_diff1": 0.9211292217199042, + "nauc_mrr_at_5_max": 0.813521575817287, + "nauc_mrr_at_5_std": 0.5499813695856418, + "nauc_ndcg_at_1000_diff1": 0.9239514411202763, + "nauc_ndcg_at_1000_max": 0.821789464930351, + "nauc_ndcg_at_1000_std": 0.5598331155582364, + "nauc_ndcg_at_100_diff1": 0.9260315730234768, + "nauc_ndcg_at_100_max": 0.8266640006867892, + "nauc_ndcg_at_100_std": 0.5660969071497668, + "nauc_ndcg_at_10_diff1": 0.9284846723213055, + "nauc_ndcg_at_10_max": 0.8393273402590029, + "nauc_ndcg_at_10_std": 0.5818004894563819, + "nauc_ndcg_at_1_diff1": 0.8960328317373455, + "nauc_ndcg_at_1_max": 0.7247899159663876, + "nauc_ndcg_at_1_std": 0.4426584587323289, + "nauc_ndcg_at_20_diff1": 0.9294552597042011, + "nauc_ndcg_at_20_max": 0.8357732205562041, + "nauc_ndcg_at_20_std": 0.5769560117791322, + "nauc_ndcg_at_3_diff1": 0.9302972756100403, + "nauc_ndcg_at_3_max": 0.8503899149427753, + "nauc_ndcg_at_3_std": 0.5929245146091644, + "nauc_ndcg_at_5_diff1": 0.9309702309051526, + "nauc_ndcg_at_5_max": 0.8449116159880193, + "nauc_ndcg_at_5_std": 0.58887701004976, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.7924836601307065, + "nauc_precision_at_10_diff1": 0.976423902894492, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.7807345160286333, + "nauc_precision_at_1_diff1": 0.8960328317373455, + "nauc_precision_at_1_max": 0.7247899159663876, + "nauc_precision_at_1_std": 0.4426584587323289, + "nauc_precision_at_20_diff1": 0.9947712418300703, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.7769374416433266, + "nauc_precision_at_3_diff1": 0.9706316635181199, + "nauc_precision_at_3_max": 0.9935400516795853, + "nauc_precision_at_3_std": 0.7656070180010007, + "nauc_precision_at_5_diff1": 0.9803532524120767, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.7816163502438045, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.7924836601307194, + "nauc_recall_at_10_diff1": 0.9764239028944921, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.7807345160286355, + "nauc_recall_at_1_diff1": 0.8960328317373455, + "nauc_recall_at_1_max": 0.7247899159663876, + "nauc_recall_at_1_std": 0.4426584587323289, + "nauc_recall_at_20_diff1": 0.9947712418300625, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.7769374416433251, + "nauc_recall_at_3_diff1": 0.9706316635181186, + "nauc_recall_at_3_max": 0.9935400516795868, + "nauc_recall_at_3_std": 0.7656070180009995, + "nauc_recall_at_5_diff1": 0.9803532524120768, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.7816163502438028, + "ndcg_at_1": 0.914, + "ndcg_at_10": 0.94507, + "ndcg_at_100": 0.94909, + "ndcg_at_1000": 0.95048, + "ndcg_at_20": 0.94626, + "ndcg_at_3": 0.94021, + "ndcg_at_5": 0.9431, + "precision_at_1": 0.914, + "precision_at_10": 0.097, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "precision_at_20": 0.04875, + "precision_at_3": 0.319, + "precision_at_5": 0.1928, + "recall_at_1": 0.914, + "recall_at_10": 0.97, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "recall_at_20": 0.975, + "recall_at_3": 0.957, + "recall_at_5": 0.964 + }, + { + "hf_subset": "php", + "languages": [ + "php-Code" + ], + "main_score": 0.92288, + "map_at_1": 0.855, + "map_at_10": 0.90312, + "map_at_100": 0.90351, + "map_at_1000": 0.90353, + "map_at_20": 0.90335, + "map_at_3": 0.89583, + "map_at_5": 0.90098, + "mrr_at_1": 0.855, + "mrr_at_10": 0.9031230158730165, + "mrr_at_100": 0.9035130514931716, + "mrr_at_1000": 0.9035337250767379, + "mrr_at_20": 0.9033520202020209, + "mrr_at_3": 0.8958333333333338, + "mrr_at_5": 0.9009833333333341, + "nauc_map_at_1000_diff1": 0.9385544381917797, + "nauc_map_at_1000_max": 0.40904724092470085, + "nauc_map_at_1000_std": -0.16463521873713016, + "nauc_map_at_100_diff1": 0.9385516451221955, + "nauc_map_at_100_max": 0.40917784128196594, + "nauc_map_at_100_std": -0.16437783475285409, + "nauc_map_at_10_diff1": 0.9382165830201601, + "nauc_map_at_10_max": 0.41163102386298805, + "nauc_map_at_10_std": -0.15954316262724616, + "nauc_map_at_1_diff1": 0.9451659451659448, + "nauc_map_at_1_max": 0.35956911305748634, + "nauc_map_at_1_std": -0.25156884459210044, + "nauc_map_at_20_diff1": 0.9383850503661383, + "nauc_map_at_20_max": 0.41019314488512637, + "nauc_map_at_20_std": -0.16237689928755425, + "nauc_map_at_3_diff1": 0.938083708059495, + "nauc_map_at_3_max": 0.4063545870325532, + "nauc_map_at_3_std": -0.17033014335677799, + "nauc_map_at_5_diff1": 0.939020242116256, + "nauc_map_at_5_max": 0.41174017150832654, + "nauc_map_at_5_std": -0.1630029652673213, + "nauc_mrr_at_1000_diff1": 0.9385544381917797, + "nauc_mrr_at_1000_max": 0.40904724092470085, + "nauc_mrr_at_1000_std": -0.16463521873713016, + "nauc_mrr_at_100_diff1": 0.9385516451221955, + "nauc_mrr_at_100_max": 0.40917784128196594, + "nauc_mrr_at_100_std": -0.16437783475285409, + "nauc_mrr_at_10_diff1": 0.9382165830201601, + "nauc_mrr_at_10_max": 0.41163102386298805, + "nauc_mrr_at_10_std": -0.15954316262724616, + "nauc_mrr_at_1_diff1": 0.9451659451659448, + "nauc_mrr_at_1_max": 0.35956911305748634, + "nauc_mrr_at_1_std": -0.25156884459210044, + "nauc_mrr_at_20_diff1": 0.9383850503661383, + "nauc_mrr_at_20_max": 0.41019314488512637, + "nauc_mrr_at_20_std": -0.16237689928755425, + "nauc_mrr_at_3_diff1": 0.938083708059495, + "nauc_mrr_at_3_max": 0.4063545870325532, + "nauc_mrr_at_3_std": -0.17033014335677799, + "nauc_mrr_at_5_diff1": 0.939020242116256, + "nauc_mrr_at_5_max": 0.41174017150832654, + "nauc_mrr_at_5_std": -0.1630029652673213, + "nauc_ndcg_at_1000_diff1": 0.9377150134331873, + "nauc_ndcg_at_1000_max": 0.4234949722008596, + "nauc_ndcg_at_1000_std": -0.13688355172335143, + "nauc_ndcg_at_100_diff1": 0.9377562254293131, + "nauc_ndcg_at_100_max": 0.4299204970567497, + "nauc_ndcg_at_100_std": -0.1242122424240366, + "nauc_ndcg_at_10_diff1": 0.9354925638165832, + "nauc_ndcg_at_10_max": 0.4455555045908999, + "nauc_ndcg_at_10_std": -0.09337958348868718, + "nauc_ndcg_at_1_diff1": 0.9451659451659448, + "nauc_ndcg_at_1_max": 0.35956911305748634, + "nauc_ndcg_at_1_std": -0.25156884459210044, + "nauc_ndcg_at_20_diff1": 0.936193269489978, + "nauc_ndcg_at_20_max": 0.43968790912006434, + "nauc_ndcg_at_20_std": -0.10495064090760449, + "nauc_ndcg_at_3_diff1": 0.9362155771108923, + "nauc_ndcg_at_3_max": 0.42893378116364106, + "nauc_ndcg_at_3_std": -0.1273029168651973, + "nauc_ndcg_at_5_diff1": 0.9378967591680557, + "nauc_ndcg_at_5_max": 0.4444155136825693, + "nauc_ndcg_at_5_std": -0.10579261977804964, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.9439775910363962, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.891387927720107, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.9451659451659448, + "nauc_precision_at_1_max": 0.35956911305748634, + "nauc_precision_at_1_std": -0.25156884459210044, + "nauc_precision_at_20_diff1": 0.8961251167133568, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.9271284271284276, + "nauc_precision_at_3_max": 0.5485442661913242, + "nauc_precision_at_3_std": 0.10262286732874971, + "nauc_precision_at_5_diff1": 0.9300449346405234, + "nauc_precision_at_5_max": 0.7352795284780563, + "nauc_precision_at_5_std": 0.4020483193277234, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9439775910364198, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_10_diff1": 0.8913879277201103, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 1.0, + "nauc_recall_at_1_diff1": 0.9451659451659448, + "nauc_recall_at_1_max": 0.35956911305748634, + "nauc_recall_at_1_std": -0.25156884459210044, + "nauc_recall_at_20_diff1": 0.8961251167133527, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_3_diff1": 0.9271284271284289, + "nauc_recall_at_3_max": 0.5485442661913248, + "nauc_recall_at_3_std": 0.10262286732875078, + "nauc_recall_at_5_diff1": 0.9300449346405258, + "nauc_recall_at_5_max": 0.7352795284780608, + "nauc_recall_at_5_std": 0.4020483193277305, + "ndcg_at_1": 0.855, + "ndcg_at_10": 0.92288, + "ndcg_at_100": 0.92493, + "ndcg_at_1000": 0.92574, + "ndcg_at_20": 0.92366, + "ndcg_at_3": 0.90851, + "ndcg_at_5": 0.91789, + "precision_at_1": 0.855, + "precision_at_10": 0.0983, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "precision_at_20": 0.0493, + "precision_at_3": 0.315, + "precision_at_5": 0.1936, + "recall_at_1": 0.855, + "recall_at_10": 0.983, + "recall_at_100": 0.993, + "recall_at_1000": 1.0, + "recall_at_20": 0.986, + "recall_at_3": 0.945, + "recall_at_5": 0.968 + } + ] + }, + "task_name": "CodeSearchNetRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanContest.json b/results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanContest.json new file mode 100644 index 000000000..f5abcc83e --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanContest.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "20da4eb20a4b17300c0986ee148c90867a7f2a4d", + "evaluation_time": 27.283839464187622, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "python-Code", + "c++-Code" + ], + "main_score": 0.9496, + "map_at_1": 0.91855, + "map_at_10": 0.94054, + "map_at_100": 0.94141, + "map_at_1000": 0.94145, + "map_at_20": 0.94124, + "map_at_3": 0.93741, + "map_at_5": 0.93922, + "mrr_at_1": 0.918552036199095, + "mrr_at_10": 0.9405354449472096, + "mrr_at_100": 0.9414134449135242, + "mrr_at_1000": 0.9414490739440583, + "mrr_at_20": 0.9412357250592545, + "mrr_at_3": 0.9374057315233786, + "mrr_at_5": 0.9392156862745098, + "nauc_map_at_1000_diff1": 0.9255002145543314, + "nauc_map_at_1000_max": 0.839544206201828, + "nauc_map_at_1000_std": 0.13818812515717432, + "nauc_map_at_100_diff1": 0.9255455211118941, + "nauc_map_at_100_max": 0.8396417863440325, + "nauc_map_at_100_std": 0.13871223041576353, + "nauc_map_at_10_diff1": 0.9258163562742964, + "nauc_map_at_10_max": 0.8420094910390933, + "nauc_map_at_10_std": 0.1482757003817901, + "nauc_map_at_1_diff1": 0.908130775986731, + "nauc_map_at_1_max": 0.8111634573555125, + "nauc_map_at_1_std": 0.061808904777442504, + "nauc_map_at_20_diff1": 0.9257706925983298, + "nauc_map_at_20_max": 0.8401267551178432, + "nauc_map_at_20_std": 0.1399068357350348, + "nauc_map_at_3_diff1": 0.9276365755738576, + "nauc_map_at_3_max": 0.8418142235516605, + "nauc_map_at_3_std": 0.11473399520154405, + "nauc_map_at_5_diff1": 0.9274270433278942, + "nauc_map_at_5_max": 0.8412361744519419, + "nauc_map_at_5_std": 0.1466866942116594, + "nauc_mrr_at_1000_diff1": 0.9255002145543314, + "nauc_mrr_at_1000_max": 0.839544206201828, + "nauc_mrr_at_1000_std": 0.13818812515717432, + "nauc_mrr_at_100_diff1": 0.9255455211118941, + "nauc_mrr_at_100_max": 0.8396417863440325, + "nauc_mrr_at_100_std": 0.13871223041576353, + "nauc_mrr_at_10_diff1": 0.9258163562742964, + "nauc_mrr_at_10_max": 0.8420094910390933, + "nauc_mrr_at_10_std": 0.1482757003817901, + "nauc_mrr_at_1_diff1": 0.908130775986731, + "nauc_mrr_at_1_max": 0.8111634573555125, + "nauc_mrr_at_1_std": 0.061808904777442504, + "nauc_mrr_at_20_diff1": 0.9257706925983298, + "nauc_mrr_at_20_max": 0.8401267551178432, + "nauc_mrr_at_20_std": 0.1399068357350348, + "nauc_mrr_at_3_diff1": 0.9276365755738576, + "nauc_mrr_at_3_max": 0.8418142235516605, + "nauc_mrr_at_3_std": 0.11473399520154405, + "nauc_mrr_at_5_diff1": 0.9274270433278942, + "nauc_mrr_at_5_max": 0.8412361744519419, + "nauc_mrr_at_5_std": 0.1466866942116594, + "nauc_ndcg_at_1000_diff1": 0.9284906743720337, + "nauc_ndcg_at_1000_max": 0.8457965223804021, + "nauc_ndcg_at_1000_std": 0.1587560281416624, + "nauc_ndcg_at_100_diff1": 0.9294862546375189, + "nauc_ndcg_at_100_max": 0.8479434023549941, + "nauc_ndcg_at_100_std": 0.17046814945571057, + "nauc_ndcg_at_10_diff1": 0.9318693605775723, + "nauc_ndcg_at_10_max": 0.8599181169093023, + "nauc_ndcg_at_10_std": 0.21453785415429227, + "nauc_ndcg_at_1_diff1": 0.908130775986731, + "nauc_ndcg_at_1_max": 0.8111634573555125, + "nauc_ndcg_at_1_std": 0.061808904777442504, + "nauc_ndcg_at_20_diff1": 0.9318182308386799, + "nauc_ndcg_at_20_max": 0.8529721008749047, + "nauc_ndcg_at_20_std": 0.18265825551265774, + "nauc_ndcg_at_3_diff1": 0.935678392571751, + "nauc_ndcg_at_3_max": 0.8568193097329507, + "nauc_ndcg_at_3_std": 0.13252390138276315, + "nauc_ndcg_at_5_diff1": 0.9357442357672642, + "nauc_ndcg_at_5_max": 0.8565305420139139, + "nauc_ndcg_at_5_std": 0.2039909648363196, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.9738693467336647, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.7608263155229871, + "nauc_precision_at_1_diff1": 0.908130775986731, + "nauc_precision_at_1_max": 0.8111634573555125, + "nauc_precision_at_1_std": 0.061808904777442504, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.6938924997712752, + "nauc_precision_at_3_diff1": 0.9709659408151863, + "nauc_precision_at_3_max": 0.9238060989189518, + "nauc_precision_at_3_std": 0.208638963856334, + "nauc_precision_at_5_diff1": 0.9813352476669084, + "nauc_precision_at_5_max": 0.941685830281352, + "nauc_precision_at_5_std": 0.5420627096881087, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_10_diff1": 0.9738693467336678, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.7608263155229885, + "nauc_recall_at_1_diff1": 0.908130775986731, + "nauc_recall_at_1_max": 0.8111634573555125, + "nauc_recall_at_1_std": 0.061808904777442504, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.6938924997712791, + "nauc_recall_at_3_diff1": 0.9709659408151863, + "nauc_recall_at_3_max": 0.9238060989189517, + "nauc_recall_at_3_std": 0.20863896385634073, + "nauc_recall_at_5_diff1": 0.9813352476669055, + "nauc_recall_at_5_max": 0.9416858302813482, + "nauc_recall_at_5_std": 0.5420627096881058, + "ndcg_at_1": 0.91855, + "ndcg_at_10": 0.9496, + "ndcg_at_100": 0.95357, + "ndcg_at_1000": 0.95422, + "ndcg_at_20": 0.95198, + "ndcg_at_3": 0.94306, + "ndcg_at_5": 0.94656, + "precision_at_1": 0.91855, + "precision_at_10": 0.09774, + "precision_at_100": 0.00995, + "precision_at_1000": 0.001, + "precision_at_20": 0.04932, + "precision_at_3": 0.31976, + "precision_at_5": 0.19367, + "recall_at_1": 0.91855, + "recall_at_10": 0.97738, + "recall_at_100": 0.99548, + "recall_at_1000": 1.0, + "recall_at_20": 0.98643, + "recall_at_3": 0.95928, + "recall_at_5": 0.96833 + } + ] + }, + "task_name": "CodeTransOceanContest" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanDL.json b/results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanDL.json new file mode 100644 index 000000000..2dd38d058 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CodeTransOceanDL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "281562cb8a1265ab5c0824bfa6ddcd9b0a15618f", + "evaluation_time": 20.593432426452637, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "python-Code" + ], + "main_score": 0.38566, + "map_at_1": 0.08889, + "map_at_10": 0.24311, + "map_at_100": 0.25414, + "map_at_1000": 0.25414, + "map_at_20": 0.25414, + "map_at_3": 0.14352, + "map_at_5": 0.18296, + "mrr_at_1": 0.07777777777777778, + "mrr_at_10": 0.23692460317460354, + "mrr_at_100": 0.2494246031746036, + "mrr_at_1000": 0.2494246031746036, + "mrr_at_20": 0.2494246031746036, + "mrr_at_3": 0.13611111111111104, + "mrr_at_5": 0.1813888888888888, + "nauc_map_at_1000_diff1": -0.11995791794456051, + "nauc_map_at_1000_max": -0.374290224077524, + "nauc_map_at_1000_std": -0.08685310117692485, + "nauc_map_at_100_diff1": -0.11995791794456051, + "nauc_map_at_100_max": -0.374290224077524, + "nauc_map_at_100_std": -0.08685310117692485, + "nauc_map_at_10_diff1": -0.1290406962525938, + "nauc_map_at_10_max": -0.36469676015537444, + "nauc_map_at_10_std": -0.0818556060930571, + "nauc_map_at_1_diff1": -0.22632143815401132, + "nauc_map_at_1_max": -0.3805674805473572, + "nauc_map_at_1_std": -0.05114703514891332, + "nauc_map_at_20_diff1": -0.11995791794456051, + "nauc_map_at_20_max": -0.374290224077524, + "nauc_map_at_20_std": -0.08685310117692485, + "nauc_map_at_3_diff1": -0.09430569631049271, + "nauc_map_at_3_max": -0.40313061277731527, + "nauc_map_at_3_std": -0.09279256680145706, + "nauc_map_at_5_diff1": -0.1117196963640011, + "nauc_map_at_5_max": -0.3671021151035073, + "nauc_map_at_5_std": -0.10227972134518214, + "nauc_mrr_at_1000_diff1": 0.07811397652262392, + "nauc_mrr_at_1000_max": -0.35827281345154455, + "nauc_mrr_at_1000_std": -0.10625492656056047, + "nauc_mrr_at_100_diff1": 0.07811397652262392, + "nauc_mrr_at_100_max": -0.35827281345154455, + "nauc_mrr_at_100_std": -0.10625492656056047, + "nauc_mrr_at_10_diff1": 0.07045787347532516, + "nauc_mrr_at_10_max": -0.34091630838685005, + "nauc_mrr_at_10_std": -0.10015877703911025, + "nauc_mrr_at_1_diff1": 0.0672505653723791, + "nauc_mrr_at_1_max": -0.3905860707577908, + "nauc_mrr_at_1_std": -0.12350032580781183, + "nauc_mrr_at_20_diff1": 0.07811397652262392, + "nauc_mrr_at_20_max": -0.35827281345154455, + "nauc_mrr_at_20_std": -0.10625492656056047, + "nauc_mrr_at_3_diff1": 0.09229424174654557, + "nauc_mrr_at_3_max": -0.39823960095369026, + "nauc_mrr_at_3_std": -0.12365662926663419, + "nauc_mrr_at_5_diff1": 0.07455314880931567, + "nauc_mrr_at_5_max": -0.344606595687379, + "nauc_mrr_at_5_std": -0.09748438671046124, + "nauc_ndcg_at_1000_diff1": -0.11393187924360974, + "nauc_ndcg_at_1000_max": -0.37143825965734234, + "nauc_ndcg_at_1000_std": -0.08714465550011122, + "nauc_ndcg_at_100_diff1": -0.11393187924360974, + "nauc_ndcg_at_100_max": -0.37143825965734234, + "nauc_ndcg_at_100_std": -0.08714465550011122, + "nauc_ndcg_at_10_diff1": -0.14503050186220048, + "nauc_ndcg_at_10_max": -0.34183276513019023, + "nauc_ndcg_at_10_std": -0.06918081193926161, + "nauc_ndcg_at_1_diff1": -0.22632143815401132, + "nauc_ndcg_at_1_max": -0.3805674805473572, + "nauc_ndcg_at_1_std": -0.05114703514891332, + "nauc_ndcg_at_20_diff1": -0.11393187924360974, + "nauc_ndcg_at_20_max": -0.37143825965734234, + "nauc_ndcg_at_20_std": -0.08714465550011122, + "nauc_ndcg_at_3_diff1": -0.0628820826432392, + "nauc_ndcg_at_3_max": -0.4051569855081342, + "nauc_ndcg_at_3_std": -0.09763678652070176, + "nauc_ndcg_at_5_diff1": -0.09784255160012018, + "nauc_ndcg_at_5_max": -0.35547401094483777, + "nauc_ndcg_at_5_std": -0.11253198818827154, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": -0.2856933940584139, + "nauc_precision_at_10_max": -0.21317845652446532, + "nauc_precision_at_10_std": 0.01654619145113311, + "nauc_precision_at_1_diff1": -0.22632143815401132, + "nauc_precision_at_1_max": -0.3805674805473572, + "nauc_precision_at_1_std": -0.05114703514891332, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_3_diff1": 0.0021101534115564526, + "nauc_precision_at_3_max": -0.408638015094404, + "nauc_precision_at_3_std": -0.10657976464982523, + "nauc_precision_at_5_diff1": -0.07729717358965525, + "nauc_precision_at_5_max": -0.3327683373383679, + "nauc_precision_at_5_std": -0.13203741846893194, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": -0.28569339405841293, + "nauc_recall_at_10_max": -0.21317845652446415, + "nauc_recall_at_10_std": 0.016546191451134153, + "nauc_recall_at_1_diff1": -0.22632143815401132, + "nauc_recall_at_1_max": -0.3805674805473572, + "nauc_recall_at_1_std": -0.05114703514891332, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.002110153411556453, + "nauc_recall_at_3_max": -0.40863801509440384, + "nauc_recall_at_3_std": -0.1065797646498251, + "nauc_recall_at_5_diff1": -0.07729717358965577, + "nauc_recall_at_5_max": -0.33276833733836847, + "nauc_recall_at_5_std": -0.13203741846893252, + "ndcg_at_1": 0.08889, + "ndcg_at_10": 0.38566, + "ndcg_at_100": 0.42063, + "ndcg_at_1000": 0.42063, + "ndcg_at_20": 0.42063, + "ndcg_at_3": 0.16488, + "ndcg_at_5": 0.23589, + "precision_at_1": 0.08889, + "precision_at_10": 0.08722, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.07593, + "precision_at_5": 0.08, + "recall_at_1": 0.08889, + "recall_at_10": 0.87222, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.22778, + "recall_at_5": 0.4 + } + ] + }, + "task_name": "CodeTransOceanDL" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/CosQA.json b/results/voyageai__voyage-code-3/no_revision_available/CosQA.json new file mode 100644 index 000000000..39000db18 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "evaluation_time": 207.96753025054932, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ], + "main_score": 0.34452, + "map_at_1": 0.152, + "map_at_10": 0.26473, + "map_at_100": 0.2801, + "map_at_1000": 0.28038, + "map_at_20": 0.2748, + "map_at_3": 0.20833, + "map_at_5": 0.24153, + "mrr_at_1": 0.192, + "mrr_at_10": 0.28281111111111107, + "mrr_at_100": 0.298632158616367, + "mrr_at_1000": 0.29891199207757196, + "mrr_at_20": 0.2927876414692977, + "mrr_at_3": 0.23966666666666658, + "mrr_at_5": 0.25636666666666663, + "nauc_map_at_1000_diff1": 0.24101468460461478, + "nauc_map_at_1000_max": 0.10598410702361434, + "nauc_map_at_1000_std": -0.20144523237787224, + "nauc_map_at_100_diff1": 0.24053508672795085, + "nauc_map_at_100_max": 0.1062060173891184, + "nauc_map_at_100_std": -0.20123151261597685, + "nauc_map_at_10_diff1": 0.2403401073979636, + "nauc_map_at_10_max": 0.10204901836438325, + "nauc_map_at_10_std": -0.20723915647527605, + "nauc_map_at_1_diff1": 0.349707775098606, + "nauc_map_at_1_max": 0.08792695974322537, + "nauc_map_at_1_std": -0.17679606534340425, + "nauc_map_at_20_diff1": 0.23635390281247903, + "nauc_map_at_20_max": 0.1051091267854992, + "nauc_map_at_20_std": -0.20503982338903548, + "nauc_map_at_3_diff1": 0.27080780238699853, + "nauc_map_at_3_max": 0.08624902998628763, + "nauc_map_at_3_std": -0.2116290389423999, + "nauc_map_at_5_diff1": 0.24375227072550879, + "nauc_map_at_5_max": 0.10446847054213335, + "nauc_map_at_5_std": -0.20416673549725003, + "nauc_mrr_at_1000_diff1": 0.1978435935694305, + "nauc_mrr_at_1000_max": 0.06389276526733645, + "nauc_mrr_at_1000_std": -0.24870292026564658, + "nauc_mrr_at_100_diff1": 0.19741897116785875, + "nauc_mrr_at_100_max": 0.06399527259062092, + "nauc_mrr_at_100_std": -0.2486454789316169, + "nauc_mrr_at_10_diff1": 0.1982954896251409, + "nauc_mrr_at_10_max": 0.05841515376599231, + "nauc_mrr_at_10_std": -0.25585472957584926, + "nauc_mrr_at_1_diff1": 0.2261366365164493, + "nauc_mrr_at_1_max": 0.05888630966674663, + "nauc_mrr_at_1_std": -0.23551522185757365, + "nauc_mrr_at_20_diff1": 0.19610569618359852, + "nauc_mrr_at_20_max": 0.06388923736326994, + "nauc_mrr_at_20_std": -0.25022579250123667, + "nauc_mrr_at_3_diff1": 0.21689261589102052, + "nauc_mrr_at_3_max": 0.05433511684692397, + "nauc_mrr_at_3_std": -0.2522461843203615, + "nauc_mrr_at_5_diff1": 0.20769666984905863, + "nauc_mrr_at_5_max": 0.05103054542191035, + "nauc_mrr_at_5_std": -0.25629167967151373, + "nauc_ndcg_at_1000_diff1": 0.21724475676486285, + "nauc_ndcg_at_1000_max": 0.1192044108916809, + "nauc_ndcg_at_1000_std": -0.19560471367028226, + "nauc_ndcg_at_100_diff1": 0.20657799715418929, + "nauc_ndcg_at_100_max": 0.12772564966149327, + "nauc_ndcg_at_100_std": -0.18530409569738068, + "nauc_ndcg_at_10_diff1": 0.19715388368513592, + "nauc_ndcg_at_10_max": 0.11290056737017848, + "nauc_ndcg_at_10_std": -0.21489877133960453, + "nauc_ndcg_at_1_diff1": 0.349707775098606, + "nauc_ndcg_at_1_max": 0.08792695974322537, + "nauc_ndcg_at_1_std": -0.17679606534340425, + "nauc_ndcg_at_20_diff1": 0.18265150652871567, + "nauc_ndcg_at_20_max": 0.12420581957680346, + "nauc_ndcg_at_20_std": -0.20560776758982285, + "nauc_ndcg_at_3_diff1": 0.2486486238625192, + "nauc_ndcg_at_3_max": 0.0851016243931613, + "nauc_ndcg_at_3_std": -0.2238799975521682, + "nauc_ndcg_at_5_diff1": 0.2038709195941383, + "nauc_ndcg_at_5_max": 0.11692332914550611, + "nauc_ndcg_at_5_std": -0.21095128458420737, + "nauc_precision_at_1000_diff1": -0.02427637721760636, + "nauc_precision_at_1000_max": 0.9346405228758199, + "nauc_precision_at_1000_std": 0.5613912231559154, + "nauc_precision_at_100_diff1": -0.12075404874593668, + "nauc_precision_at_100_max": 0.4641166811552213, + "nauc_precision_at_100_std": 0.2143662062526193, + "nauc_precision_at_10_diff1": 0.07927965224591164, + "nauc_precision_at_10_max": 0.14499053516090232, + "nauc_precision_at_10_std": -0.23215405338109257, + "nauc_precision_at_1_diff1": 0.349707775098606, + "nauc_precision_at_1_max": 0.08792695974322537, + "nauc_precision_at_1_std": -0.17679606534340425, + "nauc_precision_at_20_diff1": -0.022220305852593377, + "nauc_precision_at_20_max": 0.20688168333908144, + "nauc_precision_at_20_std": -0.19112501847927896, + "nauc_precision_at_3_diff1": 0.19523274018213513, + "nauc_precision_at_3_max": 0.08217018367841955, + "nauc_precision_at_3_std": -0.2540158982161363, + "nauc_precision_at_5_diff1": 0.10681426032017446, + "nauc_precision_at_5_max": 0.14961711488638585, + "nauc_precision_at_5_std": -0.22588397199647622, + "nauc_recall_at_1000_diff1": -0.024276377217553626, + "nauc_recall_at_1000_max": 0.9346405228758147, + "nauc_recall_at_1000_std": 0.5613912231559195, + "nauc_recall_at_100_diff1": -0.12075404874593448, + "nauc_recall_at_100_max": 0.4641166811552219, + "nauc_recall_at_100_std": 0.21436620625261668, + "nauc_recall_at_10_diff1": 0.07927965224591155, + "nauc_recall_at_10_max": 0.14499053516090227, + "nauc_recall_at_10_std": -0.2321540533810927, + "nauc_recall_at_1_diff1": 0.349707775098606, + "nauc_recall_at_1_max": 0.08792695974322537, + "nauc_recall_at_1_std": -0.17679606534340425, + "nauc_recall_at_20_diff1": -0.022220305852592503, + "nauc_recall_at_20_max": 0.2068816833390826, + "nauc_recall_at_20_std": -0.1911250184792784, + "nauc_recall_at_3_diff1": 0.19523274018213502, + "nauc_recall_at_3_max": 0.08217018367841955, + "nauc_recall_at_3_std": -0.2540158982161363, + "nauc_recall_at_5_diff1": 0.10681426032017424, + "nauc_recall_at_5_max": 0.14961711488638552, + "nauc_recall_at_5_std": -0.2258839719964763, + "ndcg_at_1": 0.152, + "ndcg_at_10": 0.34452, + "ndcg_at_100": 0.41723, + "ndcg_at_1000": 0.42421, + "ndcg_at_20": 0.38002, + "ndcg_at_3": 0.22864, + "ndcg_at_5": 0.28905, + "precision_at_1": 0.152, + "precision_at_10": 0.0606, + "precision_at_100": 0.00942, + "precision_at_1000": 0.001, + "precision_at_20": 0.0372, + "precision_at_3": 0.096, + "precision_at_5": 0.0872, + "recall_at_1": 0.152, + "recall_at_10": 0.606, + "recall_at_100": 0.942, + "recall_at_1000": 0.996, + "recall_at_20": 0.744, + "recall_at_3": 0.288, + "recall_at_5": 0.436 + } + ] + }, + "task_name": "CosQA" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/StackOverflowQA.json b/results/voyageai__voyage-code-3/no_revision_available/StackOverflowQA.json new file mode 100644 index 000000000..97c2ca5f1 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "evaluation_time": 375.39803552627563, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.97168, + "map_at_1": 0.94985, + "map_at_10": 0.96533, + "map_at_100": 0.96568, + "map_at_1000": 0.96569, + "map_at_20": 0.96554, + "map_at_3": 0.96272, + "map_at_5": 0.9648, + "mrr_at_1": 0.9498495486459378, + "mrr_at_10": 0.9653325054528665, + "mrr_at_100": 0.9656849512537935, + "mrr_at_1000": 0.9656899560792449, + "mrr_at_20": 0.9655354401474339, + "mrr_at_3": 0.9627214978268137, + "mrr_at_5": 0.9648027415580073, + "nauc_map_at_1000_diff1": 0.9677010817392381, + "nauc_map_at_1000_max": 0.8992147554128406, + "nauc_map_at_1000_std": -0.11367860416596368, + "nauc_map_at_100_diff1": 0.967701266461378, + "nauc_map_at_100_max": 0.8992249288169608, + "nauc_map_at_100_std": -0.11365088267343505, + "nauc_map_at_10_diff1": 0.9680296303411313, + "nauc_map_at_10_max": 0.8997144795621024, + "nauc_map_at_10_std": -0.11225883175842885, + "nauc_map_at_1_diff1": 0.963357923474939, + "nauc_map_at_1_max": 0.8814831631499773, + "nauc_map_at_1_std": -0.11043584429943797, + "nauc_map_at_20_diff1": 0.9678413819715109, + "nauc_map_at_20_max": 0.8996621023207143, + "nauc_map_at_20_std": -0.11236908272386724, + "nauc_map_at_3_diff1": 0.9677341739930384, + "nauc_map_at_3_max": 0.901822674961521, + "nauc_map_at_3_std": -0.12405699503933929, + "nauc_map_at_5_diff1": 0.9679677069369804, + "nauc_map_at_5_max": 0.9004406323888772, + "nauc_map_at_5_std": -0.1109579820828818, + "nauc_mrr_at_1000_diff1": 0.9677010817392381, + "nauc_mrr_at_1000_max": 0.8992147554128406, + "nauc_mrr_at_1000_std": -0.11367860416596368, + "nauc_mrr_at_100_diff1": 0.967701266461378, + "nauc_mrr_at_100_max": 0.8992249288169608, + "nauc_mrr_at_100_std": -0.11365088267343505, + "nauc_mrr_at_10_diff1": 0.9680296303411313, + "nauc_mrr_at_10_max": 0.8997144795621024, + "nauc_mrr_at_10_std": -0.11225883175842885, + "nauc_mrr_at_1_diff1": 0.963357923474939, + "nauc_mrr_at_1_max": 0.8814831631499773, + "nauc_mrr_at_1_std": -0.11043584429943797, + "nauc_mrr_at_20_diff1": 0.9678413819715109, + "nauc_mrr_at_20_max": 0.8996621023207143, + "nauc_mrr_at_20_std": -0.11236908272386724, + "nauc_mrr_at_3_diff1": 0.9677341739930384, + "nauc_mrr_at_3_max": 0.901822674961521, + "nauc_mrr_at_3_std": -0.12405699503933929, + "nauc_mrr_at_5_diff1": 0.9679677069369804, + "nauc_mrr_at_5_max": 0.9004406323888772, + "nauc_mrr_at_5_std": -0.1109579820828818, + "nauc_ndcg_at_1000_diff1": 0.9683153135375732, + "nauc_ndcg_at_1000_max": 0.9017747818029089, + "nauc_ndcg_at_1000_std": -0.1109225768112506, + "nauc_ndcg_at_100_diff1": 0.9682500676553635, + "nauc_ndcg_at_100_max": 0.9021640577830734, + "nauc_ndcg_at_100_std": -0.11051187241816189, + "nauc_ndcg_at_10_diff1": 0.9701040984737974, + "nauc_ndcg_at_10_max": 0.9053590661634338, + "nauc_ndcg_at_10_std": -0.10181364845409424, + "nauc_ndcg_at_1_diff1": 0.963357923474939, + "nauc_ndcg_at_1_max": 0.8814831631499773, + "nauc_ndcg_at_1_std": -0.11043584429943797, + "nauc_ndcg_at_20_diff1": 0.9692876646822131, + "nauc_ndcg_at_20_max": 0.9053613648406619, + "nauc_ndcg_at_20_std": -0.10211015921117654, + "nauc_ndcg_at_3_diff1": 0.9692301963415002, + "nauc_ndcg_at_3_max": 0.9096341737957319, + "nauc_ndcg_at_3_std": -0.1239787122601762, + "nauc_ndcg_at_5_diff1": 0.9699223706029623, + "nauc_ndcg_at_5_max": 0.907297142361567, + "nauc_ndcg_at_5_std": -0.09789879773605441, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.9564367251079331, + "nauc_precision_at_100_max": 0.9564367251079331, + "nauc_precision_at_100_std": -0.0797371748154131, + "nauc_precision_at_10_diff1": 0.9927394541846477, + "nauc_precision_at_10_max": 0.9607116729984303, + "nauc_precision_at_10_std": 0.023843687107660768, + "nauc_precision_at_1_diff1": 0.963357923474939, + "nauc_precision_at_1_max": 0.8814831631499773, + "nauc_precision_at_1_std": -0.11043584429943797, + "nauc_precision_at_20_diff1": 0.9891091812769709, + "nauc_precision_at_20_max": 0.9891091812769709, + "nauc_precision_at_20_std": 0.08043758582511515, + "nauc_precision_at_3_diff1": 0.976238213695205, + "nauc_precision_at_3_max": 0.946795055966625, + "nauc_precision_at_3_std": -0.12047680267059419, + "nauc_precision_at_5_diff1": 0.9849204048450249, + "nauc_precision_at_5_max": 0.9577207938439504, + "nauc_precision_at_5_std": 0.015409353000901833, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9564367251078416, + "nauc_recall_at_100_max": 0.9564367251078416, + "nauc_recall_at_100_std": -0.07973717481540185, + "nauc_recall_at_10_diff1": 0.9927394541846406, + "nauc_recall_at_10_max": 0.9607116729984456, + "nauc_recall_at_10_std": 0.023843687107667485, + "nauc_recall_at_1_diff1": 0.963357923474939, + "nauc_recall_at_1_max": 0.8814831631499773, + "nauc_recall_at_1_std": -0.11043584429943797, + "nauc_recall_at_20_diff1": 0.9891091812769823, + "nauc_recall_at_20_max": 0.9891091812769823, + "nauc_recall_at_20_std": 0.08043758582515963, + "nauc_recall_at_3_diff1": 0.9762382136952051, + "nauc_recall_at_3_max": 0.9467950559666313, + "nauc_recall_at_3_std": -0.12047680267059348, + "nauc_recall_at_5_diff1": 0.9849204048450327, + "nauc_recall_at_5_max": 0.9577207938439567, + "nauc_recall_at_5_std": 0.0154093530009189, + "ndcg_at_1": 0.94985, + "ndcg_at_10": 0.97168, + "ndcg_at_100": 0.97333, + "ndcg_at_1000": 0.97351, + "ndcg_at_20": 0.97243, + "ndcg_at_3": 0.96665, + "ndcg_at_5": 0.97038, + "precision_at_1": 0.94985, + "precision_at_10": 0.0991, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "precision_at_20": 0.0497, + "precision_at_3": 0.32598, + "precision_at_5": 0.19739, + "recall_at_1": 0.94985, + "recall_at_10": 0.99097, + "recall_at_100": 0.9985, + "recall_at_1000": 1.0, + "recall_at_20": 0.99398, + "recall_at_3": 0.97793, + "recall_at_5": 0.98696 + } + ] + }, + "task_name": "StackOverflowQA" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/SyntheticText2SQL.json b/results/voyageai__voyage-code-3/no_revision_available/SyntheticText2SQL.json new file mode 100644 index 000000000..27cd17516 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "evaluation_time": 1028.11314535141, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ], + "main_score": 0.62871, + "map_at_1": 0.05931, + "map_at_10": 0.50172, + "map_at_100": 0.50221, + "map_at_1000": 0.50221, + "map_at_20": 0.50217, + "map_at_3": 0.49126, + "map_at_5": 0.49966, + "mrr_at_1": 0.4684669287301316, + "mrr_at_10": 0.7128829558371541, + "mrr_at_100": 0.7133501465192627, + "mrr_at_1000": 0.7133501465192627, + "mrr_at_20": 0.7133145566423184, + "mrr_at_3": 0.7051501167891556, + "mrr_at_5": 0.7110978180368052, + "nauc_map_at_1000_diff1": 0.16850595256299142, + "nauc_map_at_1000_max": 0.07209526996354072, + "nauc_map_at_1000_std": -0.31394151564475675, + "nauc_map_at_100_diff1": 0.16850595256299142, + "nauc_map_at_100_max": 0.07209526996354072, + "nauc_map_at_100_std": -0.31394151564475675, + "nauc_map_at_10_diff1": 0.16005030500086084, + "nauc_map_at_10_max": 0.07264507684893849, + "nauc_map_at_10_std": -0.31715470923775074, + "nauc_map_at_1_diff1": 0.8869496772604496, + "nauc_map_at_1_max": 0.02919830321081601, + "nauc_map_at_1_std": -0.123169881099131, + "nauc_map_at_20_diff1": 0.16782304707510037, + "nauc_map_at_20_max": 0.07245475061050963, + "nauc_map_at_20_std": -0.31399548485799106, + "nauc_map_at_3_diff1": 0.007297119245486146, + "nauc_map_at_3_max": 0.08421968652977062, + "nauc_map_at_3_std": -0.36375361795432126, + "nauc_map_at_5_diff1": 0.12348535589464996, + "nauc_map_at_5_max": 0.0767524835028725, + "nauc_map_at_5_std": -0.3303849075726066, + "nauc_mrr_at_1000_diff1": -0.19145171066076583, + "nauc_mrr_at_1000_max": 0.03456240689954698, + "nauc_mrr_at_1000_std": -0.16911698657686483, + "nauc_mrr_at_100_diff1": -0.19145171066076583, + "nauc_mrr_at_100_max": 0.03456240689954698, + "nauc_mrr_at_100_std": -0.16911698657686483, + "nauc_mrr_at_10_diff1": -0.19383555832224394, + "nauc_mrr_at_10_max": 0.034956712263663245, + "nauc_mrr_at_10_std": -0.1708872485212556, + "nauc_mrr_at_1_diff1": -0.0724868950687162, + "nauc_mrr_at_1_max": 0.016279165510593368, + "nauc_mrr_at_1_std": -0.1076401320944665, + "nauc_mrr_at_20_diff1": -0.19166540939871457, + "nauc_mrr_at_20_max": 0.03471598334930003, + "nauc_mrr_at_20_std": -0.16916595467621381, + "nauc_mrr_at_3_diff1": -0.23156601373830685, + "nauc_mrr_at_3_max": 0.040412426368698455, + "nauc_mrr_at_3_std": -0.1905754313395974, + "nauc_mrr_at_5_diff1": -0.20312683178010765, + "nauc_mrr_at_5_max": 0.03617044817759521, + "nauc_mrr_at_5_std": -0.17611736247571125, + "nauc_ndcg_at_1000_diff1": 0.12200580013571008, + "nauc_ndcg_at_1000_max": 0.0757281364868011, + "nauc_ndcg_at_1000_std": -0.3277358441042764, + "nauc_ndcg_at_100_diff1": 0.12200580013571008, + "nauc_ndcg_at_100_max": 0.0757281364868011, + "nauc_ndcg_at_100_std": -0.3277358441042764, + "nauc_ndcg_at_10_diff1": 0.08346563719126793, + "nauc_ndcg_at_10_max": 0.07984058762338721, + "nauc_ndcg_at_10_std": -0.3420079429928775, + "nauc_ndcg_at_1_diff1": 0.8869496772604496, + "nauc_ndcg_at_1_max": 0.02919830321081601, + "nauc_ndcg_at_1_std": -0.123169881099131, + "nauc_ndcg_at_20_diff1": 0.11618780757172203, + "nauc_ndcg_at_20_max": 0.07879670436948671, + "nauc_ndcg_at_20_std": -0.3283249761509627, + "nauc_ndcg_at_3_diff1": -0.21635858761258073, + "nauc_ndcg_at_3_max": 0.10142935236930697, + "nauc_ndcg_at_3_std": -0.4348822107044396, + "nauc_ndcg_at_5_diff1": -0.014973636275297837, + "nauc_ndcg_at_5_max": 0.08981298991414444, + "nauc_ndcg_at_5_std": -0.37874764070991734, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": -1.2158511191301635, + "nauc_precision_at_10_max": 0.25248667404560426, + "nauc_precision_at_10_std": -0.8122406477400408, + "nauc_precision_at_1_diff1": 0.8869496772604496, + "nauc_precision_at_1_max": 0.02919830321081601, + "nauc_precision_at_1_std": -0.123169881099131, + "nauc_precision_at_20_diff1": -1.4040859643368495, + "nauc_precision_at_20_max": 0.8746836627148438, + "nauc_precision_at_20_std": -0.5027346081469745, + "nauc_precision_at_3_diff1": -1.1653772258202089, + "nauc_precision_at_3_max": 0.1767074016899205, + "nauc_precision_at_3_std": -0.7436661207778328, + "nauc_precision_at_5_diff1": -1.2386337036708948, + "nauc_precision_at_5_max": 0.2190906977698616, + "nauc_precision_at_5_std": -0.8369114367867372, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": -1.2158511191302261, + "nauc_recall_at_10_max": 0.2524866740455998, + "nauc_recall_at_10_std": -0.8122406477400794, + "nauc_recall_at_1_diff1": 0.8869496772604496, + "nauc_recall_at_1_max": 0.02919830321081601, + "nauc_recall_at_1_std": -0.123169881099131, + "nauc_recall_at_20_diff1": -1.4040859643373693, + "nauc_recall_at_20_max": 0.8746836627148096, + "nauc_recall_at_20_std": -0.5027346081473573, + "nauc_recall_at_3_diff1": -1.1653772258202, + "nauc_recall_at_3_max": 0.17670740168992516, + "nauc_recall_at_3_std": -0.7436661207778251, + "nauc_recall_at_5_diff1": -1.238633703670898, + "nauc_recall_at_5_max": 0.21909069776986764, + "nauc_recall_at_5_std": -0.8369114367867464, + "ndcg_at_1": 0.05931, + "ndcg_at_10": 0.62871, + "ndcg_at_100": 0.6305, + "ndcg_at_1000": 0.6305, + "ndcg_at_20": 0.63027, + "ndcg_at_3": 0.6089, + "ndcg_at_5": 0.62382, + "precision_at_1": 0.05931, + "precision_at_10": 0.09928, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04994, + "precision_at_3": 0.31408, + "precision_at_5": 0.19559, + "recall_at_1": 0.05931, + "recall_at_10": 0.99282, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.9988, + "recall_at_3": 0.94223, + "recall_at_5": 0.97795 + } + ] + }, + "task_name": "SyntheticText2SQL" +} \ No newline at end of file diff --git a/results/voyageai__voyage-code-3/no_revision_available/model_meta.json b/results/voyageai__voyage-code-3/no_revision_available/model_meta.json new file mode 100644 index 000000000..102dab5c8 --- /dev/null +++ b/results/voyageai__voyage-code-3/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "voyageai/voyage-code-3", "revision": "no_revision_available", "release_date": "2024-12-02", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 32000, "embed_dim": 1024, "license": null, "open_source": false, "similarity_fn_name": null, "framework": [], "loader": "VoyageWrapper"} \ No newline at end of file