From 9b4b8e7a0311879725dd15014701ce3b57ef6112 Mon Sep 17 00:00:00 2001 From: sys_vdms Date: Tue, 24 Sep 2024 21:00:37 +0000 Subject: [PATCH] Automated updates: Format and/or coverage --- .../coverage/cpp.develop.coverage_report.txt | 10 +-- .../coverage/cpp.develop.coverage_value.txt | 2 +- client/cpp/DescriptorSetQueryParser.h | 2 +- src/vcl/FaissDescriptorSet.cc | 78 ++++++++++--------- src/vcl/FaissDescriptorSet.h | 7 +- tests/unit_tests/DescriptorSetAdd_test.cc | 38 +++++---- 6 files changed, 69 insertions(+), 68 deletions(-) diff --git a/.github/coverage/cpp.develop.coverage_report.txt b/.github/coverage/cpp.develop.coverage_report.txt index c60a3f56..15c2a91f 100644 --- a/.github/coverage/cpp.develop.coverage_report.txt +++ b/.github/coverage/cpp.develop.coverage_report.txt @@ -11,7 +11,7 @@ src/BackendNeo4j.cc 121 0 0% 4,6-17,20,24,2 src/BlobCommand.cc 87 66 75% 76,130-132,136-139,145,147,165,186-189,192-196,202 src/BoundingBoxCommand.cc 180 4 2% 45,49,51,53-54,56-59,62,64-67,70-73,76,83,87,90-91,93-97,101,103,105,114,118,122-123,125-132,137-138,140-144,147-150,152,154-160,162-165,167-169,171-173,176-177,179-181,183-184,186-187,190,193,196-197,199,201-204,206-210,213,215-219,222-223,225-227,229-237,240-244,246,251-256,259-261,263,265-266,268,270,272-274,276-277,281-283,286,288,292-294,296,298,300-303,307-308,310-313,316-319,321-326,329-330,335,338-339,341 src/CommunicationManager.cc 46 0 0% 42-43,46-47,49-50,52-54,57,61-66,68-71,73-81,84,86-88,93,96-97,100-101,105,107-108,110,113-116 -src/DescriptorsCommand.cc 668 107 16% 56,63-68,73,75-79,81-85,87,89,92-93,96-98,101,103-104,107-113,115,118,121,168-170,174,188-192,232-243,253,267-269,273,288-295,297,309-312,317,322-326,343,351,353-356,359,362-363,365-369,372,375,377-378,381-383,385-386,388-389,392-393,395-397,403,408-409,411,413-414,417,419-422,425-430,436-437,440,442-443,445,447-448,451,454,456,458,461,463-468,470-474,476,479,482-483,486-487,498,501,506,512-513,518-519,521-524,527,533-534,539-540,543-549,552,554-555,557,559-560,562-563,567,569-574,577-582,585,588-593,595,621-622,625-626,629,632,639-640,642-644,646,649-650,652,655,658,660-662,670,674,676,679,681-684,687-692,694-698,700,703,705,707,710,713-714,716,718,720-725,728,730-732,735-736,738,740,742-743,745,747,749-751,753,755-760,765-766,769,771,773,780-781,784,788,790,793,795-798,801-805,807-811,813,815-818,820-822,825,829-838,843-844,847,853,855,858-859,862,866,871,873,876,879-880,884,889,891,893,896,899-900,902-905,909,913-914,916,918-920,922,924-925,927,929-931,933-935,940-942,945-946,948,951-955,959,962,966,968-969,971-972,974,976-977,979-981,983,988,990-991,993-995,997-998,1002,1004-1007,1009-1012,1017,1020-1022,1024,1026,1028-1031,1033-1037,1040-1041,1044,1046,1048,1050-1054,1057-1058,1061-1064,1066,1068-1075,1080,1082,1084-1085,1088-1091,1093,1095,1097-1104,1108-1113,1119,1122,1124,1126-1129,1132,1134,1137-1140,1142-1146,1152,1154,1156-1159,1164,1166,1168-1169,1172-1174,1176,1178,1180-1183,1186-1187,1189-1190,1192,1194-1195,1197-1203,1207-1208,1212-1213,1215-1216,1224-1225,1228-1229,1231,1233-1240,1244,1248-1249,1253-1257,1262-1263,1265 +src/DescriptorsCommand.cc 670 107 16% 56,63-68,73,75-79,81-85,87,89,92-93,96-98,101,103-104,107-113,115,118,121,168-170,174,188-192,232-243,253,267-269,273,288-297,299,311-314,319,324-328,345,353,355-358,361,364-365,367-371,374,377,379-380,383-385,387-388,390-391,394-395,397-399,405,410-411,413,415-416,419,421-424,427-432,438-439,442,444-445,447,449-450,453,456,458,460,463,465-470,472-476,478,481,484-485,488-489,500,503,508,514-515,520-521,523-526,529,535-536,541-542,545-551,554,556-557,559,561-562,564-565,569,571-576,579-584,587,590-595,597,623-624,627-628,631,634,641-642,644-646,648,651-652,654,657,660,662-664,672,676,678,681,683-686,689-694,696-700,702,705,707,709,712,715-716,718,720,722-727,730,732-734,737-738,740,742,744-745,747,749,751-753,755,757-762,767-768,771,773,775,782-783,786,790,792,795,797-800,803-807,809-813,815,817-820,822-824,827,831-840,845-846,849,855,857,860-861,864,868,873,875,878,881-882,886,891,893,895,898,901-902,904-907,911,915-916,918,920-922,924,926-927,929,931-933,935-937,942-944,947-948,950,953-957,961,964,968,970-971,973-974,976,978-979,981-983,985,990,992-993,995-997,999-1000,1004,1006-1009,1011-1014,1019,1022-1024,1026,1028,1030-1033,1035-1039,1042-1043,1046,1048,1050,1052-1056,1059-1060,1063-1066,1068,1070-1077,1082,1084,1086-1087,1090-1093,1095,1097,1099-1106,1110-1115,1121,1124,1126,1128-1131,1134,1136,1139-1142,1144-1148,1154,1156,1158-1161,1166,1168,1170-1171,1174-1176,1178,1180,1182-1185,1188-1189,1191-1192,1194,1196-1197,1199-1205,1209-1210,1214-1215,1217-1218,1226-1227,1230-1231,1233,1235-1242,1246,1250-1251,1255-1259,1264-1265,1267 src/DescriptorsManager.cc 24 19 79% 49-50,57-58,73 src/ExceptionsCommand.cc 6 0 0% 35-40 src/ImageCommand.cc 322 157 48% 55,59,63,65,67-69,71,73-76,78,81,86,88-89,97,99,106,109,111-112,114-115,117-118,120-121,124,151,162-163,174-175,177,182-185,195-196,198,203-206,221-229,231-233,246-247,257-267,269-270,272-273,278,286,297,304,308,311,313,315,337,339-340,343-348,350,352,374-376,379-381,385-388,394,396,403-406,420,427,433-436,440-441,452-455,458-463,468-470,481-484,489-493,498-499,501-502,504-508,511,513-517,520-523,526-527,530,532,537 @@ -31,10 +31,10 @@ src/RSCommand.cc 144 105 72% 65-67,73-74,98 src/SearchExpression.cc 99 38 38% 59,132-133,135,137-139,143,146,148-153,157,160,168-170,177,180-181,183-185,188,192-195,197,201,217-222,224-225,227,235-240,243,247-249,252-256,263,276,284-285 src/Server.cc 138 0 0% 57-58,60,64,68-70,72-73,77-78,80,85,88,90,92,95,97-98,103-106,108-109,112,116,118,122,125,128,131,133-134,136-138,140,142,145-152,154-155,159,162-167,170,172-173,176-177,181,183,185-186,188-190,192,194,197-199,203-206,208-209,212-216,218-219,221,223,225,228,231-232,236-237,239-240,242,246-250,253,255-257,260-263,265-268,272-273,276,280-281,284-287,292,294-301,304-309 src/vcl/CustomVCL.cc 51 22 43% 55,57-58,60-63,66,69-70,72,74,76-78,82-83,89-93,95,98-99,102,104-105,110 -src/vcl/DescriptorSet.cc 205 150 73% 64-65,67-68,89-90,111-112,129,131,133,186-189,192,217-218,220-221,224-227,236-240,252,264,315-316,319,321-324,327,341-342,344-346,350-352,354,361-363,365-366,369-370 +src/vcl/DescriptorSet.cc 209 155 74% 65,69-70,93-94,115-116,133,135,137,190-193,196,221-222,224-225,228-231,240-244,256,268,319-320,323,325-328,331,345-346,348-350,354-356,358,365-367,369-370,373-374 src/vcl/DescriptorSetData.cc 55 47 85% 48,58,64,67,114,116-118 src/vcl/Exception.cc 7 6 85% 38 -src/vcl/FaissDescriptorSet.cc 182 157 86% 83,115-116,132,167,187-188,204-205,224-225,238-239,245,258-259,261,272-273,279,299-300,302-303,305 +src/vcl/FaissDescriptorSet.cc 206 177 85% 83,115-116,132,167,187-188,204-205,224-225,238-239,245,258-259,261,272-273,279,303-304,306-307,309,372-373,379,398 src/vcl/FlinngDescriptorSet.cc 149 109 73% 60-66,89,109-111,113-114,118-121,124,126,128,130,132,134-137,140-141,143-144,170-171,176-177,182,206,208,228,248,279 src/vcl/Image.cc 910 689 75% 62,73-74,76-78,81-84,86,92,101,122-123,125,132-133,135,147,165,170,193,196-199,223,246,249-252,264,273,276-279,291,323,326-329,341,347,349-352,360-362,369,393-396,415,417,425,427,432,436,441,445,459,462,467-468,471-472,474,490,500,513,531,553-556,594,605-606,608,615,619,624,627-630,658-660,712,757-758,809,838-842,844-850,852,854-855,896,899-900,939-940,944-945,966,985-986,988,1028-1030,1032-1036,1038-1042,1044-1048,1050-1054,1056-1060,1062-1065,1088,1109,1128-1136,1147-1148,1167-1186,1198-1199,1207,1218,1220-1222,1224-1226,1228,1242,1246-1247,1249,1254-1255,1257,1278,1282,1285,1292,1307,1313,1322,1336,1361,1379,1462,1481 src/vcl/KeyFrame.cc 303 244 80% 58,62,86,90,95,97,102,105-107,109-111,113,119,139,148,154,172,186,190,216,220,224,235,239,249,255,274,284,288,307,315,341,345,347,359,367,369,394,396,405,430,442,449,465,469,478,483,495,500,507,514,518,525,541,547,557,563 @@ -51,11 +51,11 @@ src/VDMSConfig.cc 181 168 92% 119-121,196,19 src/VideoCommand.cc 474 117 24% 50,53-54,56-58,60,62,65-66,68-69,72,74-76,78-80,82,84-87,89-90,92-93,95,97-99,102,109,111,116,121-124,130,132,158-161,167-168,170,181,184,201,213,217-220,227-229,231-233,239,241-247,249-250,253-255,257-259,261-262,264,266-278,280-282,284-285,296,300,325,329,331,333,335,337,340-341,343,346,350,352,357-358,380-381,383-384,387-392,394,396,398-399,405,407,429-431,436,442-445,449-454,456-463,467-473,475,480-485,488,490-491,494-496,504,509,527-532,535-539,555,558,560-562,565-567,569-570,572-576,579-580,583-585,587,589-591,594-597,601-606,611-612,614-615,617-621,624-626,628,630-632,634-637,640-641,644,646,651,664,666-673,677,680,683,688-689,691-695,698-699,701,703,705,708,712,714,716-719,721-723,726,728,730,732-733,735-736,740,745,748-749,751-753,755,757,759-761,763-764,767-769,773-776,780-786,790-794,798,801,803,805,807,809-813,817-821,824-825,827-830,833-836,841-842,846-851,855-856,859-860 src/VideoLoop.cc 249 200 80% 33,81,98-101,103-109,180,188,197,201,207,211,217,220,290,312,315,324-325,327,331-332,334-335,339-342,344,346-349,351-354,356-357,359,361,370 utils/src/comm/ConnClient.cc 69 57 82% 49,55,59-60,98,103,108,114,120,127,130,149 -utils/src/comm/Connection.cc 82 61 74% 48-53,75,77-79,84,86,97,111,135,140,153,157,159,168,172 +utils/src/comm/Connection.cc 82 62 75% 48-53,75,77-79,84,86,97,111,135,140,153,157,159,168 utils/src/comm/ConnServer.cc 61 49 80% 60,64,68,75,84,91,103,108,128,135,140,145 utils/src/comm/Exception.cc 6 0 0% 35-40 utils/src/stats/SystemStats.cc 250 249 99% 453 utils/src/timers/TimerMap.cc 82 75 91% 126,151,153,155-158 ------------------------------------------------------------------------------ -TOTAL 10142 6488 64% +TOTAL 10172 6514 64% ------------------------------------------------------------------------------ diff --git a/.github/coverage/cpp.develop.coverage_value.txt b/.github/coverage/cpp.develop.coverage_value.txt index 50b05238..e47ec90e 100644 --- a/.github/coverage/cpp.develop.coverage_value.txt +++ b/.github/coverage/cpp.develop.coverage_value.txt @@ -1 +1 @@ -63.9716 +64.0385 diff --git a/client/cpp/DescriptorSetQueryParser.h b/client/cpp/DescriptorSetQueryParser.h index 89fd4629..dbe2b554 100644 --- a/client/cpp/DescriptorSetQueryParser.h +++ b/client/cpp/DescriptorSetQueryParser.h @@ -51,5 +51,5 @@ bool VDMS::DescriptorSetQueryParser::isValidMetric(string &metric) { bool VDMS::DescriptorSetQueryParser::isValidEngine(string &engine) { return (engine == "TileDBDense" || engine == "TileDBSparse" || engine == "FaissFlat" || engine == "FaissIVFFlat" || - engine == "Flinng" || engine =="FaissHNSWFlat"); + engine == "Flinng" || engine == "FaissHNSWFlat"); } diff --git a/src/vcl/FaissDescriptorSet.cc b/src/vcl/FaissDescriptorSet.cc index 78fd7eb8..4adb5e52 100644 --- a/src/vcl/FaissDescriptorSet.cc +++ b/src/vcl/FaissDescriptorSet.cc @@ -289,12 +289,12 @@ FaissIVFFlatDescriptorSet::FaissIVFFlatDescriptorSet( // TODO: Revise nlist param for future optimizations. // 4 is a suggested value by faiss for the IVFFlat index, // that's why we leave it for now. - //int nlist = 4; + // int nlist = 4; + + // default value of 4 is too low for any sizeable dataset - // default value of 4 is too low for any sizeable dataset - int nlist = 16; - + if (metric == L2) { faiss::IndexFlatL2 *quantizer = new faiss::IndexFlatL2(_dimensions); @@ -344,24 +344,24 @@ long FaissIVFFlatDescriptorSet::add(float *descriptors, unsigned n, return id_first; } - // FaissHNSWFlat -//Note: -//setting value of hnsw m= 48 - // M is number of connections each vertex will have - //i.e. number of nearest neighbors that each vertex will connect to. - // Total memory usage is (dim * 4 + M * 2 * 4) bytes per vector. - - // Rule of thumb to support 1M entries - // according to https://github.com/facebookresearch/faiss/wiki/Indexing-1M-vectors - // HNSW_M= 48 - // efConstruction = 2 x m - // efSearch = 32 - // Default values from hnsw.h - //int efConstruction = 40; - //int efSearch = 16; - //efsearch will be defined in the search params when IndexHNSW::search() is called (can dynamically change during runtime) - //efconstruction is a compile time paramerter will be defined in IndexHNSW.hnsw struct +// Note: +// setting value of hnsw m= 48 +// M is number of connections each vertex will have +// i.e. number of nearest neighbors that each vertex will connect to. +// Total memory usage is (dim * 4 + M * 2 * 4) bytes per vector. + +// Rule of thumb to support 1M entries +// according to +// https://github.com/facebookresearch/faiss/wiki/Indexing-1M-vectors HNSW_M= 48 +// efConstruction = 2 x m +// efSearch = 32 +// Default values from hnsw.h +// int efConstruction = 40; +// int efSearch = 16; +// efsearch will be defined in the search params when IndexHNSW::search() is +// called (can dynamically change during runtime) efconstruction is a compile +// time paramerter will be defined in IndexHNSW.hnsw struct FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet( const std::string &set_path) @@ -380,33 +380,35 @@ FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet( } _dimensions = _index->d; - _n_total = _index->ntotal; - + _n_total = _index->ntotal; } FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet( const std::string &set_path, unsigned dim, DistanceMetric metric) : FaissDescriptorSet(set_path, dim) { - int hnsw_M= 48; + int hnsw_M = 48; if (metric == L2) { - _index = new faiss::IndexHNSWFlat(dim, hnsw_M, faiss::METRIC_L2); - ((faiss::IndexHNSWFlat*)_index)->hnsw.efConstruction = 96; - } else { - //only metric L2 is supported for HNSWFLAT for FAISS v1.7.4 - //newer version of Faiss e.g. V1.8.0 supports I.P. metric for HNSW + _index = new faiss::IndexHNSWFlat(dim, hnsw_M, faiss::METRIC_L2); + ((faiss::IndexHNSWFlat *)_index)->hnsw.efConstruction = 96; + } else { + // only metric L2 is supported for HNSWFLAT for FAISS v1.7.4 + // newer version of Faiss e.g. V1.8.0 supports I.P. metric for HNSW throw VCLException(UnsupportedIndex, "Metric Not implemented"); } - } -void FaissHNSWFlatDescriptorSet::search(float *query, unsigned n_queries, unsigned k, long *descriptors, float *distances) { - ((faiss::IndexHNSWFlat*)_index)->hnsw.efSearch = 64; - //set according to https://github.com/facebookresearch/faiss/wiki/Indexing-1M-vectors for R@1 accuracy of 0.9779 - // The higher the value the slower the search is but better accuracy - //efsearch is a runtime parameter. - // ToDO - VDMS should expose an API to set runtime parameters to users of the different indices - +void FaissHNSWFlatDescriptorSet::search(float *query, unsigned n_queries, + unsigned k, long *descriptors, + float *distances) { + ((faiss::IndexHNSWFlat *)_index)->hnsw.efSearch = 64; + // set according to + // https://github.com/facebookresearch/faiss/wiki/Indexing-1M-vectors for R@1 + // accuracy of 0.9779 + // The higher the value the slower the search is but better accuracy + // efsearch is a runtime parameter. + // ToDO - VDMS should expose an API to set runtime parameters to users of the + // different indices + _index->search(n_queries, query, k, distances, descriptors); } - diff --git a/src/vcl/FaissDescriptorSet.h b/src/vcl/FaissDescriptorSet.h index c63526ac..d484969e 100644 --- a/src/vcl/FaissDescriptorSet.h +++ b/src/vcl/FaissDescriptorSet.h @@ -44,8 +44,8 @@ #include "DescriptorSetData.h" #include -#include #include +#include namespace VCL { @@ -116,9 +116,10 @@ class FaissHNSWFlatDescriptorSet : public FaissDescriptorSet { public: FaissHNSWFlatDescriptorSet(const std::string &set_path); FaissHNSWFlatDescriptorSet(const std::string &set_path, unsigned dim, - DistanceMetric metric); + DistanceMetric metric); - void search(float *query, unsigned n_queries, unsigned k, long *descriptors, float *distances); + void search(float *query, unsigned n_queries, unsigned k, long *descriptors, + float *distances); }; }; // namespace VCL diff --git a/tests/unit_tests/DescriptorSetAdd_test.cc b/tests/unit_tests/DescriptorSetAdd_test.cc index 69cdd11d..a90ebac5 100644 --- a/tests/unit_tests/DescriptorSetAdd_test.cc +++ b/tests/unit_tests/DescriptorSetAdd_test.cc @@ -218,16 +218,17 @@ TEST(Descriptors_Add, add_flatl2_100d_2add) { delete[] xb; } -//HNSW Tests +// HNSW Tests TEST(Descriptors_Add, add_hnswflatl2_100d) { - - //test to add 1K descriptors of 100D each - // descriptors are created by incrementing a random initial value as follows - // init init ... init (D times) - // init+1 init+1 ... init+1 (D times) - // ... - // init+nb-1 init+nb-1 ... init+nb-1 (d times) - // hence, nearest neigbor of any query descriptor are the IDs that is around the query ID + + // test to add 1K descriptors of 100D each + // descriptors are created by incrementing a random initial value as follows + // init init ... init (D times) + // init+1 init+1 ... init+1 (D times) + // ... + // init+nb-1 init+nb-1 ... init+nb-1 (d times) + // hence, nearest neigbor of any query descriptor are the IDs that is around + // the query ID int d = 100; int nb = 10000; @@ -266,9 +267,9 @@ TEST(Descriptors_Add, add_hnswflatl2_100d) { TEST(Descriptors_Add, add_recons_hnswflatl2_100d) { - //test to add 1K descriptors of 100D each - // Same as last previous test case but addes classes as labels - // classes will be searched and checked of nearest neighbors + // test to add 1K descriptors of 100D each + // Same as last previous test case but addes classes as labels + // classes will be searched and checked of nearest neighbors int d = 100; int nb = 10000; @@ -307,11 +308,11 @@ TEST(Descriptors_Add, add_recons_hnswflatl2_100d) { } TEST(Descriptors_Add, add_hnswflatl2_100d_2add) { - //test to add 2K descriptors of 100D each - // this is done in 2 steps - // first 1K and then the index is stored to a file - // second 1K are added after the index is read from a file - // the test case is to test file i/o of the index + // test to add 2K descriptors of 100D each + // this is done in 2 steps + // first 1K and then the index is stored to a file + // second 1K are added after the index is read from a file + // the test case is to test file i/o of the index int d = 100; int nb = 10000; @@ -342,9 +343,6 @@ TEST(Descriptors_Add, add_hnswflatl2_100d_2add) { delete[] xb; } - - - // Flinng Tests TEST(Descriptors_Add, add_flinngIP_100d) {