merge

IntelLabs · Sep 25, 2024 · bd2e9c6 · bd2e9c6
2 parents c59eab9 + b6ea57f
commit bd2e9c6
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 60 deletions.
diff --git a/.github/coverage/cpp.develop.coverage_report.txt b/.github/coverage/cpp.develop.coverage_report.txt
@@ -11,7 +11,7 @@ src/BackendNeo4j.cc                          121       0     0%   4,6-17,20,24,2
 src/BlobCommand.cc                            87      66    75%   76,130-132,136-139,145,147,165,186-189,192-196,202
 src/BoundingBoxCommand.cc                    180       4     2%   45,49,51,53-54,56-59,62,64-67,70-73,76,83,87,90-91,93-97,101,103,105,114,118,122-123,125-132,137-138,140-144,147-150,152,154-160,162-165,167-169,171-173,176-177,179-181,183-184,186-187,190,193,196-197,199,201-204,206-210,213,215-219,222-223,225-227,229-237,240-244,246,251-256,259-261,263,265-266,268,270,272-274,276-277,281-283,286,288,292-294,296,298,300-303,307-308,310-313,316-319,321-326,329-330,335,338-339,341
 src/CommunicationManager.cc                   46       0     0%   42-43,46-47,49-50,52-54,57,61-66,68-71,73-81,84,86-88,93,96-97,100-101,105,107-108,110,113-116
-src/DescriptorsCommand.cc                    668     107    16%   56,63-68,73,75-79,81-85,87,89,92-93,96-98,101,103-104,107-113,115,118,121,168-170,174,188-192,232-243,253,267-269,273,288-295,297,309-312,317,322-326,343,351,353-356,359,362-363,365-369,372,375,377-378,381-383,385-386,388-389,392-393,395-397,403,408-409,411,413-414,417,419-422,425-430,436-437,440,442-443,445,447-448,451,454,456,458,461,463-468,470-474,476,479,482-483,486-487,498,501,506,512-513,518-519,521-524,527,533-534,539-540,543-549,552,554-555,557,559-560,562-563,567,569-574,577-582,585,588-593,595,621-622,625-626,629,632,639-640,642-644,646,649-650,652,655,658,660-662,670,674,676,679,681-684,687-692,694-698,700,703,705,707,710,713-714,716,718,720-725,728,730-732,735-736,738,740,742-743,745,747,749-751,753,755-760,765-766,769,771,773,780-781,784,788,790,793,795-798,801-805,807-811,813,815-818,820-822,825,829-838,843-844,847,853,855,858-859,862,866,871,873,876,879-880,884,889,891,893,896,899-900,902-905,909,913-914,916,918-920,922,924-925,927,929-931,933-935,940-942,945-946,948,951-955,959,962,966,968-969,971-972,974,976-977,979-981,983,988,990-991,993-995,997-998,1002,1004-1007,1009-1012,1017,1020-1022,1024,1026,1028-1031,1033-1037,1040-1041,1044,1046,1048,1050-1054,1057-1058,1061-1064,1066,1068-1075,1080,1082,1084-1085,1088-1091,1093,1095,1097-1104,1108-1113,1119,1122,1124,1126-1129,1132,1134,1137-1140,1142-1146,1152,1154,1156-1159,1164,1166,1168-1169,1172-1174,1176,1178,1180-1183,1186-1187,1189-1190,1192,1194-1195,1197-1203,1207-1208,1212-1213,1215-1216,1224-1225,1228-1229,1231,1233-1240,1244,1248-1249,1253-1257,1262-1263,1265
+src/DescriptorsCommand.cc                    683     111    16%   56,63-68,73,75-79,81-85,87,89,92-93,96-98,101,103-104,107-113,115,118,121,168-170,174,188-192,243-254,264,273-276,290-292,296,311-318,320,332-335,340,346-350,367,375,377-380,383,386-387,389-393,396,399,401-402,405-407,409-410,412-413,416-417,419-421,427,432-433,435,437-438,441,443-446,449-454,460-461,464,466-467,469,471-472,475,479-480,482,484,487,489-494,496-500,502,505,508-509,512-513,524,527,532,538-539,544-545,547-550,553,559-560,565-566,569-575,578,580-581,583,585-586,588-589,593,595-600,603-608,611,614-617,620-621,623,625,628-629,632-633,636,639,646-647,649-651,653,656-657,659,662,665,667-669,677,681,683,686,688-691,694-699,701-705,707,710,712,714,717,720-721,723,725,727-732,735,737-739,742-743,745,747,749-750,752,754,756-758,760,762-767,772-773,776,778,780,787-788,791,795,797,800,802,804-807,810-814,816-820,822,824-827,829-831,834,838-847,852-853,856,862,864,867-868,871,875,880,882,885,888-889,893,898,900,902,905,908-909,911-914,918,922-923,925,927-929,931,933-934,936,938-940,942-944,949-951,954-955,957,960-964,968,971,978-981,983-984,986,988-989,991-993,995,1000,1003-1004,1007-1010,1012-1013,1017,1019-1022,1024-1027,1032,1035-1037,1039,1041,1043,1045,1047-1050,1052-1056,1059-1060,1063,1065,1067,1069-1073,1076-1077,1080-1083,1085,1087-1094,1099,1101,1103-1104,1107-1110,1112,1114,1116-1123,1127-1132,1138,1141,1143,1145-1148,1151,1153,1156-1159,1161-1165,1171,1173,1175-1178,1183,1185,1187-1188,1191-1193,1195,1197,1199-1202,1205-1206,1208-1209,1211,1213-1214,1216-1222,1226-1227,1231-1232,1234-1235,1243-1244,1247-1248,1250,1252-1259,1263,1267-1268,1272-1276,1281-1282,1284
 src/DescriptorsManager.cc                     24      19    79%   49-50,57-58,73
 src/ExceptionsCommand.cc                       6       0     0%   35-40
 src/ImageCommand.cc                          322     157    48%   55,59,63,65,67-69,71,73-76,78,81,86,88-89,97,99,106,109,111-112,114-115,117-118,120-121,124,151,162-163,174-175,177,182-185,195-196,198,203-206,221-229,231-233,246-247,257-267,269-270,272-273,278,286,297,304,308,311,313,315,337,339-340,343-348,350,352,374-376,379-381,385-388,394,396,403-406,420,427,433-436,440-441,452-455,458-463,468-470,481-484,489-493,498-499,501-502,504-508,511,513-517,520-523,526-527,530,532,537
@@ -20,8 +20,8 @@ src/Neo4jBaseCommands.cc                      38       0     0%   7-8,12,14-15,1
 src/Neo4JHandlerCommands.cc                  109       0     0%   50,54-55,57-58,61,65-69,72,74,76-80,82,84,86,92,95-96,98,100,102,106-107,110-111,114-119,123-124,126,128-129,132-133,137-139,141,144,149-151,155,157,160,162,165-166,169,172,176-178,180-186,188-191,196,198-201,203,206,209,211-213,217-220,222-225,228-231,233-237,239-244,249
 src/OpsIOCoordinator.cc                       96      73    76%   48,52,54-55,57,61-63,65,74,78,80,91,93,100,102-103,123,125,133,149,153,203
 src/PMGDIterators.cc                          52      44    84%   62,76,96-101
-src/PMGDQuery.cc                             463     363    78%   89-92,94-96,129,131,135,140,143-144,167-169,171-172,211-212,216-218,248,254,258,288,298,302-305,307,309,311,317,321-322,354,356,358,360-361,364-373,375-377,379,383-384,386-388,409,412-414,419-424,446,449-450,480-481,492,547,549-557,653-654,658-662,664-668
-src/PMGDQueryHandler.cc                      623     517    83%   82-84,166-167,169-170,194-197,208-209,230,279,281,285,290,292,320-321,338,340,344,346,397-398,400,402-407,409,411,463-464,478-479,488,490,496,498,524-526,537,566,605,607,612-613,649,651-655,677-679,681-686,688,720,729-730,737,741-743,745,748,751,815,822,850,870-876,878-879,881,883,895-896,915-917,921-922,965,1012-1013,1015-1017
+src/PMGDQuery.cc                             466     366    78%   89-92,94-96,129,131,135,140,143-144,167-169,171-172,211-212,216-218,248,254,258,288,298,302-305,307,309,311,317,321-322,354,356,358,360-361,364-373,375-377,379,383-384,386-388,409,412-414,419-424,446,449-450,480-481,492,547,549-557,653-654,658-662,664-668
+src/PMGDQueryHandler.cc                      643     523    81%   82-84,166-167,169-170,194-197,208-209,230,279,281,285,290,292,320-321,338,340,344,346,397-398,400,402-407,409,411,463-464,478-479,488,490,496,498,524-526,537,566,605,607,612-613,649,651-655,677-679,681-686,688,720,729-730,737,741-743,745,748,751,815,822,850,870-876,878-879,881,883,895-896,915-917,921-922,965,1012-1013,1015-1017,1024,1027,1029-1038,1050,1052
 src/QueryHandlerBase.cc                       32       6    18%   26-29,35-36,38,42-43,45,47-48,52,56-58,60-62,64-66,68-69,72-73
 src/QueryHandlerExample.cc                    33      18    54%   65-67,75-78,84-85,89-92,94-95
 src/QueryHandlerNeo4j.cc                     139       0     0%   53,55-56,58,60-62,64-65,67,70-76,80-81,83-87,91,93,95-100,104-108,111-115,119-126,129-132,134-136,139-147,149-153,159,162,169,172-175,177-179,181,184-187,189-190,192,194,197,199,201-204,207-208,210,212-213,216,218,222-223,225,229,231-232,235,237-240,243-247,250-254,256-257,261-268,271-274,277
@@ -57,5 +57,5 @@ utils/src/comm/Exception.cc                    6       0     0%   35-40
 utils/src/stats/SystemStats.cc               250     249    99%   453
 utils/src/timers/TimerMap.cc                  82      75    91%   126,151,153,155-158
 ------------------------------------------------------------------------------
-TOTAL                                      10142    6488    64%
+TOTAL                                      10180    6501    63%
 ------------------------------------------------------------------------------
diff --git a/.github/coverage/cpp.develop.coverage_value.txt b/.github/coverage/cpp.develop.coverage_value.txt
@@ -1 +1 @@
-63.9716
+63.8605
diff --git a/src/DescriptorsCommand.cc b/src/DescriptorsCommand.cc
@@ -264,15 +264,15 @@ int AddDescriptorSet::construct_protobuf(PMGDQuery &query,
     add_link(query, cmd["link"], node_ref, VDMS_DESC_SET_EDGE_TAG);
   }
 
-  //create a new index based on the descriptor set name
-  try{
-      std::string idx_prop = VDMS_DESC_ID_PROP + std::string("_") + set_name;
-      query.AddIntNodeIndexImmediate(VDMS_DESC_TAG, (char *)idx_prop.c_str());
-  } catch(...) {
-      printf("Descriptor Set Index Creation Failed for %s\n", set_name.c_str());
-      error["info"] = "Set index creation failed for" + set_name;
-      error["status"] = RSCommand::Error;
-      return -1;
+  // create a new index based on the descriptor set name
+  try {
+    std::string idx_prop = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+    query.AddIntNodeIndexImmediate(VDMS_DESC_TAG, (char *)idx_prop.c_str());
+  } catch (...) {
+    printf("Descriptor Set Index Creation Failed for %s\n", set_name.c_str());
+    error["info"] = "Set index creation failed for" + set_name;
+    error["status"] = RSCommand::Error;
+    return -1;
   }
 
   return 0;
@@ -474,7 +474,8 @@ int AddDescriptor::add_single_descriptor(PMGDQuery &query,
     return -1;
   }
 
-  std::string desc_id_prop_name = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+  std::string desc_id_prop_name =
+      VDMS_DESC_ID_PROP + std::string("_") + set_name;
   props[desc_id_prop_name] = Json::Int64(id);
 
   int node_ref = get_value<int>(cmd, "_ref", query.get_available_reference());
@@ -614,7 +615,8 @@ int AddDescriptor::add_descriptor_batch(PMGDQuery &query,
     Json::Value cur_props;
     cur_props = prop_list[i];
 
-    std::string desc_id_prop_name = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+    std::string desc_id_prop_name =
+        VDMS_DESC_ID_PROP + std::string("_") + set_name;
     cur_props[desc_id_prop_name.c_str()] = Json::Int64(id + i);
 
     cur_props[VDMS_DESC_LABEL_PROP] = label;
@@ -795,7 +797,8 @@ int FindDescriptor::construct_protobuf(PMGDQuery &query,
 
   int dimensions;
   const std::string set_path = get_set_path(query, set_name, dimensions);
-  std::string desc_id_prop_name = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+  std::string desc_id_prop_name =
+      VDMS_DESC_ID_PROP + std::string("_") + set_name;
 
   if (set_path.empty()) {
     cp_result["status"] = RSCommand::Error;
@@ -1004,7 +1007,8 @@ void FindDescriptor::populate_blobs(const std::string &set_path,
                                     Json::Value &entities,
                                     protobufs::queryMessage &query_res) {
 
-  std::string desc_id_prop_name = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+  std::string desc_id_prop_name =
+      VDMS_DESC_ID_PROP + std::string("_") + set_name;
   if (get_value<bool>(results, "blob", false)) {
     VCL::DescriptorSet *set = _dm->get_descriptors_handler(set_path);
     int dim = set->get_dimensions();
@@ -1027,11 +1031,13 @@ void FindDescriptor::populate_blobs(const std::string &set_path,
 }
 
 void FindDescriptor::convert_properties(Json::Value &entities,
-                                        Json::Value &list, std::string set_name) {
+                                        Json::Value &list,
+                                        std::string set_name) {
   bool flag_label = false;
   bool flag_id = false;
 
-  std::string desc_id_prop_name = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+  std::string desc_id_prop_name =
+      VDMS_DESC_ID_PROP + std::string("_") + set_name;
   for (auto &prop : list) {
     if (prop.asString() == "_label") {
       flag_label = true;
@@ -1068,7 +1074,8 @@ Json::Value FindDescriptor::construct_responses(
   bool flag_error = false;
 
   const std::string set_name = cmd["set"].asString();
-  std::string desc_id_prop_name = VDMS_DESC_ID_PROP + std::string("_") + set_name;
+  std::string desc_id_prop_name =
+      VDMS_DESC_ID_PROP + std::string("_") + set_name;
 
   auto error = [&](Json::Value &res) {
     ret[_cmd_name] = res;

diff --git a/src/DescriptorsCommand.h b/src/DescriptorsCommand.h
@@ -59,7 +59,7 @@ class DescriptorsCommand : public RSCommand {
   tbb::concurrent_unordered_map<long, IDDistancePair *> _cache_map;
 
   static tbb::concurrent_unordered_map<std::string, std::string>
-    _desc_set_locator;
+      _desc_set_locator;
   static tbb::concurrent_unordered_map<std::string, int> _desc_set_dims;
 
   // Will return the path to the set and the dimensions
@@ -133,12 +133,12 @@ class AddDescriptor : public DescriptorsCommand {
   void retrieve_aws_descriptorSet(const std::string &set_path);
 
   int add_single_descriptor(PMGDQuery &tx, const Json::Value &root,
-                         const std::string &blob, int grp_id,
-                         Json::Value &error);
+                            const std::string &blob, int grp_id,
+                            Json::Value &error);
 
   int add_descriptor_batch(PMGDQuery &tx, const Json::Value &root,
-                         const std::string &blob, int grp_id,
-                         Json::Value &error);
+                           const std::string &blob, int grp_id,
+                           Json::Value &error);
 
 public:
   AddDescriptor();
@@ -175,10 +175,10 @@ class ClassifyDescriptor : public DescriptorsCommand {
 class FindDescriptor : public DescriptorsCommand {
 
 private:
-  void convert_properties(Json::Value &entities, Json::Value &list, std::string set_name);
+  void convert_properties(Json::Value &entities, Json::Value &list,
+                          std::string set_name);
   void populate_blobs(const std::string &set_path, std::string set_name,
-                      const Json::Value &results,
-                      Json::Value &entities,
+                      const Json::Value &results, Json::Value &entities,
                       protobufs::queryMessage &query_res);
 
 public:

diff --git a/src/PMGDQuery.cc b/src/PMGDQuery.cc
@@ -749,5 +749,5 @@ void PMGDQuery::DeleteExpired() {
 }
 
 void PMGDQuery::AddIntNodeIndexImmediate(char *node_class, char *prop_name) {
-    _pmgd_qh.build_node_int_index(node_class, prop_name);
+  _pmgd_qh.build_node_int_index(node_class, prop_name);
 }
diff --git a/src/PMGDQuery.h b/src/PMGDQuery.h
@@ -114,8 +114,8 @@ class PMGDQuery {
                  const Json::Value &constraints, const Json::Value &results,
                  bool unique = false);
 
-  //Note: This runs at call time and does not "build" a stack of queries
-  //in constrast to the others.
+  // Note: This runs at call time and does not "build" a stack of queries
+  // in constrast to the others.
   void AddIntNodeIndexImmediate(char *node_class, char *prop_name);
 
   void DeleteExpired();

diff --git a/src/PMGDQueryHandler.cc b/src/PMGDQueryHandler.cc
@@ -1021,35 +1021,36 @@ void insert_into_queue(std::list<AutoDeleteNode *> *queue,
   }
 }
 
-void PMGDQueryHandler::print_node_idx_stats(char *tag_name, char *prop_id){
-
-    try{
-        Transaction tx(*_db, Transaction::ReadWrite);
-        struct Graph::IndexStats istats = _db->get_index_stats(Graph::NodeIndex, tag_name, prop_id);
-        printf("Index Stats for : %s %s\n", tag_name, prop_id);
-        printf("Total Entries: %lu\n", istats.total_unique_entries);
-        printf("Unique Entry Size: %lu\n", istats.unique_entry_size);
-        printf("Total Elements: %lu\n", istats.total_elements);
-        printf("Total Size Bytes: %lu\n", istats.total_size_bytes);
-        printf("Health Factor: %lu\n", istats.health_factor);
-        tx.commit();
-    } catch (PMGD::Exception e) {
-        print_exception(e);
-    }
+void PMGDQueryHandler::print_node_idx_stats(char *tag_name, char *prop_id) {
 
+  try {
+    Transaction tx(*_db, Transaction::ReadWrite);
+    struct Graph::IndexStats istats =
+        _db->get_index_stats(Graph::NodeIndex, tag_name, prop_id);
+    printf("Index Stats for : %s %s\n", tag_name, prop_id);
+    printf("Total Entries: %lu\n", istats.total_unique_entries);
+    printf("Unique Entry Size: %lu\n", istats.unique_entry_size);
+    printf("Total Elements: %lu\n", istats.total_elements);
+    printf("Total Size Bytes: %lu\n", istats.total_size_bytes);
+    printf("Health Factor: %lu\n", istats.health_factor);
+    tx.commit();
+  } catch (PMGD::Exception e) {
+    print_exception(e);
+  }
 }
 
-void PMGDQueryHandler::build_node_int_index(char *node_class, char *prop_name){
-
-    try{
-        Transaction tx(*_db, Transaction::ReadWrite);
-        _db->create_index(Graph::NodeIndex, node_class, prop_name, PropertyType::Integer);
-        tx.commit();
-    } catch (PMGD::Exception e){
-        printf("Warning: Failed to create new integer index-%s %s", node_class, prop_name);
-        print_exception(e);
-    }
+void PMGDQueryHandler::build_node_int_index(char *node_class, char *prop_name) {
 
+  try {
+    Transaction tx(*_db, Transaction::ReadWrite);
+    _db->create_index(Graph::NodeIndex, node_class, prop_name,
+                      PropertyType::Integer);
+    tx.commit();
+  } catch (PMGD::Exception e) {
+    printf("Warning: Failed to create new integer index-%s %s", node_class,
+           prop_name);
+    print_exception(e);
+  }
 }
 
 void delete_by_value(std::list<AutoDeleteNode *> *queue, void *p_delete_node) {
@@ -1078,5 +1079,3 @@ void cleanup_pmgd_files(std::vector<std::string> *p_cleanup_list) {
     it++;
   }
 }
-
-
diff --git a/src/PMGDQueryHandler.h b/src/PMGDQueryHandler.h
@@ -166,7 +166,6 @@ class PMGDQueryHandler {
   void cleanup_files();
   void build_node_int_index(char *node_class, char *prop_name);
   void print_node_idx_stats(char *tag_name, char *prop_id);
-
 };
 
 }; // namespace VDMS