From 0be89f7324d027eca51759acdbed5de233e49581 Mon Sep 17 00:00:00 2001 From: Luis Remis Date: Mon, 14 Jan 2019 07:23:12 -0800 Subject: [PATCH] Fix directories handling --- config-vdms.json | 13 ++--- run_server.sh | 11 +--- src/DescriptorsCommand.cc | 3 +- src/DescriptorsCommand.h | 1 - src/ImageCommand.cc | 9 +-- src/ImageCommand.h | 4 -- src/PMGDQueryHandler.cc | 3 +- src/RSCommand.cc | 3 +- src/RSCommand.h | 2 - src/VDMSConfig.cc | 94 +++++++++++++++++++++++++++++++- src/VDMSConfig.h | 31 +++++++++++ tests/config-add10-tests.json | 2 +- tests/config-addfind-tests.json | 2 +- tests/config-tests.json | 6 +- tests/config-update-tests.json | 6 +- tests/python/clean.sh | 9 --- tests/python/config-tests.json | 5 +- tests/python/run_python_tests.sh | 3 +- 18 files changed, 152 insertions(+), 55 deletions(-) delete mode 100644 tests/python/clean.sh diff --git a/config-vdms.json b/config-vdms.json index c3df33e6..ee3e0d21 100644 --- a/config-vdms.json +++ b/config-vdms.json @@ -3,16 +3,11 @@ // Sets database paths and other parameters { // Network - "port": 55555, // Default is 55555 - "max_simultaneous_clients": 20, // Default is 500 + "port": 55555, + "max_simultaneous_clients": 100, // Database paths - "pmgd_path": "db/graph", // This will be an IP address in the future - "png_path": "db/images/pngs/", - "jpg_path": "db/images/jpgs/", - "tdb_path": "db/images/tiledb/tdb/", - "blob_path":"db/blobs/", + "db_root_path": "db", - "support_info": "a-team@intel.com", - "support_phone": "1-800-A-TEAM" + "more-info": "github.com/IntelLabs/vdms" } diff --git a/run_server.sh b/run_server.sh index 8c66b62c..8ad1fe07 100644 --- a/run_server.sh +++ b/run_server.sh @@ -1,12 +1,5 @@ rm log.log -rm -r db - -mkdir db -mkdir db/images -mkdir db/images/png -mkdir db/images/jpg -mkdir db/images/tiledb -mkdir db/blobs -mkdir db/descriptors ./vdms 2> log.log + +# If problems with initialization, try deleting db folder. diff --git a/src/DescriptorsCommand.cc b/src/DescriptorsCommand.cc index 2f9b66dc..aca5c1ed 100644 --- a/src/DescriptorsCommand.cc +++ b/src/DescriptorsCommand.cc @@ -106,8 +106,7 @@ bool DescriptorsCommand::check_blob_size(const std::string& blob, const int dime AddDescriptorSet::AddDescriptorSet() : DescriptorsCommand("AddDescriptorSet") { - _storage_sets = VDMSConfig::instance() - ->get_string_value("descriptors", DEFAULT_DESCRIPTORS_PATH); + _storage_sets = VDMSConfig::instance()->get_path_descriptors(); } int AddDescriptorSet::construct_protobuf( diff --git a/src/DescriptorsCommand.h b/src/DescriptorsCommand.h index 1195fe6b..84811447 100644 --- a/src/DescriptorsCommand.h +++ b/src/DescriptorsCommand.h @@ -85,7 +85,6 @@ namespace VDMS{ class AddDescriptorSet: public DescriptorsCommand { std::string _storage_sets; - const std::string DEFAULT_DESCRIPTORS_PATH = "db/descriptors"; public: AddDescriptorSet(); diff --git a/src/ImageCommand.cc b/src/ImageCommand.cc index 5af6d9e5..c2981e67 100644 --- a/src/ImageCommand.cc +++ b/src/ImageCommand.cc @@ -80,12 +80,9 @@ void ImageCommand::enqueue_operations(VCL::Image& img, const Json::Value& ops) AddImage::AddImage() : ImageCommand("AddImage") { - _storage_tdb = VDMSConfig::instance() - ->get_string_value("tdb_path", DEFAULT_TDB_PATH); - _storage_png = VDMSConfig::instance() - ->get_string_value("png_path", DEFAULT_PNG_PATH); - _storage_jpg = VDMSConfig::instance() - ->get_string_value("jpg_path", DEFAULT_JPG_PATH); + _storage_tdb = VDMSConfig::instance()->get_path_tdb(); + _storage_png = VDMSConfig::instance()->get_path_png(); + _storage_jpg = VDMSConfig::instance()->get_path_jpg(); } int AddImage::construct_protobuf(PMGDQuery& query, diff --git a/src/ImageCommand.h b/src/ImageCommand.h index 49f8f908..69ecdecb 100644 --- a/src/ImageCommand.h +++ b/src/ImageCommand.h @@ -63,10 +63,6 @@ namespace VDMS { class AddImage: public ImageCommand { - const std::string DEFAULT_TDB_PATH = "db/images/tiledb/tdb"; - const std::string DEFAULT_PNG_PATH = "db/images/png"; - const std::string DEFAULT_JPG_PATH = "db/images/jpg"; - std::string _storage_tdb; std::string _storage_png; std::string _storage_jpg; diff --git a/src/PMGDQueryHandler.cc b/src/PMGDQueryHandler.cc index a6f7d534..c86e650f 100644 --- a/src/PMGDQueryHandler.cc +++ b/src/PMGDQueryHandler.cc @@ -45,8 +45,7 @@ PMGD::Graph *PMGDQueryHandler::_db; void PMGDQueryHandler::init() { - std::string dbname = VDMSConfig::instance() - ->get_string_value("pmgd_path", "db/graph"); + std::string dbname = VDMSConfig::instance()->get_path_pmgd(); // Create a db _db = new PMGD::Graph(dbname.c_str(), PMGD::Graph::Create); diff --git a/src/RSCommand.cc b/src/RSCommand.cc index f3a46d7f..c87d48f0 100644 --- a/src/RSCommand.cc +++ b/src/RSCommand.cc @@ -161,8 +161,7 @@ void RSCommand::add_link(PMGDQuery& query, const Json::Value& link, AddEntity::AddEntity() : RSCommand("AddEntity") { - _storage_blob = VDMSConfig::instance() - ->get_string_value("blob_path", DEFAULT_BLOB_PATH); + _storage_blob = VDMSConfig::instance()->get_path_blobs(); } bool AddEntity::need_blob(const Json::Value& jsoncmd) diff --git a/src/RSCommand.h b/src/RSCommand.h index 155f8d0f..1a3fb62d 100644 --- a/src/RSCommand.h +++ b/src/RSCommand.h @@ -91,8 +91,6 @@ namespace VDMS { class AddEntity : public RSCommand { private: - const std::string DEFAULT_BLOB_PATH = "db/blobs"; - std::string _storage_blob; public: diff --git a/src/VDMSConfig.cc b/src/VDMSConfig.cc index 440f9156..40859f9c 100644 --- a/src/VDMSConfig.cc +++ b/src/VDMSConfig.cc @@ -34,10 +34,23 @@ #include #include +#include +#include +#include + #include #include "VDMSConfig.h" +#define DEFAULT_PATH_ROOT "db" +#define DEFAULT_PATH_PMGD "graph" +#define DEFAULT_PATH_IMAGES "images" +#define DEFAULT_PATH_JPG "jpg" +#define DEFAULT_PATH_PNG "png" +#define DEFAULT_PATH_TDB "tdb" +#define DEFAULT_PATH_BLOBS "blobs" +#define DEFAULT_PATH_DESCRIPTORS "descriptors" + using namespace VDMS; VDMSConfig* VDMSConfig::cfg; @@ -76,8 +89,12 @@ VDMSConfig::VDMSConfig(std::string config_file) bool parsingSuccessful = reader.parse(file, json_config); if (!parsingSuccessful){ - std::cout << "Error parsing config file" << std::endl; + std::cout << "Error parsing config file." << std::endl; + std::cout << "Exiting..." << std::endl; + exit(0); } + + build_dirs(); } int VDMSConfig::get_int_value(std::string val, int def) @@ -89,3 +106,78 @@ std::string VDMSConfig::get_string_value(std::string val, std::string def) { return json_config.get(val, def).asString(); } + +// This method will check if the dir exists, +// and create the dir if it does not exist. +int VDMSConfig::create_dir(std::string path) +{ + struct stat sb; + while (1) + if (stat(path.c_str(), &sb) == 0) + if (sb.st_mode & S_IFDIR) + return 0; + else + return EEXIST; + else if (errno != ENOENT) + return errno; + else if (mkdir(path.c_str(), 0777) == 0) + return 0; + else if (errno != EEXIST) + return errno; +} + +void VDMSConfig::check_or_create(std::string path) +{ + if (create_dir(path) == 0){ + return; + } + else{ + std::cout << "Cannot open/create directories structure." << std::endl; + std::cout << "Failed dir: " << path << std::endl; + std::cout << "Check paths and permissions." << std::endl; + std::cout << "Exiting..." << std::endl; + exit(0); + } +} + +void VDMSConfig::build_dirs() +{ + // Root + path_root = get_string_value(PARAM_DB_ROOT, DEFAULT_PATH_ROOT); + check_or_create(path_root); + + // PMGD + path_pmgd = path_root + "/" + DEFAULT_PATH_PMGD; + path_pmgd = get_string_value(PARAM_DB_PMGD, path_pmgd); + check_or_create(path_pmgd); + + // IMAGES + path_images = path_root + "/" + DEFAULT_PATH_IMAGES; + path_images = get_string_value(PARAM_DB_IMAGES, path_images); + check_or_create(path_images); + + // IMAGES - PNG + path_png = path_images + "/" + DEFAULT_PATH_PNG; + path_png = get_string_value(PARAM_DB_PNG, path_png); + check_or_create(path_png); + + // IMAGES - JPG + path_jpg = path_images + "/" + DEFAULT_PATH_JPG; + path_jpg = get_string_value(PARAM_DB_JPG, path_jpg); + check_or_create(path_jpg); + + // IMAGES - TDB + path_tdb = path_images + "/" + DEFAULT_PATH_TDB; + path_tdb = get_string_value(PARAM_DB_TDB, path_tdb); + check_or_create(path_tdb); + + // BLOBS + path_blobs = path_root + "/" + DEFAULT_PATH_BLOBS; + path_blobs = get_string_value(PARAM_DB_BLOBS, path_blobs); + check_or_create(path_blobs); + + // DESCRIPTORS + path_descriptors = path_root + "/" + DEFAULT_PATH_DESCRIPTORS; + path_descriptors = get_string_value(PARAM_DB_DESCRIPTORS, path_descriptors); + check_or_create(path_descriptors); +} diff --git a/src/VDMSConfig.h b/src/VDMSConfig.h index 509975a7..c1d3362c 100644 --- a/src/VDMSConfig.h +++ b/src/VDMSConfig.h @@ -34,6 +34,16 @@ #include #include +// Parameters in the JSON config file +#define PARAM_DB_ROOT "db_root_path" +#define PARAM_DB_PMGD "pmgd_path" +#define PARAM_DB_IMAGES "images_path" +#define PARAM_DB_PNG "png_path" +#define PARAM_DB_JPG "jpg_path" +#define PARAM_DB_TDB "tdb_path" +#define PARAM_DB_BLOBS "blobs_path" +#define PARAM_DB_DESCRIPTORS "descriptors_path" + namespace VDMS{ class VDMSConfig @@ -48,11 +58,32 @@ namespace VDMS{ static VDMSConfig* cfg; Json::Value json_config; + // Dirs + std::string path_root; + std::string path_pmgd; + std::string path_images; + std::string path_png; + std::string path_jpg; + std::string path_tdb; + std::string path_blobs; + std::string path_descriptors; + VDMSConfig(std::string config_file); + void build_dirs(); + void check_or_create(std::string path); + int create_dir(std::string path); + public: int get_int_value(std::string val, int def); std::string get_string_value(std::string val, std::string def); + std::string get_path_root() {return path_root;} + std::string get_path_pmgd() {return path_pmgd;} + std::string get_path_jpg() {return path_jpg;} + std::string get_path_png() {return path_png;} + std::string get_path_tdb() {return path_tdb;} + std::string get_path_blobs() {return path_blobs;} + std::string get_path_descriptors() {return path_descriptors;} }; }; // vdms namespace diff --git a/tests/config-add10-tests.json b/tests/config-add10-tests.json index d20b8fb3..0fa2bb15 100644 --- a/tests/config-add10-tests.json +++ b/tests/config-add10-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, // Default is 55555 + "port": 55555, // Database paths "pmgd_path": "simpleAddx10_db" diff --git a/tests/config-addfind-tests.json b/tests/config-addfind-tests.json index f53364f5..e243bcec 100644 --- a/tests/config-addfind-tests.json +++ b/tests/config-addfind-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55555, // Default is 55555 + "port": 55555, // Database paths "pmgd_path": "jsongraph" diff --git a/tests/config-tests.json b/tests/config-tests.json index 1ff6b2c5..2d158362 100644 --- a/tests/config-tests.json +++ b/tests/config-tests.json @@ -3,8 +3,10 @@ // Sets database paths and other parameters { // Network - "port": 55555, // Default is 55555 + "port": 55555, // Database paths - "pmgd_path": "simpleAdd_db" + "pmgd_path": "simpleAdd_db", + + "more-info": "github.com/IntelLabs/vdms" } diff --git a/tests/config-update-tests.json b/tests/config-update-tests.json index 93adaa52..8765c8c4 100644 --- a/tests/config-update-tests.json +++ b/tests/config-update-tests.json @@ -3,8 +3,10 @@ // Sets database paths and other parameters { // Network - "port": 55555, // Default is 55555 + "port": 55555, // Database paths - "pmgd_path": "simpleUpdate_db" + "pmgd_path": "simpleUpdate_db", + + "more-info": "github.com/IntelLabs/vmds" } diff --git a/tests/python/clean.sh b/tests/python/clean.sh deleted file mode 100644 index 7010f242..00000000 --- a/tests/python/clean.sh +++ /dev/null @@ -1,9 +0,0 @@ -rm log.log screen.log -rm -r db -mkdir db -mkdir db/images -mkdir db/images/png -mkdir db/images/jpg -mkdir db/images/tiledb/ -mkdir db/blobs -mkdir db/descriptors diff --git a/tests/python/config-tests.json b/tests/python/config-tests.json index 9b0b0948..fb1d3077 100644 --- a/tests/python/config-tests.json +++ b/tests/python/config-tests.json @@ -3,5 +3,8 @@ // Sets database paths and other parameters { // Network - "port": 55557 // Default is 55555 + "port": 55557, + "db_root_path": "test_db", + + "more-info": "github.com/IntelLabs/vdms" } diff --git a/tests/python/run_python_tests.sh b/tests/python/run_python_tests.sh index 5a8cc818..bef45ae0 100644 --- a/tests/python/run_python_tests.sh +++ b/tests/python/run_python_tests.sh @@ -1,4 +1,5 @@ -sh clean.sh +rm log.log screen.log +rm -r test_db ../../vdms -cfg config-tests.json > screen.log 2> log.log & python -m unittest discover --pattern=*.py -v