From fccb2eb960f418c33ea0c4e5838634e189c659a1 Mon Sep 17 00:00:00 2001 From: Ragaad Date: Mon, 6 Mar 2023 12:25:10 -0800 Subject: [PATCH] Add generic AddBlob command (#79) * Add generic AddBlob command * Format CMakeLists.txt * Fix the AddBlob test and include all other tests * Fix run_test.sh * Add the add_blob test and fix the blob reading functions for the client examples * Fix the run_Test to include all the tests * Aff FindBlob Test code * Fix the FindBLob test * ADD UPDATE BLOB TEST and FINDBLOB TEST * Update dockerfiles (#81) (#83) * Update dockerfiles; Checkin dockerfiles now only copy vdms folders; remove maven; centralize dependencies in /dependencies folder * Correct zlib and tiledb paths * Remove coverage file check * Change coverage script path * Change coverage script path * cd before coverage script * Add maven build arg to avoid target failure * Add blob: Update coverage (#84) * Modify python port and run coverage w/ same cmd * move large1.jpg to test_images folder, updated paths to image, remove unused image and video in images folder --------- Co-authored-by: Chaunte W. Lacewell --- .github/workflows/coverage.yml | 19 +- CMakeLists.txt | 22 +- client/cpp/VDMSClient.h | 2 +- src/BlobCommand.cc | 243 +++++++++++++++++++++++ src/BlobCommand.h | 112 +++++++++++ src/QueryHandler.cc | 5 + src/defines.h | 2 + tests/CMakeLists.txt | 1 + tests/cleandbs.sh | 7 +- tests/python/TestCommand.py | 2 +- tests/python/config-tests.json | 2 +- tests/server/AddFindUpdate_blob.json | 39 ++++ tests/server/json_queries.cc | 74 ++++++- tests/{images => test_images}/large1.jpg | Bin tests/unit_tests/Image_test.cc | 18 +- tests/unit_tests/TDBImage_test.cc | 2 +- tests/unit_tests/client_blob.cc | 48 +++++ tests/unit_tests/client_bounding_box.cc | 18 +- tests/unit_tests/client_image.cc | 41 ++-- tests/unit_tests/client_videos.cc | 28 +-- tests/unit_tests/meta_data.cc | 74 ++++++- tests/unit_tests/meta_data_helper.h | 8 +- utils/src/api_schema/api_schema.json | 66 +++++- 23 files changed, 725 insertions(+), 108 deletions(-) create mode 100644 src/BlobCommand.cc create mode 100644 src/BlobCommand.h create mode 100644 tests/server/AddFindUpdate_blob.json rename tests/{images => test_images}/large1.jpg (100%) create mode 100644 tests/unit_tests/client_blob.cc diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6ff3aa13..3f29ba78 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -88,26 +88,11 @@ jobs: mkdir -p coverage echo "${{ matrix.container_name }}" - # Make sure CPP coverage script is available - # if [ ! -f docker/check-in/run_coverage_cpp.sh ]; then - # docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_cpp.sh && echo 'cd /vdms/tests && chmod +x run_tests.sh && ./run_tests.sh' >> /run_coverage_cpp.sh && chmod +x /run_coverage_cpp.sh && mkdir -p /vdms/tests/coverage_report" - # docker exec ${{ matrix.container_name }} bash -c "echo 'gcovr -k --root /vdms -e /vdms/src/pmgd -e /vdms/build/CMakeFiles -f "/vdms/client/.*\.cc" -f "/vdms/ext/.*\.cc" -f "/vdms/src/.*\.cc" -f src/SearchExpression.cc --exclude-unreachable-branches --xml-pretty --xml=/vdms/tests/coverage_report/c_coverage_report.xml --txt=/vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh && echo "echo 'DONE'" >> /run_coverage_cpp.sh" - # docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/c_coverage_report.txt' >> /run_coverage_cpp.sh" - # fi - - # # Make sure Python coverage script is available - # if [ ! -f docker/check-in/run_coverage_py.sh ]; then - # docker exec ${{ matrix.container_name }} bash -c "touch /run_coverage_py.sh && echo 'cd /vdms/tests/python && ./run_python_tests.sh' >> /run_coverage_py.sh && chmod +x /run_coverage_py.sh && mkdir -p /vdms/tests/coverage_report" - # docker exec ${{ matrix.container_name }} bash -c "echo 'python -m coverage report > /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh && echo "echo 'DONE'" >> /run_coverage_py.sh" - # docker exec ${{ matrix.container_name }} bash -c "echo 'cat /vdms/tests/coverage_report/py_coverage_report.txt' >> /run_coverage_py.sh" - # fi - - docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_cpp.sh" + docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_cpp.sh && cd / && ./run_coverage_py.sh" docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.txt coverage/c_coverage_report_target.txt docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/c_coverage_report.xml coverage/c_coverage_report_target.xml echo "coverage_value_cpp=$(cat coverage/c_coverage_report_target.xml | grep -oP 'coverage line-rate="([-+]?\d*\.\d+|\d+)"' | grep -oP "[-+]?\d*\.\d+|\d+" | awk '{print $1*100}')" >> $GITHUB_ENV - docker exec ${{ matrix.container_name }} bash -c "cd / && ./run_coverage_py.sh" docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/py_coverage_report.txt coverage/py_coverage_report_target.txt || true docker cp $(docker ps -a | grep ${{ matrix.container_name }} | awk '{print $1}'):/vdms/tests/coverage_report/py_coverage_report.xml coverage/py_coverage_report_target.xml || true if [ ! -f coverage/py_coverage_report_target.xml ] && [ "${{ matrix.coverage_type }}" == "Target" ]; then @@ -175,7 +160,7 @@ jobs: echo "Source Python Coverage: ${{needs.coverage_job.outputs.source_coverage_py}}" echo "Target Python Coverage: ${{needs.coverage_job.outputs.target_coverage_py}}" - if ${{needs.coverage_job.outputs.target_coverage_py}} != 0 && ${{ needs.coverage_job.outputs.target_coverage_py > needs.coverage_job.outputs.source_coverage_py }} + if ${{ needs.coverage_job.outputs.target_coverage_py > needs.coverage_job.outputs.source_coverage_py }} then echo 'Python coverage below target' exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index b629dde3..b51e6fbd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,27 @@ else() link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/) include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf) - add_library(dms SHARED src/BoundingBoxCommand.cc src/CommunicationManager.cc src/DescriptorsCommand.cc src/DescriptorsManager.cc src/ExceptionsCommand.cc src/ImageCommand.cc src/PMGDIterators.cc src/PMGDQuery.cc src/PMGDQueryHandler.cc src/QueryHandler.cc src/QueryMessage.cc src/RSCommand.cc src/SearchExpression.cc src/Server.cc src/VDMSConfig.cc src/VideoCommand.cc src/AutoDeleteNode.cc ${PROTO_SRCS} ${PROTO_HDRS}) + add_library(dms SHARED + src/BoundingBoxCommand.cc + src/BlobCommand.cc + src/CommunicationManager.cc + src/DescriptorsCommand.cc + src/DescriptorsManager.cc + src/ExceptionsCommand.cc + src/ImageCommand.cc + src/PMGDIterators.cc + src/PMGDQuery.cc + src/PMGDQueryHandler.cc + src/QueryHandler.cc + src/QueryMessage.cc + src/RSCommand.cc + src/SearchExpression.cc + src/Server.cc + src/VDMSConfig.cc + src/VideoCommand.cc + src/AutoDeleteNode.cc + ${PROTO_SRCS} ${PROTO_HDRS} + ) target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread) diff --git a/client/cpp/VDMSClient.h b/client/cpp/VDMSClient.h index 54ca8f9e..866a9b8e 100644 --- a/client/cpp/VDMSClient.h +++ b/client/cpp/VDMSClient.h @@ -55,7 +55,7 @@ namespace VDMS { // Blocking call VDMS::Response query(const std::string &json_query, - const std::vector blobs = {}); + const std::vector blobs = {}); }; }; diff --git a/src/BlobCommand.cc b/src/BlobCommand.cc new file mode 100644 index 00000000..76b64b8d --- /dev/null +++ b/src/BlobCommand.cc @@ -0,0 +1,243 @@ +/** + * @file BlobCommand.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include + +#include "BlobCommand.h" +#include "VDMSConfig.h" +#include "defines.h" + +using namespace VDMS; + +//========= AddBlob definitions ========= + +BlobCommand::BlobCommand(const std::string &cmd_name): + RSCommand(cmd_name) +{ +} + +AddBlob::AddBlob() : BlobCommand("AddBlob") +{ + + _storage_bin = VDMSConfig::instance()->get_path_bin(); +} + +int AddBlob::construct_protobuf(PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + std::cout << " inside ADDBLOB" <(cmd, "_ref", + query.get_available_reference()); + + + std::string format = "bin"; + char binary_img_flag = 1; + VCL::Image img((void*)blob.data(), blob.size(), binary_img_flag); + + + std::string blob_root = _storage_bin; + VCL::Image::Format blob_format = VCL::Image::Format::BIN; + std::string file_name = VCL::create_unique(blob_root, format); + std::cout << "Blob was added in " <<_storage_bin << "\t"<< file_name << std::endl; + Json::Value props = get_value(cmd, "properties"); + props[VDMS_EN_BLOB_PATH_PROP] = file_name; + + + query.AddNode(node_ref, VDMS_BLOB_TAG, props, Json::Value()); + + img.store(file_name, blob_format); + + + error["Blob_added"] = file_name; + + if (cmd.isMember("link")) { + add_link(query, cmd["link"], node_ref, VDMS_BLOB_EDGE_TAG); + } + + return 0; +} + +//========= UpdateBLOB definitions ========= + +UpdateBlob::UpdateBlob() : BlobCommand("UpdateBlob") +{ +} + +int UpdateBlob::construct_protobuf(PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + // Update Image node + query.UpdateNode(get_value(cmd, "_ref", -1), + VDMS_BLOB_TAG, + cmd["properties"], + cmd["remove_props"], + cmd["constraints"], + get_value(cmd, "unique", false)); + + return 0; +} + +//========= FindBLOB definitions ========= + +FindBlob::FindBlob() : BlobCommand("FindBlob") +{ +} + +int FindBlob::construct_protobuf( + PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + Json::Value results = get_value(cmd, "results"); + + // Unless otherwhis specified, we return the blob. + if (get_value(results, "blob", true)){ + results["list"].append(VDMS_EN_BLOB_PATH_PROP); + } + + query.QueryNode( + get_value(cmd, "_ref", -1), + VDMS_BLOB_TAG, + cmd["link"], + cmd["constraints"], + results, + get_value(cmd, "unique", false) + ); + + return 0; +} + +Json::Value FindBlob::construct_responses( + Json::Value& responses, + const Json::Value& json, + protobufs::queryMessage &query_res, + const std::string &blob) +{ + const Json::Value& cmd = json[_cmd_name]; + + Json::Value ret; + + auto error = [&](Json::Value& res) + { + ret[_cmd_name] = res; + return ret; + }; + + if (responses.size() != 1) { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "PMGD Response Bad Size"; + return error(return_error); + } + + Json::Value& findBlob = responses[0]; + + if (findBlob["status"] != 0) { + findBlob["status"] = RSCommand::Error; + // Uses PMGD info error. + return error(findBlob); + } + + Json::Value results = get_value(cmd, "results"); + + bool flag_empty = false; + + // Check if blob (image) must be returned + if (get_value(results, "blob", true)) { + + for (auto& ent : findBlob["entities"]) { + + assert(ent.isMember(VDMS_EN_BLOB_PATH_PROP)); + + std::string blob_path = ent[VDMS_EN_BLOB_PATH_PROP].asString(); + ent.removeMember(VDMS_EN_BLOB_PATH_PROP); + + if (ent.getMemberNames().size() == 0) { + flag_empty = true; + } + + try { + VCL::Image blob_im(blob_path); + + + // We will return the image in the format the user + // request, or on its format in disk, except for the case + // of .tdb, where we will encode as png. + VCL::Image::Format format =VCL::Image::Format::BIN; + + std::vector blob_buffer; + blob_buffer = blob_im.get_encoded_image(format); + + if (!blob_buffer.empty()) { + + std::string* blob_str = query_res.add_blobs(); + blob_str->resize(blob_buffer.size()); + std::memcpy((void*)blob_str->data(), + (void*)blob_buffer.data(), + blob_buffer.size()); + } + else { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "Blob Data not found"; + return error(return_error); + } + } catch (VCL::Exception e) { + print_exception(e); + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "VCL Exception"; + return error(return_error); + } + } + } + + if (flag_empty) { + findBlob.removeMember("entities"); + } + + ret[_cmd_name].swap(findBlob); + return ret; +} diff --git a/src/BlobCommand.h b/src/BlobCommand.h new file mode 100644 index 00000000..c211a162 --- /dev/null +++ b/src/BlobCommand.h @@ -0,0 +1,112 @@ +/** + * @file BlobCommand.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#pragma once +#include +#include +#include +#include "vcl/Image.h" +#include "vcl/CustomVCL.h" + +#include "RSCommand.h" +#include "ExceptionsCommand.h" + +namespace VDMS { + +// Helper classes for handling various JSON commands. + + class BlobCommand: public RSCommand + { + public: + + BlobCommand(const std::string &cmd_name); + + virtual int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error) = 0; + + virtual bool need_blob(const Json::Value& cmd) { return false; } + + + + }; + + class AddBlob: public BlobCommand + { + + std::string _storage_bin; + + public: + AddBlob(); + + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error); + + bool need_blob(const Json::Value& cmd) { return true; } + }; + + class UpdateBlob: public BlobCommand + { + public: + UpdateBlob(); + + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error); + + + }; + + class FindBlob: public BlobCommand + { + public: + FindBlob(); + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error); + + Json::Value construct_responses( + Json::Value &json_responses, + const Json::Value &json, + protobufs::queryMessage &response, + const std::string &blob); + }; + +}; // namespace VDMS diff --git a/src/QueryHandler.cc b/src/QueryHandler.cc index f8a249b5..229a2ab9 100644 --- a/src/QueryHandler.cc +++ b/src/QueryHandler.cc @@ -38,6 +38,7 @@ #include "DescriptorsCommand.h" #include "BoundingBoxCommand.h" #include "VideoCommand.h" +#include "BlobCommand.h" #include "ExceptionsCommand.h" @@ -88,6 +89,10 @@ void QueryHandler::init() _rs_cmds["FindVideo"] = new FindVideo(); _rs_cmds["FindFrames"] = new FindFrames(); + _rs_cmds["AddBlob"] = new AddBlob(); + _rs_cmds["UpdateBlob"] = new UpdateBlob(); + _rs_cmds["FindBlob"] = new FindBlob(); + // Load the string containing the schema (api_schema/APISchema.h) Json::Reader reader; Json::Value api_schema; diff --git a/src/defines.h b/src/defines.h index ecc7df08..0a76b97a 100644 --- a/src/defines.h +++ b/src/defines.h @@ -45,6 +45,8 @@ // Entities #define VDMS_EN_BLOB_PATH_PROP "VD:blobPath" +#define VDMS_BLOB_TAG "VD:BLOB" +#define VDMS_BLOB_EDGE_TAG "VD:BLOBLINK" // Images diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dd323a2d..9fd1754a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,6 +46,7 @@ add_executable(unit_tests unit_tests/client_bounding_box.cc unit_tests/client_descriptors.cc unit_tests/client_videos.cc + unit_tests/client_blob.cc ) target_link_libraries(unit_tests diff --git a/tests/cleandbs.sh b/tests/cleandbs.sh index 88e4676c..8515a264 100755 --- a/tests/cleandbs.sh +++ b/tests/cleandbs.sh @@ -6,7 +6,8 @@ rm -r dbs rm -r temp rm -r videos_tests rm -r vdms -rm images/tdb_to_jpg.jpg -rm images/tdb_to_png.png -rm images/test_image.jpg +rm test_images/tdb_to_jpg.jpg +rm test_images/tdb_to_png.png +rm test_images/test_image.jpg +rm test_images/test_image.jpg rm -r backups diff --git a/tests/python/TestCommand.py b/tests/python/TestCommand.py index c151a53b..bb1e0e3b 100644 --- a/tests/python/TestCommand.py +++ b/tests/python/TestCommand.py @@ -39,7 +39,7 @@ def __init__(self, *args, **kwargs): # VDMS Server Info self.hostname = "localhost" - self.port = 55557 + self.port = 55558 db_up = False attempts = 0 diff --git a/tests/python/config-tests.json b/tests/python/config-tests.json index fb1d3077..4c6961b7 100644 --- a/tests/python/config-tests.json +++ b/tests/python/config-tests.json @@ -3,7 +3,7 @@ // Sets database paths and other parameters { // Network - "port": 55557, + "port": 55558, "db_root_path": "test_db", "more-info": "github.com/IntelLabs/vdms" diff --git a/tests/server/AddFindUpdate_blob.json b/tests/server/AddFindUpdate_blob.json new file mode 100644 index 00000000..664acda0 --- /dev/null +++ b/tests/server/AddFindUpdate_blob.json @@ -0,0 +1,39 @@ +[ + { + "AddBlob": + { + + "_ref": 12, + + "properties": { + "Name":"blob-sample-1", + "colored": "true", + "file":"audio" + } + } + + }, + { + "UpdateBlob" : { + + "constraints": { + "Name" : [ "==", "blob-sample-1" ] + }, + + "properties": { + "colored" : "false", + "length" : 200 + } + } + }, + { + "FindBlob" : { + "constraints" : { + "Name" : [ "==", "blob-sample-1" ] + }, + "results" : { + "list" : [ "Name" ] + } + } + } +] diff --git a/tests/server/json_queries.cc b/tests/server/json_queries.cc index 6132119b..b6077871 100644 --- a/tests/server/json_queries.cc +++ b/tests/server/json_queries.cc @@ -63,25 +63,25 @@ std::string singleAddImage(" \ "); TEST( AutoReplicate, default_replicate) { - + VDMSConfig::init("server/config-auto-replicate-tests.json"); PMGDQueryHandler::init(); QueryHandler::init(); std::string backup_path ="backups"; - std::string db_path="db_backup"; + std::string db_path="db_backup"; int port =55555; QueryHandler qh_base; qh_base.regualar_run_autoreplicate(backup_path, db_path, port); // set flag to show autodelete queue has been initialized QueryHandlerTester query_handler(qh_base); - + VDMSConfig::destroy(); PMGDQueryHandler::destroy(); -} +} TEST(AddImage, simpleAdd) @@ -413,7 +413,7 @@ TEST(QueryHandler, EmptyResultCheck) PMGDQueryHandler::init(); QueryHandler::init(); - QueryHandler qh_base; + QueryHandler qh_base; qh_base.reset_autodelete_init_flag(); // set flag to show autodelete queue has been initialized QueryHandlerTester query_handler(qh_base); @@ -633,3 +633,67 @@ TEST(QueryHandler, CustomFunctionNoProcess) VDMSConfig::destroy(); PMGDQueryHandler::destroy(); } + + +TEST(QueryHandler, AddUpdateFind_Blob) +{ + + Json::StyledWriter writer; + + std::ifstream ifile; + int fsize; + char * inBuf; + ifile.open("server/AddFindUpdate_blob.json", std::ifstream::in); + ifile.seekg(0, std::ios::end); + fsize = (int)ifile.tellg(); + ifile.seekg(0, std::ios::beg); + inBuf = new char[fsize]; + ifile.read(inBuf, fsize); + std::string json_query = std::string(inBuf); + ifile.close(); + delete[] inBuf; + + Json::Reader reader; + Json::Value root; + Json::Value parsed; + + VDMSConfig::init("unit_tests/config-tests.json"); + PMGDQueryHandler::init(); + QueryHandler::init(); + + QueryHandler qh_base; + qh_base.reset_autodelete_init_flag(); // set flag to show autodelete queue has been initialized + QueryHandlerTester query_handler(qh_base); + + VDMS::protobufs::queryMessage proto_query; + proto_query.set_json(json_query); + + std::string image; + std::ifstream file("test_images/brain.png", + std::ios::in | std::ios::binary | std::ios::ate); + + image.resize(file.tellg()); + + file.seekg(0, std::ios::beg); + if( !file.read(&image[ 0 ], image.size())) + std::cout << "error" << std::endl; + + proto_query.add_blobs(image); + VDMS::protobufs::queryMessage response; + + query_handler.pq(proto_query, response ); + + reader.parse(response.json().c_str(), parsed); + // std::cout << writer.write(parsed) << std::endl; + + // Verify results returned. + for (int j = 0; j < parsed.size(); j++) { + const Json::Value& query = parsed[j]; + ASSERT_EQ(query.getMemberNames().size(), 1); + std::string cmd = query.getMemberNames()[0]; + EXPECT_EQ(query[cmd]["status"].asInt(), 0); + } + + VDMSConfig::destroy(); + PMGDQueryHandler::destroy(); +} diff --git a/tests/images/large1.jpg b/tests/test_images/large1.jpg similarity index 100% rename from tests/images/large1.jpg rename to tests/test_images/large1.jpg diff --git a/tests/unit_tests/Image_test.cc b/tests/unit_tests/Image_test.cc index fcc8cef8..c48be40e 100644 --- a/tests/unit_tests/Image_test.cc +++ b/tests/unit_tests/Image_test.cc @@ -44,7 +44,7 @@ class ImageTest : public ::testing::Test { protected: virtual void SetUp() { - img_ = "images/large1.jpg"; + img_ = "test_images/large1.jpg"; tdb_img_ = "tdb/test_image.tdb"; cv_img_ = cv::imread(img_, -1); @@ -215,7 +215,7 @@ TEST_F(ImageTest, MatConstructor) TEST_F(ImageTest, EncodedBufferConstructor) { - std::fstream jpgimage("images/large1.jpg"); + std::fstream jpgimage("test_images/large1.jpg"); jpgimage.seekg(0, jpgimage.end); int length = jpgimage.tellg(); @@ -538,19 +538,19 @@ TEST_F(ImageTest, Read) VCL::ImageTest img_data; img_data.set_format("jpg"); - ASSERT_THROW(img_data.read("images/.jpg"), VCL::Exception); + ASSERT_THROW(img_data.read("test_images/.jpg"), VCL::Exception); - img_data.read("images/large1"); + img_data.read("test_images/large1"); - EXPECT_EQ("images/large1.jpg", img_data.get_image_id()); + EXPECT_EQ("test_images/large1.jpg", img_data.get_image_id()); } TEST_F(ImageTest, WriteMatToJPG) { VCL::Image img(cv_img_); - img.store("images/test_image", VCL::Image::Format::JPG); + img.store("test_images/test_image", VCL::Image::Format::JPG); - cv::Mat test = cv::imread("images/test_image.jpg"); + cv::Mat test = cv::imread("test_images/test_image.jpg"); EXPECT_FALSE( test.empty() ); } @@ -786,14 +786,14 @@ TEST_F(ImageTest, TDBToPNG) { VCL::Image img(tdb_img_); - img.store("images/tdb_to_png", VCL::Image::Format::PNG); + img.store("test_images/tdb_to_png", VCL::Image::Format::PNG); } TEST_F(ImageTest, TDBToJPG) { VCL::Image img(tdb_img_); - img.store("images/tdb_to_jpg", VCL::Image::Format::JPG); + img.store("test_images/tdb_to_jpg", VCL::Image::Format::JPG); } TEST_F(ImageTest, EncodedImage) diff --git a/tests/unit_tests/TDBImage_test.cc b/tests/unit_tests/TDBImage_test.cc index ed8e31b9..2b9097f2 100644 --- a/tests/unit_tests/TDBImage_test.cc +++ b/tests/unit_tests/TDBImage_test.cc @@ -43,7 +43,7 @@ class TDBImageTest : public ::testing::Test { virtual void SetUp() { tdb_img_ = "tdb/test_image.tdb"; tdb_test_ = "tdb/write_test.tdb"; - cv_img_ = cv::imread("images/large1.jpg", cv::IMREAD_ANYCOLOR); + cv_img_ = cv::imread("test_images/large1.jpg", cv::IMREAD_ANYCOLOR); rect_ = VCL::Rectangle(100, 100, 100, 100); } diff --git a/tests/unit_tests/client_blob.cc b/tests/unit_tests/client_blob.cc new file mode 100644 index 00000000..190478ce --- /dev/null +++ b/tests/unit_tests/client_blob.cc @@ -0,0 +1,48 @@ +#include "meta_data_helper.h" +TEST(BLOB, add_Blob){ + std::string filename ="../tests/test_images/large1.jpg"; + std::vector blobs; + + Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); + meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); + Json::Value tuple ; + tuple=meta_obj->construct_Blob(); + + + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); + Json::Value result; + + meta_obj->_reader.parse(response.json.c_str(), result); + int status1 = result[0]["AddBlob"]["status"].asInt(); + EXPECT_EQ(status1, 0); +} + +TEST(BLOB, update_Blob){ + + Meta_Data* meta_obj=new Meta_Data(); + meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); + Json::Value tuple ; + tuple=meta_obj->construct_updateBlob(); + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple)); + Json::Value result; + + meta_obj->_reader.parse(response.json.c_str(), result); + int status1 = result[0]["status"].asInt(); + + EXPECT_EQ(status1, 0); +} +TEST(BLOB, find_Blob){ + + Meta_Data* meta_obj=new Meta_Data(); + meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); + Json::Value tuple ; + tuple=meta_obj->construct_findBlob(); + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple)); + Json::Value result; + + meta_obj->_reader.parse(response.json.c_str(), result); + int status1 = result[0]["status"].asInt(); + + EXPECT_EQ(status1, 0); +} \ No newline at end of file diff --git a/tests/unit_tests/client_bounding_box.cc b/tests/unit_tests/client_bounding_box.cc index ae85ab86..0bde4d75 100644 --- a/tests/unit_tests/client_bounding_box.cc +++ b/tests/unit_tests/client_bounding_box.cc @@ -14,23 +14,12 @@ TEST(CLIENT_CPP, add_BB){ } TEST(CLIENT_CPP, add_BB_with_image){ - std::fstream jpgimage("../tests/images/large1.jpg"); - jpgimage.seekg(0, jpgimage.end); - int length = jpgimage.tellg(); - // std::cout<<"Length " < blobs; - - std::string *bytes_str = new std::string(buffer); - - blobs.push_back(bytes_str); - Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_BB(true); @@ -38,7 +27,8 @@ TEST(CLIENT_CPP, add_BB_with_image){ VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); + // std::cout << result < blobs; - - std::string *bytes_str = new std::string(image); - - blobs.push_back(bytes_str); - Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_image(); - + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); @@ -38,10 +27,10 @@ TEST(CLIENT_CPP, add_image){ TEST(CLIENT_CPP, add_image_resize_operation){ - + std::string image; - std::fstream file("../tests/images/large1.jpg", + std::fstream file("../tests/test_images/large1.jpg", std::ios::in | std::ios::binary | std::ios::ate); image.resize(file.tellg()); @@ -51,42 +40,42 @@ TEST(CLIENT_CPP, add_image_resize_operation){ std::cout << "error" << std::endl; std::vector blobs; - + std::string *bytes_str = new std::string(image); - + blobs.push_back(bytes_str); Json::Value op; op["type"]="resize"; op["width"]=100; - op["height"]=100; + op["height"]=100; Meta_Data* meta_obj=new Meta_Data(); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_image(true, op); - - + + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); - + int status1 = result[0]["AddImage"]["status"].asInt(); EXPECT_EQ(status1, 0); } TEST(CLIENT_CPP, find_image){ - + Meta_Data* meta_obj=new Meta_Data(); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->construct_find_image(); - - + + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple)); Json::Value result; meta_obj->_reader.parse(response.json.c_str(), result); - + int status1 = result[0]["FindImage"]["status"].asInt(); EXPECT_EQ(status1, 0); diff --git a/tests/unit_tests/client_videos.cc b/tests/unit_tests/client_videos.cc index 20c18f39..35c56a22 100644 --- a/tests/unit_tests/client_videos.cc +++ b/tests/unit_tests/client_videos.cc @@ -32,31 +32,15 @@ TEST(CLIENT_CPP, add_single_video){ std::vector blobs; - const char *_video_id ="../tests/videos/Megamind.avi"; - std::ifstream ifile; - ifile.open(_video_id); - - int fsize; - char* inBuf; - ifile.seekg(0, std::ios::end); - fsize = (long)ifile.tellg(); - ifile.seekg(0, std::ios::beg); - inBuf = new char[fsize]; - ifile.read(inBuf, fsize); - std::string blob = (std::string(inBuf)); - ifile.close(); - delete[] inBuf; - - - std::string* bytes_str =new std::string(blob); - blobs.push_back(bytes_str); - Meta_Data* meta_obj=new Meta_Data(); + std::string filename ="../tests/videos/Megamind.avi"; + + + Meta_Data* meta_obj=new Meta_Data(); + blobs.push_back(meta_obj->read_blob(filename)); meta_obj->_aclient.reset ( new VDMS::VDMSClient(meta_obj->get_server(), meta_obj->get_port())); Json::Value tuple ; tuple=meta_obj->constuct_video(false); - - - std::cout<< "Printing bytes_str " << bytes_str << "\t"<< blobs[0] << std::endl; + VDMS::Response response =meta_obj->_aclient->query(meta_obj->_fastwriter.write(tuple), blobs); Json::Value result; diff --git a/tests/unit_tests/meta_data.cc b/tests/unit_tests/meta_data.cc index 11edd7bb..9ddb3b84 100644 --- a/tests/unit_tests/meta_data.cc +++ b/tests/unit_tests/meta_data.cc @@ -174,6 +174,77 @@ Json::Value Meta_Data::constuct_image(bool add_operation, Json::Value operations } +std::string* Meta_Data::read_blob(std::string& fname){ + std::string video; + std::ifstream video_file(fname, + std::ios::in | std::ios::binary | std::ios::ate); + + video.resize(video_file.tellg()); + + video_file.seekg(0, std::ios::beg); + if( !video_file.read(&video[ 0 ], video.size())) + std::cout << "error" << std::endl; + std::string* bytes_str =new std::string(video); + // std::cout << *bytes_str < _aclient; std::string _server_name="localhost"; - int _port =55557; + int _port =55555; Json::FastWriter _fastwriter; Json::Reader _reader; @@ -34,8 +34,12 @@ class Meta_Data{ Json::Value construct_add_query(int ref, bool const_on, bool experiation); Json::Value construct_add_area(int ref, bool const_on); Json::Value construct_add_connection(int ref1, int ref2, bool const_on); - Json::Value construct_find_entity(bool ,bool); + Json::Value construct_find_entity(bool ,bool ); Json::Value constuct_BB(bool); + Json::Value construct_Blob(); + Json::Value construct_updateBlob(); + Json::Value construct_findBlob(); + std::string* read_blob(std::string&); Json::Value constuct_image(bool =false, Json::Value operations={}); Json::Value constuct_video(bool =false); Json::Value construct_find_image(); diff --git a/utils/src/api_schema/api_schema.json b/utils/src/api_schema/api_schema.json index 26ea8347..cafcb948 100644 --- a/utils/src/api_schema/api_schema.json +++ b/utils/src/api_schema/api_schema.json @@ -59,7 +59,11 @@ { "$ref": "#/definitions/AddVideoTop" }, { "$ref": "#/definitions/UpdateVideoTop" }, { "$ref": "#/definitions/FindVideoTop" }, - { "$ref": "#/definitions/FindFramesTop" } + { "$ref": "#/definitions/FindFramesTop" }, + + { "$ref": "#/definitions/AddBlobTop" }, + { "$ref": "#/definitions/UpdateBlobTop" }, + { "$ref": "#/definitions/FindBlobTop" } ] }, "uniqueItems": false, @@ -451,6 +455,26 @@ "additionalProperties": false }, + "AddBlobTop": { + "properties": { + "AddBlob" : { "type": "object", "$ref": "#/definitions/AddBlob" } + }, + "additionalProperties": false + }, + "UpdateBlobTop": { + "properties": { + "UpdateBlob" : { "type": "object", "$ref": "#/definitions/UpdateBlob" } + }, + "additionalProperties": false + }, + + "FindBlobTop": { + "properties": { + "FindBlob" : { "type": "object", "$ref": "#/definitions/FindBlob" } + }, + "additionalProperties": false + }, + "AddVideoTop": { "properties": { "AddVideo" : { "type": "object", "$ref": "#/definitions/AddVideo" } @@ -458,6 +482,9 @@ "additionalProperties": false }, + + + "UpdateVideoTop": { "properties": { "UpdateVideo" : { "type": "object", "$ref": "#/definitions/UpdateVideo" } @@ -685,6 +712,41 @@ "additionalProperties": false }, + "AddBlob": { + "properties": { + "_ref": { "$ref": "#/definitions/refInt" }, + "from_server_file": { "type": "string"}, + "link": { "$ref": "#/definitions/blockLink" }, + "properties": { "type": "object" } + + }, + "additionalProperties": false + }, + + "UpdateBlob": { + "properties": { + "_ref": { "$ref": "#/definitions/refInt" }, + "properties": { "type": "object" }, + "remove_props": { "$ref": "#/definitions/stringArray" }, + "constraints": { "type": "object" } + }, + "additionalProperties": false + }, + + "FindBlob": { + "properties": { + "_ref": { "$ref": "#/definitions/refInt" }, + "link": { "$ref": "#/definitions/blockLink" }, + "constraints": { "type": "object" }, + "results": { "$ref": "#/definitions/blockResults" }, + "unique": { "type": "boolean" } + }, + + "additionalProperties": false + }, + + + "AddVideo": { "properties": { "_ref": { "$ref": "#/definitions/refInt" }, @@ -698,7 +760,7 @@ }, "additionalProperties": false }, - + "UpdateVideo": { "properties": { "_ref": { "$ref": "#/definitions/refInt" },