From 6c38459e11235e2fb10c40607a4eeccbc72d208d Mon Sep 17 00:00:00 2001 From: Luis Remis Date: Fri, 28 Jun 2019 10:49:03 -0700 Subject: [PATCH 1/4] Add method for parsing Image format, make enqueue_operations method public Signed-off-by: mahircg --- src/ImageCommand.cc | 35 +++++++++++++++++++++++------------ src/ImageCommand.h | 12 ++++++++---- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/ImageCommand.cc b/src/ImageCommand.cc index 65270eac..738c58bb 100644 --- a/src/ImageCommand.cc +++ b/src/ImageCommand.cc @@ -76,6 +76,25 @@ void ImageCommand::enqueue_operations(VCL::Image& img, const Json::Value& ops) } } +VCL::Image::Format ImageCommand::get_requested_format(const Json::Value& cmd) +{ + VCL::Image::Format format; + + std::string requested_format = get_value(cmd, "format", ""); + + if (requested_format == "png") { + return VCL::Image::Format::PNG; + } + if (requested_format == "jpg") { + return VCL::Image::Format::JPG; + } + if (requested_format == "tdb") { + return VCL::Image::Format::TDB; + } + + return VCL::Image::Format::NONE_IMAGE; +} + //========= AddImage definitions ========= AddImage::AddImage() : ImageCommand("AddImage") @@ -274,19 +293,12 @@ Json::Value FindImage::construct_responses( img.get_image_format() : VCL::Image::Format::PNG; if (cmd.isMember("format")) { - std::string requested_format = - get_value(cmd, "format"); - - if (requested_format == "png") { - format = VCL::Image::Format::PNG; - } - else if (requested_format == "jpg") { - format = VCL::Image::Format::JPG; - } - else { + format = get_requested_format(cmd); + if (format == VCL::Image::Format::NONE_IMAGE || + format == VCL::Image::Format::TDB) { Json::Value return_error; return_error["status"] = RSCommand::Error; - return_error["info"] = "Invalid Format for FindImage"; + return_error["info"] = "Invalid Requested Format for FindImage"; return error(return_error); } } @@ -318,7 +330,6 @@ Json::Value FindImage::construct_responses( } } - if (flag_empty) { findImage.removeMember("entities"); } diff --git a/src/ImageCommand.h b/src/ImageCommand.h index c6a80249..a1ef89b9 100644 --- a/src/ImageCommand.h +++ b/src/ImageCommand.h @@ -44,10 +44,6 @@ namespace VDMS { class ImageCommand: public RSCommand { - - protected: - void enqueue_operations(VCL::Image& img, const Json::Value& op); - public: ImageCommand(const std::string &cmd_name); @@ -59,6 +55,14 @@ namespace VDMS { Json::Value& error) = 0; virtual bool need_blob(const Json::Value& cmd) { return false; } + + // We use this function for enqueueing operations for an 'Image' object + // that is allocated outside of <*>Image operations + void enqueue_operations(VCL::Image& img, const Json::Value& op); + + // Checks if 'format' parameter is specified, and if so, returns the + // corresponding VCL::Image::Format type. + VCL::Image::Format get_requested_format(const Json::Value& cmd); }; class AddImage: public ImageCommand From 96a623c27bdba538081c506ed6d29ea63a8afcad Mon Sep 17 00:00:00 2001 From: Luis Remis Date: Tue, 2 Jul 2019 13:23:59 -0700 Subject: [PATCH 2/4] Fix bug in VideoCommand return message --- src/VideoCommand.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VideoCommand.cc b/src/VideoCommand.cc index 17bdc77e..b415f692 100644 --- a/src/VideoCommand.cc +++ b/src/VideoCommand.cc @@ -339,7 +339,7 @@ Json::Value FindVideo::construct_responses( Json::Value return_error; return_error["status"] = RSCommand::Error; return_error["info"] = "VCL Exception"; - error(return_error); + return error(return_error); } } From 746cff4be0d47bf925280c7693e1f0dc4293bed6 Mon Sep 17 00:00:00 2001 From: Luis Remis Date: Fri, 28 Jun 2019 10:49:47 -0700 Subject: [PATCH 3/4] Add FindFrames command Signed-off-by: mahircg --- src/QueryHandler.cc | 1 + src/VideoCommand.cc | 250 +++++++++++++++++++++++++-- src/VideoCommand.h | 27 +++ utils/src/api_schema/api_schema.json | 46 ++++- 4 files changed, 309 insertions(+), 15 deletions(-) diff --git a/src/QueryHandler.cc b/src/QueryHandler.cc index e332d5ee..b797ce92 100644 --- a/src/QueryHandler.cc +++ b/src/QueryHandler.cc @@ -85,6 +85,7 @@ void QueryHandler::init() _rs_cmds["AddVideo"] = new AddVideo(); _rs_cmds["UpdateVideo"] = new UpdateVideo(); _rs_cmds["FindVideo"] = new FindVideo(); + _rs_cmds["FindFrames"] = new FindFrames(); // Load the string containing the schema (api_schema/APISchema.h) Json::Reader reader; diff --git a/src/VideoCommand.cc b/src/VideoCommand.cc index b415f692..0ef783d6 100644 --- a/src/VideoCommand.cc +++ b/src/VideoCommand.cc @@ -32,6 +32,7 @@ #include #include +#include "ImageCommand.h" // for enqueue_operations of Image type #include "VideoCommand.h" #include "VDMSConfig.h" #include "defines.h" @@ -95,6 +96,26 @@ VCL::Video::Codec VideoCommand::string_to_codec(const std::string& codec) return VCL::Video::Codec::NOCODEC; } +Json::Value VideoCommand::check_responses(Json::Value& responses) +{ + if (responses.size() != 1) { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "PMGD Response Bad Size"; + return return_error; + } + + Json::Value& response = responses[0]; + + if (response["status"] != 0) { + response["status"] = RSCommand::Error; + // Uses PMGD info error. + return response; + } + + return response; +} + //========= AddVideo definitions ========= AddVideo::AddVideo() : VideoCommand("AddVideo") @@ -152,7 +173,19 @@ int AddVideo::construct_protobuf( return 0; } -//========= UpdateImage definitions ========= +Json::Value AddVideo::construct_responses( + Json::Value& response, + const Json::Value& json, + protobufs::queryMessage &query_res, + const std::string& blob) +{ + Json::Value ret; + ret[_cmd_name] = RSCommand::check_responses(response); + + return ret; +} + +//========= UpdateVideo definitions ========= UpdateVideo::UpdateVideo() : VideoCommand("UpdateVideo") { @@ -251,23 +284,13 @@ Json::Value FindVideo::construct_responses( return ret; }; - if (responses.size() != 1) { - Json::Value return_error; - return_error["status"] = RSCommand::Error; - return_error["info"] = "PMGD Response Bad Size"; - error(return_error); + Json::Value resp = check_responses(responses); + if (resp["status"] != RSCommand::Success) { + return error(resp); } Json::Value& FindVideo = responses[0]; - assert(FindVideo.isMember("entities")); - - if (FindVideo["status"] != 0) { - FindVideo["status"] = RSCommand::Error; - // Uses PMGD info error. - error(FindVideo); - } - bool flag_empty = true; for (auto& ent : FindVideo["entities"]) { @@ -350,3 +373,202 @@ Json::Value FindVideo::construct_responses( ret[_cmd_name].swap(FindVideo); return ret; } + +//========= FindFrames definitions ========= + +FindFrames::FindFrames() : VideoCommand("FindFrames") +{ +} + +bool FindFrames::get_interval_index (const Json::Value& cmd, + Json::ArrayIndex& op_index) +{ + if (cmd.isMember("operations")) { + const auto operations = cmd["operations"]; + for (auto i = 0; i < operations.size(); i++) { + const auto op = operations[i]; + const std::string& type = get_value(op, "type"); + if (type == "interval") { + op_index = i; + return true; + } + } + } + return false; +} + +int FindFrames::construct_protobuf( + PMGDQuery& query, + const Json::Value& jsoncmd, + const std::string& blob, + int grp_id, + Json::Value& error) +{ + const Json::Value& cmd = jsoncmd[_cmd_name]; + + // We try to catch the missing attribute error before + // initiating a PMGD query + Json::ArrayIndex tmp; + bool is_interval = get_interval_index(cmd, tmp); + bool is_frames = cmd.isMember("frames"); + + if (!(is_frames != is_interval)) { + error["status"] = RSCommand::Error; + error["info"] = "Either one of 'frames' or 'operations::interval' " + "must be specified"; + return -1; + } + + Json::Value results = get_value(cmd, "results"); + results["list"].append(VDMS_VID_PATH_PROP); + + query.QueryNode( + get_value(cmd, "_ref", -1), + VDMS_VID_TAG, + cmd["link"], + cmd["constraints"], + results, + get_value(cmd, "unique", false) + ); + + return 0; +} + +Json::Value FindFrames::construct_responses( + Json::Value& responses, + const Json::Value& json, + protobufs::queryMessage &query_res, + const std::string &blob) +{ + const Json::Value& cmd = json[_cmd_name]; + + Json::Value ret; + + auto error = [&](Json::Value& res) + { + ret[_cmd_name] = res; + return ret; + }; + + Json::Value resp = check_responses(responses); + if (resp["status"] != RSCommand::Success) { + return error(resp); + } + + Json::Value& FindFrames = responses[0]; + + bool flag_empty = true; + + for (auto& ent : FindFrames["entities"]) { + + std::string video_path = ent[VDMS_VID_PATH_PROP].asString(); + ent.removeMember(VDMS_VID_PATH_PROP); + + if (ent.getMemberNames().size() > 0) { + flag_empty = false; + } + + try { + std::vector frames; + + // Copy of operations is needed, as we pass the operations to + // the enqueue_operations() method of ImageCommands class, and + // it should not include 'interval' operation. + Json::Value operations = cmd["operations"]; + + Json::ArrayIndex interval_idx; + bool is_interval = get_interval_index(cmd, interval_idx); + bool is_frames = cmd.isMember("frames"); + + if (is_frames) { + for (auto& fr : cmd["frames"]) { + frames.push_back(fr.asUInt()); + } + } + else if (is_interval) { + + Json::Value interval_op = operations[interval_idx]; + + int start = get_value(interval_op, "start"); + int stop = get_value(interval_op, "stop"); + int step = get_value(interval_op, "step"); + + for (int i = start; i < stop; i += step) { + frames.push_back(i); + } + + Json::Value deleted; + operations.removeIndex(interval_idx, &deleted); + } + else { + // This should never happen, as we check this condition in + // FindFrames::construct_protobuf(). In case this happens, it + // is better to signal it rather than to continue + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "No 'frames' or 'interval' parameter"; + return error(return_error); + } + + VCL::Video video(video_path); + + // By default, return frames as PNGs + VCL::Image::Format format = VCL::Image::Format::PNG; + + FindImage img_cmd; + + if (cmd.isMember("format")) { + + format = img_cmd.get_requested_format(cmd); + + if (format == VCL::Image::Format::NONE_IMAGE || + format == VCL::Image::Format::TDB) { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "Invalid Return Format for FindFrames"; + return error(return_error); + } + } + + for (auto idx : frames) { + cv::Mat mat = video.get_frame(idx); + VCL::Image img(mat, false); + if (!operations.empty()) { + img_cmd.enqueue_operations(img, operations); + } + + std::vector img_enc; + img_enc = img.get_encoded_image(format); + + if (!img_enc.empty()) { + std::string* img_str = query_res.add_blobs(); + img_str->resize(img_enc.size()); + std::memcpy((void*)img_str->data(), + (void*)img_enc.data(), + img_enc.size()); + } + else { + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "Image Data not found"; + return error(return_error); + } + } + } + + catch (VCL::Exception e) { + print_exception(e); + Json::Value return_error; + return_error["status"] = RSCommand::Error; + return_error["info"] = "VCL Exception"; + return error(return_error); + } + } + + if (flag_empty) { + FindFrames.removeMember("entities"); + } + + ret[_cmd_name].swap(FindFrames); + return ret; +} diff --git a/src/VideoCommand.h b/src/VideoCommand.h index e0900a12..5ea5e56e 100644 --- a/src/VideoCommand.h +++ b/src/VideoCommand.h @@ -49,6 +49,8 @@ namespace VDMS { VCL::Video::Codec string_to_codec(const std::string& codec); + virtual Json::Value check_responses(Json::Value& responses); + public: VideoCommand(const std::string &cmd_name); @@ -77,6 +79,12 @@ namespace VDMS { int grp_id, Json::Value& error); + Json::Value construct_responses( + Json::Value &json_responses, + const Json::Value &json, + protobufs::queryMessage &response, + const std::string &blob); + bool need_blob(const Json::Value& cmd) { return true; } }; @@ -116,4 +124,23 @@ namespace VDMS { const std::string &blob); }; + class FindFrames: public VideoCommand + { + bool get_interval_index (const Json::Value& cmd, Json::ArrayIndex& op_index); + public: + FindFrames(); + + int construct_protobuf(PMGDQuery& tx, + const Json::Value& root, + const std::string& blob, + int grp_id, + Json::Value& error) override; + + Json::Value construct_responses( + Json::Value &json_responses, + const Json::Value &json, + protobufs::queryMessage &response, + const std::string &blob) override; + }; + }; // namespace VDMS diff --git a/utils/src/api_schema/api_schema.json b/utils/src/api_schema/api_schema.json index c9a47fb0..22f1ceed 100644 --- a/utils/src/api_schema/api_schema.json +++ b/utils/src/api_schema/api_schema.json @@ -57,7 +57,8 @@ { "$ref": "#/definitions/AddVideoTop" }, { "$ref": "#/definitions/UpdateVideoTop" }, - { "$ref": "#/definitions/FindVideoTop" } + { "$ref": "#/definitions/FindVideoTop" }, + { "$ref": "#/definitions/FindFramesTop" } ] }, "uniqueItems": false, @@ -76,6 +77,12 @@ "minimum": 0 }, + "nonNegativeIntArray": { + "type": "array", + "items": {"type": "integer", "minimum": 0}, + "minimum": 1 + }, + "positiveDouble": { "type": "double", "minimum": 0.0 @@ -210,6 +217,22 @@ "uniqueItems": false }, + "blockFrameOperations": { + "type": "array", + "minItems": 1, + "items": { + "anyOf": [ + { "$ref": "#/definitions/operationThreshold" }, + { "$ref": "#/definitions/operationResize" }, + { "$ref": "#/definitions/operationCrop" }, + { "$ref": "#/definitions/operationFlip" }, + { "$ref": "#/definitions/operationRotate" }, + { "$ref": "#/definitions/operationInterval" } + ] + }, + "uniqueItems": false + }, + // Operations "operationInterval": { @@ -430,6 +453,13 @@ "additionalProperties": false }, + "FindFramesTop": { + "properties": { + "FindFrames" : { "type": "object", "$ref": "#/definitions/FindFrames" } + }, + "additionalProperties": false + }, + // Commands "AddEntity": { @@ -664,6 +694,20 @@ "unique": { "type": "boolean" } }, + "additionalProperties": false + }, + + "FindFrames": { + "properties": { + "frames": { "$ref": "#/definitions/nonNegativeIntArray" }, + "_ref": { "$ref": "#/definitions/refInt" }, + "link": { "$ref": "#/definitions/blockLink" }, + "operations": { "$ref": "#/definitions/blockFrameOperations" }, + "format": { "$ref": "#/definitions/imgFormatString" }, + "constraints": { "type": "object" }, + "results": { "$ref": "#/definitions/blockResults" }, + "unique": { "type": "boolean" } + }, "additionalProperties": false } From 5d02db253e760a55299a33ed5b6e87863ec52ed4 Mon Sep 17 00:00:00 2001 From: Luis Remis Date: Fri, 28 Jun 2019 10:50:14 -0700 Subject: [PATCH 4/4] Add testing for FindFrames Signed-off-by: mahircg --- tests/python/TestVideos.py | 130 +++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/python/TestVideos.py b/tests/python/TestVideos.py index 1f7b8546..4f1d1751 100644 --- a/tests/python/TestVideos.py +++ b/tests/python/TestVideos.py @@ -130,6 +130,136 @@ def test_findVideo(self): for i in range(0, number_of_inserts): self.assertEqual(response[i]["FindVideo"]["status"], 0) + def test_FindFramesByFrames(self): + + db = self.create_connection() + + prefix_name = "video_2_" + + number_of_inserts = 2 + + for i in range(0, number_of_inserts): + props = {} + props["name"] = prefix_name + str(i) + self.insertVideo(db, props=props) + + all_queries = [] + + for i in range(0,number_of_inserts): + constraints = {} + constraints["name"] = ["==", prefix_name + str(i)] + + video_params = {} + video_params["constraints"] = constraints + video_params["frames"] = [f for f in range(0, 10)] + + query = {} + query["FindFrames"] = video_params + + all_queries.append(query) + + response, img_array = db.query(all_queries) + + self.assertEqual(response[0]["FindFrames"]["status"], 0) + self.assertEqual(response[1]["FindFrames"]["status"], 0) + self.assertEqual(len(img_array), 2 * len(video_params["frames"]) ) + + def test_FindFramesByInterval(self): + + db = self.create_connection() + + prefix_name = "video_3_" + + number_of_inserts = 2 + + for i in range(0, number_of_inserts): + props = {} + props["name"] = prefix_name + str(i) + self.insertVideo(db, props=props) + + all_queries = [] + + for i in range(0,number_of_inserts): + constraints = {} + constraints["name"] = ["==", prefix_name + str(i)] + + number_of_frames = 10 + operations = [] + interval_operation = {} + interval_operation["type"] = "interval" + interval_operation["start"] = 0 + interval_operation["stop"] = number_of_frames + interval_operation["step"] = 1 + operations.append(interval_operation) + + video_params = {} + video_params["constraints"] = constraints + video_params["operations"] = operations + + query = {} + query["FindFrames"] = video_params + + all_queries.append(query) + + response, img_array = db.query(all_queries) + + self.assertEqual(response[0]["FindFrames"]["status"], 0) + self.assertEqual(response[1]["FindFrames"]["status"], 0) + self.assertEqual(len(img_array), 2 * number_of_frames) + + def test_FindFramesMissingParameters(self): + + db = self.create_connection() + + constraints = {} + constraints["name"] = ["==", "video_1"] + + video_params = {} + video_params["constraints"] = constraints + + query = {} + query["FindFrames"] = video_params + + all_queries = [] + all_queries.append(query) + + response, img = db.query(all_queries) + + self.assertEqual(response[0]["status"], -1) + self.assertEqual(img, []) + + def test_FindFramesInvalidParameters(self): + + db = self.create_connection() + + constraints = {} + constraints["name"] = ["==", "video_1"] + + operations = [] + interval_operation = {} + interval_operation["type"] = "interval" + interval_operation["start"] = 10 + interval_operation["stop"] = 20 + interval_operation["step"] = 1 + operations.append(interval_operation) + + video_params = {} + video_params["constraints"] = constraints + video_params["operations"] = operations + video_params["frames"] = [1] + + + query = {} + query["FindFrames"] = video_params + + all_queries = [] + all_queries.append(query) + + response, img = db.query(all_queries) + + self.assertEqual(response[0]["status"], -1) + self.assertEqual(img, []) + def test_findVideoResults(self): db = self.create_connection()