Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix find desc (#43, #91) #97

Merged
merged 5 commits into from
Apr 3, 2019
Merged

Fix find desc (#43, #91) #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
64de98c
Add missing link to FindDescriptor
luisremis Mar 29, 2019
623c2a1
Fix FindDesc to avoid always returning id and label (#43)
luisremis Mar 29, 2019
d734a82
Fix FindDesc by constraint with blob=true (#91)
luisremis Mar 29, 2019
51ce4b1
Add more testing for FindDesc
luisremis Mar 29, 2019
3379ab4
Fix blob check condition on FindDesc
luisremis Apr 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 69 additions & 17 deletions src/DescriptorsCommand.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,11 +540,11 @@ int FindDescriptor::construct_protobuf(

// Query for the Descriptors related to user-defined link
// that match the user-defined constraints
// We will need to the the AND operation
// We will need to do the AND operation
// on the construct_response.

int desc_ref = get_value<int>(cmd, "_ref",
query.get_available_reference());
query.get_available_reference());

query.QueryNode(
desc_ref,
Expand Down Expand Up @@ -668,16 +668,31 @@ int FindDescriptor::construct_protobuf(
return 0;
}

void FindDescriptor::convert_properties(Json::Value& entities)
void FindDescriptor::convert_properties(Json::Value& entities,
Json::Value& list)
{
bool flag_label = false;
bool flag_id = false;

for (auto& prop : list) {
if (prop.asString() == "_label") {
flag_label = true;
}
if (prop.asString() == "_id") {
flag_id = true;
}
}

for (auto& element : entities) {

if (element.isMember(VDMS_DESC_LABEL_PROP)) {
element["_label"] = element[VDMS_DESC_LABEL_PROP];
if (flag_label)
element["_label"] = element[VDMS_DESC_LABEL_PROP];
element.removeMember(VDMS_DESC_LABEL_PROP);
}
if (element.isMember(VDMS_DESC_ID_PROP)) {
element["_id"] = element[VDMS_DESC_ID_PROP];
if (flag_id)
element["_id"] = element[VDMS_DESC_ID_PROP];
element.removeMember(VDMS_DESC_ID_PROP);
}
}
Expand Down Expand Up @@ -711,6 +726,7 @@ Json::Value FindDescriptor::construct_responses(
}

const Json::Value& results = cmd["results"];
Json::Value list = get_value<Json::Value>(results, "list");

// Case (1)
if (cmd.isMember("link")) {
Expand All @@ -719,6 +735,10 @@ Json::Value FindDescriptor::construct_responses(

findDesc = json_responses[0];

if (findDesc.isMember("entities")) {
convert_properties(findDesc["entities"], list);
}

if (findDesc["status"] != 0) {
Json::Value return_error;
return_error["status"] = RSCommand::Error;
Expand All @@ -731,10 +751,44 @@ Json::Value FindDescriptor::construct_responses(

assert(json_responses.size() == 2);

const Json::Value& set_response = json_responses[0];
const Json::Value& set = set_response["entities"][0];

// These properties should always exist
assert(set.isMember(VDMS_DESC_SET_PATH_PROP));
assert(set.isMember(VDMS_DESC_SET_DIM_PROP));
std::string set_path = set[VDMS_DESC_SET_PATH_PROP].asString();
int dim = set[VDMS_DESC_SET_DIM_PROP].asInt();

findDesc = json_responses[1];

if (findDesc.isMember("entities")) {
convert_properties(findDesc["entities"]);

if (get_value<bool>(results, "blob", false)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Our wiki page says that blob is assumed true by default. Here it seems false is default. Which one needs correction?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given that descriptors usually are a proxy for retrieving other objects, we decided some time back that the default behavior would be not to return the blob by default. Need a wiki update.


VCL::DescriptorSet* set =
_dm->get_descriptors_handler(set_path);

for (auto& ent : findDesc["entities"]) {
long id = ent[VDMS_DESC_ID_PROP].asInt64();

try {
std::string* desc_blob = query_res.add_blobs();
desc_blob->resize(sizeof(float) * dim);

set->get_descriptors(&id, 1,
(float*)(*desc_blob).data());

} catch (VCL::Exception e) {
print_exception(e);
findDesc["status"] = RSCommand::Error;
findDesc["info"] = "VCL Exception";
return error(findDesc);
}
}
}

convert_properties(findDesc["entities"], list);
}

if (findDesc["status"] != 0) {
Expand Down Expand Up @@ -783,14 +837,13 @@ Json::Value FindDescriptor::construct_responses(
std::vector<float>* distances;

bool compute_distance = false;
if (results.isMember("list")) {

for (int i = 0; i < results["list"].size(); ++i) {
if (results["list"][i].asString() == "_distance") {
compute_distance = true;
break;
}
Json::Value list = get_value<Json::Value>(results, "list");

for (auto& prop : list) {
if (prop.asString() == "_distance") {
compute_distance = true;
break;
}
}

Expand All @@ -817,8 +870,6 @@ Json::Value FindDescriptor::construct_responses(
Json::Value entities = findDesc["entities"];
findDesc.removeMember("entities");

convert_properties(entities);

for (int i = 0; i < (*ids).size(); ++i) {

Json::Value desc_data;
Expand All @@ -827,7 +878,7 @@ Json::Value FindDescriptor::construct_responses(
bool pass_constraints = false;

for (auto ent : entities) {
if (ent["_id"].asInt64() == d_id) {
if (ent[VDMS_DESC_ID_PROP].asInt64() == d_id) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like some cleanup mixed in with the actual changes. Just make sure they aren't numerous enough to warrant a separate commit. Otherwise its ok.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is actually needed here in this commit as the "convertproperties" function (that was doing this translation internally) was moved below.

desc_data = ent;
pass_constraints = true;
break;
Expand All @@ -847,7 +898,7 @@ Json::Value FindDescriptor::construct_responses(
// desc_data["cache_id"] = Json::Int64((*ids)[i]);
}

if (results.isMember("blob")) {
if (get_value<bool>(results, "blob", false)) {

desc_data["blob"] = true;

Expand All @@ -867,12 +918,13 @@ Json::Value FindDescriptor::construct_responses(
findDesc["info"] = "VCL Exception";
return error(findDesc);
}

}

findDesc["entities"].append(desc_data);
}

convert_properties(findDesc["entities"], list);

if (cache.isMember("cache_obj_id")) {
// We remove the vectors associated with that entry to
// free memory, without removing the entry from _cache_map
Expand Down
2 changes: 1 addition & 1 deletion src/DescriptorsCommand.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ namespace VDMS{
{

private:
void convert_properties(Json::Value& entities);
void convert_properties(Json::Value& entities, Json::Value& list);

public:
FindDescriptor();
Expand Down
82 changes: 81 additions & 1 deletion tests/python/TestFindDescriptors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,86 @@ def test_findDescByConst_get_id(self):
self.assertEqual(response[0]["FindDescriptor"]
["entities"][0]["myid"], 205)

def test_findDescByConst_blobTrue(self):

# Add Set
set_name = "features_128d_4_findDescriptors_id_blob"
dims = 128
total = 100
self.create_set_and_insert(set_name, dims, total)

db = self.create_connection()

all_queries = []

finddescriptor = {}
finddescriptor["set"] = set_name

constraints = {}
constraints["myid"] = ["==", 205]
finddescriptor["constraints"] = constraints

results = {}
results["list"] = ["myid", "_label", "_id"]
results["blob"] = True
finddescriptor["results"] = results

query = {}
query["FindDescriptor"] = finddescriptor

all_queries = []
all_queries.append(query)

response, fv_array = db.query(all_queries)

# Check success
self.assertEqual(response[0]["FindDescriptor"]["status"], 0)
self.assertEqual(response[0]["FindDescriptor"]["returned"], 1)
self.assertEqual(response[0]["FindDescriptor"]
["entities"][0]["myid"], 205)
self.assertEqual(len(fv_array), 1)
self.assertEqual(len(fv_array[0]), dims*4)

def test_findDescByConst_multiple_blobTrue(self):

# Add Set
set_name = "features_128d_4_findDescriptors_m_blob"
dims = 128
total = 100
self.create_set_and_insert(set_name, dims, total)

db = self.create_connection()

all_queries = []

finddescriptor = {}
finddescriptor["set"] = set_name

constraints = {}
constraints["myid"] = ["<=", 205]
finddescriptor["constraints"] = constraints

results = {}
results["list"] = ["myid"]
results["blob"] = True
finddescriptor["results"] = results

query = {}
query["FindDescriptor"] = finddescriptor

all_queries = []
all_queries.append(query)

response, fv_array = db.query(all_queries)

# Check success
self.assertEqual(response[0]["FindDescriptor"]["status"], 0)
self.assertEqual(response[0]["FindDescriptor"]["returned"], 6)
self.assertEqual(response[0]["FindDescriptor"]
["entities"][5]["myid"], 200)
self.assertEqual(len(fv_array), 6)
self.assertEqual(len(fv_array[0]), dims*4)

def test_findDescByBlob(self):

# Add Set
Expand Down Expand Up @@ -416,7 +496,6 @@ def test_findDescByBlobUnusedRef(self):
# descriptor_blob.append(x.tobytes())

# response, blob_array = db.query(all_queries, [descriptor_blob])
# print(db.get_last_response_str())

# self.assertEqual(len(blob_array), kn)
# self.assertEqual(descriptor_blob[0], blob_array[0])
Expand Down Expand Up @@ -553,6 +632,7 @@ def test_findDescByBlobWithLink(self):

response, blob_array = db.query(all_queries, [descriptor_blob])


self.assertEqual(len(blob_array), kn)
# This checks that the received blobs is the same as the inserted.
self.assertEqual(descriptor_blob[0], blob_array[0])
Expand Down
1 change: 1 addition & 0 deletions utils/src/api_schema/api_schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,7 @@
"_ref": { "$ref": "#/definitions/refInt" },
"k_neighbors": { "$ref": "#/definitions/positiveInt" },
"results": { "$ref": "#/definitions/blockResults" },
"link": { "$ref": "#/definitions/blockLink" },
"constraints": { "type": "object" },
"properties": { "type": "object" }
},
Expand Down