From bbbf861a4beabf1f17ab07d0e2d9ef5ce85d2004 Mon Sep 17 00:00:00 2001 From: jialipen Date: Sat, 12 Jun 2021 00:55:41 +0800 Subject: [PATCH 1/6] fixes. --- .../src/runtime/reference/multiclass_nms.cpp | 123 ++++++++++++++++-- ngraph/test/backend/multiclass_nms.in.cpp | 97 ++++++++++++-- 2 files changed, 198 insertions(+), 22 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp index f41739f765e219..4099057a2080ec 100644 --- a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp +++ b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -117,6 +117,11 @@ namespace ngraph return score < rhs.score || (score == rhs.score && index > rhs.index); } + inline bool operator>(const BoxInfo& rhs) const + { + return !(score < rhs.score || (score == rhs.score && index > rhs.index)); + } + Rectangle box; int64_t index = 0; int64_t suppress_begin_index = 0; @@ -124,7 +129,37 @@ namespace ngraph int64_t class_index = 0; float score = 0.0f; }; + + inline std::ostream& operator<<(std::ostream& s, const BoxInfo& b) + { + //s << "BoxInfo{"; + s << b.score; + //s << "}"; + return s; + } } // namespace + + template + void print_queue(T q) { // NB: pass by value so the print uses a copy + std::cout << "\n{"; + while(!q.empty()) + { + std::cout << q.top() << ", "; + q.pop(); + } + std::cout << "}\n"; + } + + template + void print_list(T &q) { + std::cout << "\n{"; + for(auto& v : q) + { + std::cout << v << ", "; + } + std::cout << "}\n"; + } + void multiclass_nms(const float* boxes_data, const Shape& boxes_data_shape, const float* scores_data, @@ -157,8 +192,6 @@ namespace ngraph SelectedOutput* selected_scores_ptr = reinterpret_cast(selected_outputs); - size_t boxes_per_class = static_cast(nms_top_k); - std::vector filteredBoxes; for (int64_t batch = 0; batch < num_batches; batch++) @@ -174,7 +207,6 @@ namespace ngraph scores_data + batch * (num_classes * num_boxes) + class_idx * num_boxes; std::vector candidate_boxes; - candidate_boxes.reserve(num_boxes); for (int64_t box_idx = 0; box_idx < num_boxes; box_idx++) { @@ -185,8 +217,33 @@ namespace ngraph } } - std::priority_queue sorted_boxes(std::less(), - std::move(candidate_boxes)); + int candiate_size = candidate_boxes.size(); + + // threshold nms_top_k for each class + if (nms_top_k > -1 && nms_top_k < candiate_size) + { + candiate_size = nms_top_k; + } + + if (candiate_size <= 0) // early drop + { + continue; + } + + // sort by score + std::partial_sort(candidate_boxes.begin(), + candidate_boxes.begin() + candiate_size, + candidate_boxes.end(), + std::greater()); + + print_list(candidate_boxes); + + std::priority_queue sorted_boxes(candidate_boxes.begin(), + candidate_boxes.begin() + candiate_size, std::less()); + + print_list(candidate_boxes); + + print_queue(sorted_boxes); std::vector selected; // Get the next box with top score, filter by iou_threshold @@ -194,7 +251,7 @@ namespace ngraph BoxInfo next_candidate; float original_score; - while (!sorted_boxes.empty() && selected.size() < boxes_per_class) + while (!sorted_boxes.empty()) { next_candidate = sorted_boxes.top(); original_score = next_candidate.score; @@ -241,17 +298,44 @@ namespace ngraph { filteredBoxes.push_back(box_info); } - num_dets += filteredBoxes.size(); + num_dets += selected.size(); + } // for each class + + // threshold keep_top_k for each batch element + if (keep_top_k > -1 && keep_top_k < num_dets) + { + num_dets = nms_top_k; } + if (num_dets <= 0) // early drop + { + continue; + } + *valid_outputs++ = num_dets; - } + } // for each batch element + /* sort */ bool sort_result_across_batch = false; // TODO if (sort_result_across_batch) - { - std::sort(filteredBoxes.begin(), + { /* sort across batch */ + if (sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) + { + std::sort(filteredBoxes.begin(), + filteredBoxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return (l.score > r.score) || + (l.score == r.score && l.batch_index < r.batch_index) || + (l.score == r.score && l.batch_index == r.batch_index && + l.class_index < r.class_index) || + (l.score == r.score && l.batch_index == r.batch_index && + l.class_index == r.class_index && l.index < r.index); + }); + } + else if(sort_result_type == op::v8::MulticlassNms::SortResultType::CLASSID) + { + std::sort(filteredBoxes.begin(), filteredBoxes.end(), [](const BoxInfo& l, const BoxInfo& r) { return (l.score > r.score) || @@ -261,8 +345,27 @@ namespace ngraph (l.score == r.score && l.batch_index == r.batch_index && l.class_index == r.class_index && l.index < r.index); }); + } + } + else + { + /* sort inside batch element */ + if (sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) + { + std::sort(filteredBoxes.begin(), + filteredBoxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return ((l.batch_index == r.batch_index) && + ((l.score >= r.score) || + (l.score == r.score && l.class_index < r.class_index) || + (l.score == r.score && l.class_index == r.class_index && l.index < r.index))); + }); + } + // in case of "NONE" and "CLASSID", pass through } + /* output */ + size_t max_num_of_selected_indices = selected_indices_shape[0]; size_t output_size = std::min(filteredBoxes.size(), max_num_of_selected_indices); diff --git a/ngraph/test/backend/multiclass_nms.in.cpp b/ngraph/test/backend/multiclass_nms.in.cpp index 5162d036ac0e01..a1be2f608390fe 100644 --- a/ngraph/test/backend/multiclass_nms.in.cpp +++ b/ngraph/test/backend/multiclass_nms.in.cpp @@ -30,13 +30,15 @@ using namespace ngraph; static string s_manifest = "${MANIFEST}"; -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_score) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; - std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; const int64_t nms_top_k = 3; const float iou_threshold = 0.5f; @@ -46,8 +48,8 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) const auto background_class = -1; const auto nms_eta = 1.0f; - const auto boxes_shape = Shape{1, 6, 4}; - const auto scores_shape = Shape{1, 1, 6}; + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); @@ -67,9 +69,9 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_outputs = backend->create_tensor(element::f32, Shape{3, 6}); // TODO: dynamic shape - auto selected_indeces = backend->create_tensor(element::i64, Shape{3}); // TODO - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -85,16 +87,87 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) auto selected_indeces_value = read_vector(selected_indeces); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {3, 0, 5}; - std::vector expected_selected_scores = {0.0, 0.95, 0.0, 10.0, 1.0, 11.0, - 0.0, 0.9, 0.0, 0.0, 1.0, 1.0, - 0.0, 0.3, 0.0, 100.0, 1.0, 101.0}; - std::vector expected_valid_outputs = {3}; + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_class_id) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } + /* NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) { From 347c146a841a4325ed40ec9e785e4d5ba19b6128 Mon Sep 17 00:00:00 2001 From: jialipen Date: Tue, 15 Jun 2021 21:44:43 +0800 Subject: [PATCH 2/6] fix: now sort by class_id and score work. --- .../src/runtime/reference/multiclass_nms.cpp | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp index 4099057a2080ec..80a0ad0ced0ce3 100644 --- a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp +++ b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -130,11 +130,26 @@ namespace ngraph float score = 0.0f; }; + inline std::ostream& operator<<(std::ostream& s, const Rectangle& b) + { + s << "Rectangle{"; + s << b.x1 << ", "; + s << b.y1 << ", "; + s << b.x2 << ", "; + s << b.y2; + s << "}"; + return s; + } + inline std::ostream& operator<<(std::ostream& s, const BoxInfo& b) { - //s << "BoxInfo{"; + s << "BoxInfo{"; + s << b.batch_index << ", "; + s << b.class_index << ", "; + s << b.index << ", "; + s << b.box << ", "; s << b.score; - //s << "}"; + s << "}"; return s; } } // namespace @@ -338,12 +353,12 @@ namespace ngraph std::sort(filteredBoxes.begin(), filteredBoxes.end(), [](const BoxInfo& l, const BoxInfo& r) { - return (l.score > r.score) || - (l.score == r.score && l.batch_index < r.batch_index) || - (l.score == r.score && l.batch_index == r.batch_index && - l.class_index < r.class_index) || - (l.score == r.score && l.batch_index == r.batch_index && - l.class_index == r.class_index && l.index < r.index); + return (l.class_index < r.class_index) || + (l.class_index == r.class_index && l.batch_index < r.batch_index) || + (l.class_index == r.class_index && l.batch_index == r.batch_index && + l.score > r.score) || + (l.class_index == r.class_index && l.batch_index == r.batch_index && + l.score == r.score && l.index < r.index); }); } } @@ -356,9 +371,9 @@ namespace ngraph filteredBoxes.end(), [](const BoxInfo& l, const BoxInfo& r) { return ((l.batch_index == r.batch_index) && - ((l.score >= r.score) || - (l.score == r.score && l.class_index < r.class_index) || - (l.score == r.score && l.class_index == r.class_index && l.index < r.index))); + ((l.score > r.score) || + ((std::fabs(l.score - r.score) < 1e-6) && l.class_index < r.class_index) || + ((std::fabs(l.score - r.score) < 1e-6) && l.class_index == r.class_index && l.index < r.index))); }); } // in case of "NONE" and "CLASSID", pass through From 02dacffd895bfaf6aabf0b27b0484071a5cf50ab Mon Sep 17 00:00:00 2001 From: jialipen Date: Wed, 16 Jun 2021 17:09:45 +0800 Subject: [PATCH 3/6] more test cases verified. --- ngraph/test/backend/multiclass_nms.in.cpp | 763 ++++++++++++++-------- 1 file changed, 497 insertions(+), 266 deletions(-) diff --git a/ngraph/test/backend/multiclass_nms.in.cpp b/ngraph/test/backend/multiclass_nms.in.cpp index a1be2f608390fe..0a2a41490e6244 100644 --- a/ngraph/test/backend/multiclass_nms.in.cpp +++ b/ngraph/test/backend/multiclass_nms.in.cpp @@ -168,7 +168,159 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_class_id) EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -/* +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) { std::vector boxes_data = {1.0, 1.0, 0.0, 0.0, 0.0, 0.1, 1.0, 1.1, @@ -177,37 +329,37 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 6, 4}; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 1, M 6 const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{3, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{3, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -216,15 +368,17 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); - auto valid_outputs_value = read_vector(valid_outputs); + auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 0, 0, 5}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 0.0, 0.0, 0.3}; + std::vector expected_selected_indices = {3, 0, 1}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 1.00, 1.00, 0.00, 0.00 , + 0.00, 0.75, 0.00, 0.10, 1.00, 1.10}; std::vector expected_valid_outputs = {3}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); @@ -241,37 +395,37 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) std::vector scores_data = {0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 10, 4}; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 10, 4}; // N 1, C 1, M 10 const auto scores_shape = Shape{1, 1, 10}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -280,15 +434,15 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.9}; + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; std::vector expected_valid_outputs = {1}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); @@ -304,37 +458,37 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 2; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 2; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{2, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{2, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -343,15 +497,16 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9}; + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 }; std::vector expected_valid_outputs = {2}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); @@ -365,37 +520,37 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_single_box) std::vector scores_data = {0.9}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 1, 4}; const auto scores_shape = Shape{1, 1, 1}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -404,15 +559,15 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_single_box) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.9}; + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; std::vector expected_valid_outputs = {1}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); @@ -428,37 +583,37 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.2f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{3, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{3, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -467,16 +622,17 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 0, 0, 5}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 0.0, 0.0, 0.3}; - std::vector expected_valid_outputs = {3}; + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {2}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); @@ -491,37 +647,37 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.4f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.95f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{2, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{2, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -530,63 +686,133 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9}; - std::vector expected_valid_outputs = {2}; + std::vector expected_selected_indices = {3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00}; + std::vector expected_valid_outputs = {1}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_no_output) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 2.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_indeces_value = read_vector(selected_indeces); + auto selected_scores_value = read_vector(selected_outputs); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {}; + std::vector expected_selected_scores = {}; + std::vector expected_valid_outputs = {0}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_background) { - std::vector boxes_data = { - 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, - 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, - 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; std::vector scores_data = { - 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, 0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; - const int64_t max_output_boxes_per_class_data = 2; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{2, 6, 4}; - const auto scores_shape = Shape{2, 1, 6}; + const auto keep_top_k = -1; + const auto background_class = 0; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{4, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{4, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -595,63 +821,71 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 1, 0, 3, 1, 0, 0}; - std::vector expected_selected_scores = { - 0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 1.0, 0.0, 0.95, 1.0, 0.0, 0.9}; - std::vector expected_valid_outputs = {4}; + std::vector expected_selected_indices = {0, 3, 6, 9}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {2, 2}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_classes) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_keep_top_k) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, - 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; std::vector scores_data = { - 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, 0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; - const int64_t max_output_boxes_per_class_data = 2; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 6, 4}; - const auto scores_shape = Shape{1, 2, 6}; + const auto keep_top_k = 3; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{4, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{4, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -660,101 +894,98 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_classes) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 0, 1, 3, 0, 1, 0}; - std::vector expected_selected_scores = { - 0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 0.0, 1.0, 0.95, 0.0, 1.0, 0.9}; - std::vector expected_valid_outputs = {4}; + std::vector expected_selected_indices = {3, 0, 0, + 9, 6, 6}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 }; // 1 + std::vector expected_valid_outputs = {3, 3}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores_without_constants) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_nms_eta) { - std::vector boxes_data = {0.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.1f, 1.0f, 1.1f, - 0.0f, -0.1f, 1.0f, 0.9f, 0.0f, 10.0f, 1.0f, 11.0f, - 0.0f, 10.1f, 1.0f, 11.1f, 0.0f, 100.0f, 1.0f, 101.0f}; + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; - std::vector scores_data = {0.9f, 0.75f, 0.6f, 0.95f, 0.5f, 0.3f}; + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; - std::vector max_output_boxes_per_class_data = {1}; - std::vector iou_threshold_data = {0.4f}; - std::vector score_threshold_data = {0.2f}; + const int64_t nms_top_k = -1; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 6, 4}; - const auto scores_shape = Shape{1, 1, 6}; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 0.5f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - const auto max_output_boxes_per_class = make_shared(element::i64, Shape{1}); - const auto score_treshold = make_shared(element::f32, Shape{1}); - const auto iou_threshold = make_shared(element::f32, Shape{1}); - const auto soft_nms_sigma = make_shared(element::f32, Shape{1}); - auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_treshold, - soft_nms_sigma, - sort_result_type, - false); - - auto f = make_shared(nms, - ParameterVector{boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_treshold, - soft_nms_sigma}); + scores, + sort_result_type, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); - auto backend_max_output_boxes_per_class = backend->create_tensor(element::i64, {1}); - auto backend_iou_threshold = backend->create_tensor(element::f32, {1}); - auto backend_score_threshold = backend->create_tensor(element::f32, {1}); - auto backend_soft_nms_sigma = backend->create_tensor(element::f32, {1}); copy_data(backend_boxes, boxes_data); copy_data(backend_scores, scores_data); - copy_data(backend_max_output_boxes_per_class, max_output_boxes_per_class_data); - copy_data(backend_iou_threshold, iou_threshold_data); - copy_data(backend_score_threshold, score_threshold_data); - copy_data(backend_soft_nms_sigma, std::vector(0.0)); auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, - {backend_boxes, - backend_scores, - backend_max_output_boxes_per_class, - backend_iou_threshold, - backend_score_threshold, - backend_soft_nms_sigma}); + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3}; - std::vector expected_selected_scores = {0.0f, 0.0f, 0.95f}; - std::vector expected_valid_outputs = {1}; + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -*/ From 7790d8357a160ed07407e16609843a6b62160764 Mon Sep 17 00:00:00 2001 From: jialipen Date: Wed, 16 Jun 2021 17:10:18 +0800 Subject: [PATCH 4/6] fixes in ref impl. --- .../src/runtime/reference/multiclass_nms.cpp | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp index 80a0ad0ced0ce3..e1284dbb0a5152 100644 --- a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp +++ b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -207,7 +207,7 @@ namespace ngraph SelectedOutput* selected_scores_ptr = reinterpret_cast(selected_outputs); - std::vector filteredBoxes; + std::vector filteredBoxes; // container for the whole batch for (int64_t batch = 0; batch < num_batches; batch++) { @@ -215,9 +215,13 @@ namespace ngraph Rectangle* r = reinterpret_cast(const_cast(boxesPtr)); int64_t num_dets = 0; + std::vector selected_boxes; // container for a batch element for (int64_t class_idx = 0; class_idx < num_classes; class_idx++) { + //std::cout << "##############" << batch << ", " << class_idx << " bg:" << background_class << std::endl; + if(class_idx == background_class) continue; + const float* scoresPtr = scores_data + batch * (num_classes * num_boxes) + class_idx * num_boxes; @@ -225,7 +229,7 @@ namespace ngraph for (int64_t box_idx = 0; box_idx < num_boxes; box_idx++) { - if (scoresPtr[box_idx] > score_threshold) + if (scoresPtr[box_idx] >= score_threshold) /* NOTE: ">=" instead of ">" used in PDPD */ { candidate_boxes.emplace_back( r[box_idx], box_idx, scoresPtr[box_idx], 0, batch, class_idx); @@ -235,6 +239,9 @@ namespace ngraph int candiate_size = candidate_boxes.size(); // threshold nms_top_k for each class + // NOTE: "nms_top_k" in PDPD not exactly equal to + // "max_output_boxes_per_class" in ONNX. + //std::cout << "##############" << batch << ", " << class_idx << " nms_top_k=" << nms_top_k << " candiate_size=" << candiate_size << std::endl; if (nms_top_k > -1 && nms_top_k < candiate_size) { candiate_size = nms_top_k; @@ -260,7 +267,7 @@ namespace ngraph print_queue(sorted_boxes); - std::vector selected; + std::vector selected; // container for a class // Get the next box with top score, filter by iou_threshold BoxInfo next_candidate; @@ -311,23 +318,37 @@ namespace ngraph for (const auto& box_info : selected) { - filteredBoxes.push_back(box_info); + selected_boxes.push_back(box_info); } num_dets += selected.size(); } // for each class + /* sort inside batch element */ + if (sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) + { + std::sort(selected_boxes.begin(), + selected_boxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return ((l.batch_index == r.batch_index) && + ((l.score > r.score) || + ((std::fabs(l.score - r.score) < 1e-6) && l.class_index < r.class_index) || + ((std::fabs(l.score - r.score) < 1e-6) && l.class_index == r.class_index && l.index < r.index))); + }); + } + // in case of "NONE" and "CLASSID", pass through + // threshold keep_top_k for each batch element if (keep_top_k > -1 && keep_top_k < num_dets) { num_dets = nms_top_k; + selected_boxes.resize(num_dets); } - - if (num_dets <= 0) // early drop + + *valid_outputs++ = num_dets; + for(auto& v:selected_boxes) { - continue; + filteredBoxes.push_back(v); } - - *valid_outputs++ = num_dets; } // for each batch element /* sort */ @@ -362,22 +383,6 @@ namespace ngraph }); } } - else - { - /* sort inside batch element */ - if (sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) - { - std::sort(filteredBoxes.begin(), - filteredBoxes.end(), - [](const BoxInfo& l, const BoxInfo& r) { - return ((l.batch_index == r.batch_index) && - ((l.score > r.score) || - ((std::fabs(l.score - r.score) < 1e-6) && l.class_index < r.class_index) || - ((std::fabs(l.score - r.score) < 1e-6) && l.class_index == r.class_index && l.index < r.index))); - }); - } - // in case of "NONE" and "CLASSID", pass through - } /* output */ From db6e9d467c619fc91747399c874824b195c55f92 Mon Sep 17 00:00:00 2001 From: jialipen Date: Wed, 16 Jun 2021 18:26:14 +0800 Subject: [PATCH 5/6] attribute nms_eta works --- .../src/runtime/reference/multiclass_nms.cpp | 18 ++++++++----- ngraph/test/backend/multiclass_nms.in.cpp | 26 ++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp index e1284dbb0a5152..e9a8739ac9b07c 100644 --- a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp +++ b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -192,8 +192,8 @@ namespace ngraph const Shape& selected_indices_shape, int64_t* valid_outputs) { - auto func = [iou_threshold](float iou) { - return iou <= iou_threshold ? 1.0f : 0.0f; + auto func = [](float iou, float adaptive_threshold) { + return iou <= adaptive_threshold ? 1.0f : 0.0f; }; // boxes shape: {num_batches, num_boxes, 4} @@ -218,10 +218,11 @@ namespace ngraph std::vector selected_boxes; // container for a batch element for (int64_t class_idx = 0; class_idx < num_classes; class_idx++) - { - //std::cout << "##############" << batch << ", " << class_idx << " bg:" << background_class << std::endl; + { if(class_idx == background_class) continue; + auto adaptive_threshold = iou_threshold; + const float* scoresPtr = scores_data + batch * (num_classes * num_boxes) + class_idx * num_boxes; @@ -241,7 +242,6 @@ namespace ngraph // threshold nms_top_k for each class // NOTE: "nms_top_k" in PDPD not exactly equal to // "max_output_boxes_per_class" in ONNX. - //std::cout << "##############" << batch << ", " << class_idx << " nms_top_k=" << nms_top_k << " candiate_size=" << candiate_size << std::endl; if (nms_top_k > -1 && nms_top_k < candiate_size) { candiate_size = nms_top_k; @@ -286,9 +286,9 @@ namespace ngraph { float iou = intersectionOverUnion(next_candidate.box, selected[j].box); - next_candidate.score *= func(iou); + next_candidate.score *= func(iou, adaptive_threshold); - if (iou >= iou_threshold) + if (iou >= adaptive_threshold) { should_hard_suppress = true; break; @@ -304,6 +304,10 @@ namespace ngraph if (!should_hard_suppress) { + if(nms_eta < 1 && adaptive_threshold > 0.5) + { + adaptive_threshold *= nms_eta; + } if (next_candidate.score == original_score) { selected.push_back(next_candidate); diff --git a/ngraph/test/backend/multiclass_nms.in.cpp b/ngraph/test/backend/multiclass_nms.in.cpp index 0a2a41490e6244..0740d680d071e9 100644 --- a/ngraph/test/backend/multiclass_nms.in.cpp +++ b/ngraph/test/backend/multiclass_nms.in.cpp @@ -932,12 +932,12 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_nms_eta) }; const int64_t nms_top_k = -1; - const float iou_threshold = 0.5f; + const float iou_threshold = 1.0f; const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; const auto keep_top_k = -1; const auto background_class = -1; - const auto nms_eta = 0.5f; + const auto nms_eta = 0.1f; const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 const auto scores_shape = Shape{2, 2, 6}; @@ -977,13 +977,21 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_nms_eta) auto selected_indeces_value = read_vector(selected_indeces); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {3, 0, 0, 3, - 9, 6, 6, 9}; - std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, - 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 - 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, - 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 - std::vector expected_valid_outputs = {4, 4}; + std::vector expected_selected_indices = {3, 0, 5, 0, 3, 5, + 9, 6, 11, 6, 9, 11}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.30, 0.00, 100.00, 1.00, 101.00 }; + std::vector expected_valid_outputs = {6, 6}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); From 7727aa537689c998132370d4793aacf9a246c191 Mon Sep 17 00:00:00 2001 From: jialipen Date: Wed, 16 Jun 2021 21:11:04 +0800 Subject: [PATCH 6/6] test cross_batch and output_type i32. --- .../core/include/ngraph/op/multiclass_nms.hpp | 2 +- ngraph/test/backend/multiclass_nms.in.cpp | 248 +++++++++++++++++- 2 files changed, 248 insertions(+), 2 deletions(-) diff --git a/ngraph/core/include/ngraph/op/multiclass_nms.hpp b/ngraph/core/include/ngraph/op/multiclass_nms.hpp index 756b1a51d13302..274c211169586e 100644 --- a/ngraph/core/include/ngraph/op/multiclass_nms.hpp +++ b/ngraph/core/include/ngraph/op/multiclass_nms.hpp @@ -42,7 +42,7 @@ namespace ngraph MulticlassNms(const Output& boxes, const Output& scores, const SortResultType sort_result_type = SortResultType::NONE, - bool sort_result_across_batch = true, + bool sort_result_across_batch = false, const ngraph::element::Type& output_type = ngraph::element::i64, const float iou_threshold = 0.0f, const float score_threshold = 0.0f, diff --git a/ngraph/test/backend/multiclass_nms.in.cpp b/ngraph/test/backend/multiclass_nms.in.cpp index e1960879370d96..2e94f24e63db2e 100644 --- a/ngraph/test/backend/multiclass_nms.in.cpp +++ b/ngraph/test/backend/multiclass_nms.in.cpp @@ -127,6 +127,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_class_id) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -169,6 +170,76 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_class_id) EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_output_type_i32) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i32, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i32, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i32, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, @@ -183,7 +254,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score) 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, - 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 }; const int64_t nms_top_k = 3; @@ -203,6 +274,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -279,6 +351,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -322,6 +395,169 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id) EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + true, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 6, 3, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, + scores, + sort_result_type, + true, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 3, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) { std::vector boxes_data = {1.0, 1.0, 0.0, 0.0, 0.0, 0.1, 1.0, 1.1, @@ -346,6 +582,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -412,6 +649,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -475,6 +713,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -537,6 +776,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_single_box) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -600,6 +840,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -664,6 +905,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -727,6 +969,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_no_output) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -799,6 +1042,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_background) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -872,6 +1116,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_keep_top_k) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold, @@ -948,6 +1193,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_nms_eta) auto nms = make_shared(boxes, scores, sort_result_type, + false, element::i64, iou_threshold, score_threshold,