diff --git a/ngraph/core/include/ngraph/op/multiclass_nms.hpp b/ngraph/core/include/ngraph/op/multiclass_nms.hpp index 756b1a51d13302..274c211169586e 100644 --- a/ngraph/core/include/ngraph/op/multiclass_nms.hpp +++ b/ngraph/core/include/ngraph/op/multiclass_nms.hpp @@ -42,7 +42,7 @@ namespace ngraph MulticlassNms(const Output& boxes, const Output& scores, const SortResultType sort_result_type = SortResultType::NONE, - bool sort_result_across_batch = true, + bool sort_result_across_batch = false, const ngraph::element::Type& output_type = ngraph::element::i64, const float iou_threshold = 0.0f, const float score_threshold = 0.0f, diff --git a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp index 83456c9657c947..7bf79f41c0ce73 100644 --- a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp +++ b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -117,6 +117,11 @@ namespace ngraph return score < rhs.score || (score == rhs.score && index > rhs.index); } + inline bool operator>(const BoxInfo& rhs) const + { + return !(score < rhs.score || (score == rhs.score && index > rhs.index)); + } + Rectangle box; int64_t index = 0; int64_t suppress_begin_index = 0; @@ -124,7 +129,52 @@ namespace ngraph int64_t class_index = 0; float score = 0.0f; }; + + inline std::ostream& operator<<(std::ostream& s, const Rectangle& b) + { + s << "Rectangle{"; + s << b.x1 << ", "; + s << b.y1 << ", "; + s << b.x2 << ", "; + s << b.y2; + s << "}"; + return s; + } + + inline std::ostream& operator<<(std::ostream& s, const BoxInfo& b) + { + s << "BoxInfo{"; + s << b.batch_index << ", "; + s << b.class_index << ", "; + s << b.index << ", "; + s << b.box << ", "; + s << b.score; + s << "}"; + return s; + } } // namespace + + template + void print_queue(T q) { // NB: pass by value so the print uses a copy + std::cout << "\n{"; + while(!q.empty()) + { + std::cout << q.top() << ", "; + q.pop(); + } + std::cout << "}\n"; + } + + template + void print_list(T &q) { + std::cout << "\n{"; + for(auto& v : q) + { + std::cout << v << ", "; + } + std::cout << "}\n"; + } + void multiclass_nms(const float* boxes_data, const Shape& boxes_data_shape, const float* scores_data, @@ -143,8 +193,8 @@ namespace ngraph const Shape& selected_indices_shape, int64_t* valid_outputs) { - auto func = [iou_threshold](float iou) { - return iou <= iou_threshold ? 1.0f : 0.0f; + auto func = [](float iou, float adaptive_threshold) { + return iou <= adaptive_threshold ? 1.0f : 0.0f; }; // boxes shape: {num_batches, num_boxes, 4} @@ -158,9 +208,7 @@ namespace ngraph SelectedOutput* selected_scores_ptr = reinterpret_cast(selected_outputs); - size_t boxes_per_class = static_cast(nms_top_k); - - std::vector filteredBoxes; + std::vector filteredBoxes; // container for the whole batch for (int64_t batch = 0; batch < num_batches; batch++) { @@ -168,34 +216,65 @@ namespace ngraph Rectangle* r = reinterpret_cast(const_cast(boxesPtr)); int64_t num_dets = 0; + std::vector selected_boxes; // container for a batch element for (int64_t class_idx = 0; class_idx < num_classes; class_idx++) - { + { + if(class_idx == background_class) continue; + + auto adaptive_threshold = iou_threshold; + const float* scoresPtr = scores_data + batch * (num_classes * num_boxes) + class_idx * num_boxes; std::vector candidate_boxes; - candidate_boxes.reserve(num_boxes); for (int64_t box_idx = 0; box_idx < num_boxes; box_idx++) { - if (scoresPtr[box_idx] > score_threshold) + if (scoresPtr[box_idx] >= score_threshold) /* NOTE: ">=" instead of ">" used in PDPD */ { candidate_boxes.emplace_back( r[box_idx], box_idx, scoresPtr[box_idx], 0, batch, class_idx); } } - std::priority_queue sorted_boxes(std::less(), - std::move(candidate_boxes)); + int candiate_size = candidate_boxes.size(); + + // threshold nms_top_k for each class + // NOTE: "nms_top_k" in PDPD not exactly equal to + // "max_output_boxes_per_class" in ONNX. + if (nms_top_k > -1 && nms_top_k < candiate_size) + { + candiate_size = nms_top_k; + } + + if (candiate_size <= 0) // early drop + { + continue; + } + + // sort by score + std::partial_sort(candidate_boxes.begin(), + candidate_boxes.begin() + candiate_size, + candidate_boxes.end(), + std::greater()); - std::vector selected; + print_list(candidate_boxes); + + std::priority_queue sorted_boxes(candidate_boxes.begin(), + candidate_boxes.begin() + candiate_size, std::less()); + + print_list(candidate_boxes); + + print_queue(sorted_boxes); + + std::vector selected; // container for a class // Get the next box with top score, filter by iou_threshold BoxInfo next_candidate; float original_score; - while (!sorted_boxes.empty() && selected.size() < boxes_per_class) + while (!sorted_boxes.empty()) { next_candidate = sorted_boxes.top(); original_score = next_candidate.score; @@ -208,9 +287,9 @@ namespace ngraph { float iou = intersectionOverUnion(next_candidate.box, selected[j].box); - next_candidate.score *= func(iou); + next_candidate.score *= func(iou, adaptive_threshold); - if (iou >= iou_threshold) + if (iou >= adaptive_threshold) { should_hard_suppress = true; break; @@ -226,6 +305,10 @@ namespace ngraph if (!should_hard_suppress) { + if(nms_eta < 1 && adaptive_threshold > 0.5) + { + adaptive_threshold *= nms_eta; + } if (next_candidate.score == original_score) { selected.push_back(next_candidate); @@ -240,17 +323,44 @@ namespace ngraph for (const auto& box_info : selected) { - filteredBoxes.push_back(box_info); + selected_boxes.push_back(box_info); } - num_dets += filteredBoxes.size(); + num_dets += selected.size(); + } // for each class + + /* sort inside batch element */ + if (sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) + { + std::sort(selected_boxes.begin(), + selected_boxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return ((l.batch_index == r.batch_index) && + ((l.score > r.score) || + ((std::fabs(l.score - r.score) < 1e-6) && l.class_index < r.class_index) || + ((std::fabs(l.score - r.score) < 1e-6) && l.class_index == r.class_index && l.index < r.index))); + }); } + // in case of "NONE" and "CLASSID", pass through + // threshold keep_top_k for each batch element + if (keep_top_k > -1 && keep_top_k < num_dets) + { + num_dets = nms_top_k; + selected_boxes.resize(num_dets); + } + *valid_outputs++ = num_dets; - } + for(auto& v:selected_boxes) + { + filteredBoxes.push_back(v); + } + } // for each batch element if (sort_result_across_batch) - { - std::sort(filteredBoxes.begin(), + { /* sort across batch */ + if (sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) + { + std::sort(filteredBoxes.begin(), filteredBoxes.end(), [](const BoxInfo& l, const BoxInfo& r) { return (l.score > r.score) || @@ -260,8 +370,24 @@ namespace ngraph (l.score == r.score && l.batch_index == r.batch_index && l.class_index == r.class_index && l.index < r.index); }); + } + else if(sort_result_type == op::v8::MulticlassNms::SortResultType::CLASSID) + { + std::sort(filteredBoxes.begin(), + filteredBoxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return (l.class_index < r.class_index) || + (l.class_index == r.class_index && l.batch_index < r.batch_index) || + (l.class_index == r.class_index && l.batch_index == r.batch_index && + l.score > r.score) || + (l.class_index == r.class_index && l.batch_index == r.batch_index && + l.score == r.score && l.index < r.index); + }); + } } + /* output */ + size_t max_num_of_selected_indices = selected_indices_shape[0]; size_t output_size = std::min(filteredBoxes.size(), max_num_of_selected_indices); diff --git a/ngraph/test/backend/multiclass_nms.in.cpp b/ngraph/test/backend/multiclass_nms.in.cpp index 637e7a047a1a3e..2e94f24e63db2e 100644 --- a/ngraph/test/backend/multiclass_nms.in.cpp +++ b/ngraph/test/backend/multiclass_nms.in.cpp @@ -30,12 +30,540 @@ using namespace ngraph; static string s_manifest = "${MANIFEST}"; -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_score) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_class_id) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_output_type_i32) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i32, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i32, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i32, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, + scores, + sort_result_type, + true, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 6, 3, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, + scores, + sort_result_type, + true, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 3, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) +{ + std::vector boxes_data = {1.0, 1.0, 0.0, 0.0, 0.0, 0.1, 1.0, 1.1, + 0.0, 0.9, 1.0, -0.1, 0.0, 10.0, 1.0, 11.0, + 1.0, 10.1, 0.0, 11.1, 1.0, 101.0, 0.0, 100.0}; + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; const int64_t nms_top_k = 3; @@ -46,12 +574,78 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) const auto background_class = -1; const auto nms_eta = 1.0f; - const auto boxes_shape = Shape{1, 6, 4}; - const auto scores_shape = Shape{1, 1, 6}; + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 1, M 6 + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + auto backend = runtime::Backend::create("${BACKEND_NAME}"); + + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + + auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); + auto backend_scores = backend->create_tensor(element::f32, scores_shape); + copy_data(backend_boxes, boxes_data); + copy_data(backend_scores, scores_data); + + auto handle = backend->compile(f); + + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + + auto selected_scores_value = read_vector(selected_outputs); + auto selected_indeces_value = read_vector(selected_indeces); + auto valid_outputs_value = read_vector(valid_outputs); + + std::vector expected_selected_indices = {3, 0, 1}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 1.00, 1.00, 0.00, 0.00 , + 0.00, 0.75, 0.00, 0.10, 1.00, 1.10}; + std::vector expected_valid_outputs = {3}; + + EXPECT_EQ(expected_selected_indices, selected_indeces_value); + EXPECT_EQ(expected_selected_scores, selected_scores_value); + EXPECT_EQ(expected_valid_outputs, valid_outputs_value); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, + 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, + 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, + 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0}; + + std::vector scores_data = {0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9}; + + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 10, 4}; // N 1, C 1, M 10 + const auto scores_shape = Shape{1, 1, 10}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto nms = make_shared(boxes, scores, sort_result_type, @@ -68,9 +662,9 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_outputs = backend->create_tensor(element::f32, Shape{3, 6}); // TODO: dynamic shape - auto selected_indeces = backend->create_tensor(element::i64, Shape{3, 1}); // TODO - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -82,60 +676,59 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_SCORE_point_box_format) handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); - auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {3, 0, 5}; - std::vector expected_selected_scores = {0.0, 0.95, 0.0, 10.0, 1.0, 11.0, - 0.0, 0.9, 0.0, 0.0, 1.0, 1.0, - 0.0, 0.3, 0.0, 100.0, 1.0, 101.0}; - std::vector expected_valid_outputs = {3}; + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {1}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -/* -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) { - std::vector boxes_data = {1.0, 1.0, 0.0, 0.0, 0.0, 0.1, 1.0, 1.1, - 0.0, 0.9, 1.0, -0.1, 0.0, 10.0, 1.0, 11.0, - 1.0, 10.1, 0.0, 11.1, 1.0, 101.0, 0.0, 100.0}; + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 2; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{3, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{3, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -144,62 +737,61 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 0, 0, 5}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 0.0, 0.0, 0.3}; - std::vector expected_valid_outputs = {3}; + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 }; + std::vector expected_valid_outputs = {2}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_single_box) { - std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, - 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, - 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, - 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0}; + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0}; - std::vector scores_data = {0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9}; + std::vector scores_data = {0.9}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 10, 4}; - const auto scores_shape = Shape{1, 1, 10}; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 1, 4}; + const auto scores_shape = Shape{1, 1, 1}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -208,15 +800,15 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.9}; + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; std::vector expected_valid_outputs = {1}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); @@ -224,7 +816,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, @@ -232,37 +824,38 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 2; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.2f; + const float score_threshold = 0.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{2, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{2, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -271,15 +864,16 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9}; + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; std::vector expected_valid_outputs = {2}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); @@ -287,68 +881,7 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_single_box) -{ - std::vector boxes_data = {0.0, 0.0, 1.0, 1.0}; - - std::vector scores_data = {0.9}; - - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 1, 4}; - const auto scores_shape = Shape{1, 1, 1}; - - const auto boxes = make_shared(element::f32, boxes_shape); - const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); - auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); - - auto f = make_shared(nms, ParameterVector{boxes, scores}); - - auto backend = runtime::Backend::create("${BACKEND_NAME}"); - - auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); - - auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); - auto backend_scores = backend->create_tensor(element::f32, scores_shape); - copy_data(backend_boxes, boxes_data); - copy_data(backend_scores, scores_data); - - auto handle = backend->compile(f); - - handle->call({selected_indeces, selected_scores, valid_outputs}, - {backend_boxes, backend_scores}); - - auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); - auto valid_outputs_value = read_vector(valid_outputs); - - std::vector expected_selected_indices = {0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.9}; - std::vector expected_valid_outputs = {1}; - - EXPECT_EQ(expected_selected_indices, selected_indeces_value); - EXPECT_EQ(expected_selected_scores, selected_scores_value); - EXPECT_EQ(expected_valid_outputs, valid_outputs_value); -} - -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, @@ -356,37 +889,38 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.95f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{3, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{3, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -395,23 +929,23 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 0, 0, 5}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 0.0, 0.0, 0.3}; - std::vector expected_valid_outputs = {3}; + std::vector expected_selected_indices = {3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00}; + std::vector expected_valid_outputs = {1}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_no_output) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, @@ -419,37 +953,38 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; - const int64_t max_output_boxes_per_class_data = 3; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.4f; - const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 2.0f; + const auto sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 1.0f; + const auto boxes_shape = Shape{1, 6, 4}; const auto scores_shape = Shape{1, 1, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{2, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{2, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -458,63 +993,71 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); + auto selected_scores_value = read_vector(selected_outputs); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0}; - std::vector expected_selected_scores = {0.0, 0.0, 0.95, 0.0, 0.0, 0.9}; - std::vector expected_valid_outputs = {2}; + std::vector expected_selected_indices = {}; + std::vector expected_selected_scores = {}; + std::vector expected_valid_outputs = {0}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_background) { - std::vector boxes_data = { - 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, - 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, - 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; std::vector scores_data = { - 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, 0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; - const int64_t max_output_boxes_per_class_data = 2; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{2, 6, 4}; - const auto scores_shape = Shape{2, 1, 6}; + const auto keep_top_k = -1; + const auto background_class = 0; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{4, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{4, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -523,63 +1066,72 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 1, 0, 3, 1, 0, 0}; - std::vector expected_selected_scores = { - 0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 1.0, 0.0, 0.95, 1.0, 0.0, 0.9}; - std::vector expected_valid_outputs = {4}; + std::vector expected_selected_indices = {0, 3, 6, 9}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {2, 2}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_classes) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_keep_top_k) { std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, - 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; std::vector scores_data = { - 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, 0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; - const int64_t max_output_boxes_per_class_data = 2; - const float iou_threshold_data = 0.5f; - const float score_threshold_data = 0.0f; + const int64_t nms_top_k = 3; + const float iou_threshold = 0.5f; + const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 6, 4}; - const auto scores_shape = Shape{1, 2, 6}; + const auto keep_top_k = 3; + const auto background_class = -1; + const auto nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - auto max_output_boxes_per_class = - op::Constant::create(element::i64, Shape{}, {max_output_boxes_per_class_data}); - auto iou_threshold = op::Constant::create(element::f32, Shape{}, {iou_threshold_data}); - auto score_threshold = - op::Constant::create(element::f32, Shape{}, {score_threshold_data}); - auto soft_nms_sigma = op::Constant::create(element::f32, Shape{}, {0.0f}); auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold, - soft_nms_sigma, - sort_result_type, - false); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{4, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{4, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); @@ -588,101 +1140,107 @@ NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_classes) auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, + handle->call({selected_outputs, selected_indeces, valid_outputs}, {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3, 0, 0, 0, 0, 1, 3, 0, 1, 0}; - std::vector expected_selected_scores = { - 0.0, 0.0, 0.95, 0.0, 0.0, 0.9, 0.0, 1.0, 0.95, 0.0, 1.0, 0.9}; - std::vector expected_valid_outputs = {4}; + std::vector expected_selected_indices = {3, 0, 0, + 9, 6, 6}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 }; // 1 + std::vector expected_valid_outputs = {3, 3}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores_without_constants) +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_nms_eta) { - std::vector boxes_data = {0.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.1f, 1.0f, 1.1f, - 0.0f, -0.1f, 1.0f, 0.9f, 0.0f, 10.0f, 1.0f, 11.0f, - 0.0f, 10.1f, 1.0f, 11.1f, 0.0f, 100.0f, 1.0f, 101.0f}; - - std::vector scores_data = {0.9f, 0.75f, 0.6f, 0.95f, 0.5f, 0.3f}; + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; - std::vector max_output_boxes_per_class_data = {1}; - std::vector iou_threshold_data = {0.4f}; - std::vector score_threshold_data = {0.2f}; + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + const int64_t nms_top_k = -1; + const float iou_threshold = 1.0f; + const float score_threshold = 0.0f; const auto sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; - const auto boxes_shape = Shape{1, 6, 4}; - const auto scores_shape = Shape{1, 1, 6}; + const auto keep_top_k = -1; + const auto background_class = -1; + const auto nms_eta = 0.1f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; const auto boxes = make_shared(element::f32, boxes_shape); const auto scores = make_shared(element::f32, scores_shape); - const auto max_output_boxes_per_class = make_shared(element::i64, Shape{1}); - const auto score_treshold = make_shared(element::f32, Shape{1}); - const auto iou_threshold = make_shared(element::f32, Shape{1}); - const auto soft_nms_sigma = make_shared(element::f32, Shape{1}); - auto nms = make_shared(boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_treshold, - soft_nms_sigma, - sort_result_type, - false); - - auto f = make_shared(nms, - ParameterVector{boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_treshold, - soft_nms_sigma}); + scores, + sort_result_type, + false, + element::i64, + iou_threshold, + score_threshold, + nms_top_k, + keep_top_k, + background_class, + nms_eta); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); auto backend = runtime::Backend::create("${BACKEND_NAME}"); - auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3}); - auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3}); - auto valid_outputs = backend->create_tensor(element::i64, Shape{1}); + auto selected_outputs = backend->create_dynamic_tensor(element::f32, PartialShape::dynamic()); + auto selected_indeces = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); + auto valid_outputs = backend->create_dynamic_tensor(element::i64, PartialShape::dynamic()); auto backend_boxes = backend->create_tensor(element::f32, boxes_shape); auto backend_scores = backend->create_tensor(element::f32, scores_shape); - auto backend_max_output_boxes_per_class = backend->create_tensor(element::i64, {1}); - auto backend_iou_threshold = backend->create_tensor(element::f32, {1}); - auto backend_score_threshold = backend->create_tensor(element::f32, {1}); - auto backend_soft_nms_sigma = backend->create_tensor(element::f32, {1}); copy_data(backend_boxes, boxes_data); copy_data(backend_scores, scores_data); - copy_data(backend_max_output_boxes_per_class, max_output_boxes_per_class_data); - copy_data(backend_iou_threshold, iou_threshold_data); - copy_data(backend_score_threshold, score_threshold_data); - copy_data(backend_soft_nms_sigma, std::vector(0.0)); auto handle = backend->compile(f); - handle->call({selected_indeces, selected_scores, valid_outputs}, - {backend_boxes, - backend_scores, - backend_max_output_boxes_per_class, - backend_iou_threshold, - backend_score_threshold, - backend_soft_nms_sigma}); + handle->call({selected_outputs, selected_indeces, valid_outputs}, + {backend_boxes, backend_scores}); + auto selected_scores_value = read_vector(selected_outputs); auto selected_indeces_value = read_vector(selected_indeces); - auto selected_scores_value = read_vector(selected_scores); auto valid_outputs_value = read_vector(valid_outputs); - std::vector expected_selected_indices = {0, 0, 3}; - std::vector expected_selected_scores = {0.0f, 0.0f, 0.95f}; - std::vector expected_valid_outputs = {1}; + std::vector expected_selected_indices = {3, 0, 5, 0, 3, 5, + 9, 6, 11, 6, 9, 11}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.30, 0.00, 100.00, 1.00, 101.00 }; + std::vector expected_valid_outputs = {6, 6}; EXPECT_EQ(expected_selected_indices, selected_indeces_value); EXPECT_EQ(expected_selected_scores, selected_scores_value); EXPECT_EQ(expected_valid_outputs, valid_outputs_value); } -*/