[OV20] Reference implementation for NV12toRGB and NV12toBGR operations (

openvinotoolkit#7601)

* Reference implementation for NV12toRGB and NV12toBGR operations
Tests:
- ngraph: visitor + type_prop
- template plugin: reference implementation
- inference-engine: shared tests for plugins
- cpu plugin: compare with ref implementation tests

* Fix clang

* Serialization tests

* Fix clang-format

* Changed 'f32' to 'any supported floating-point type'
Added appropriate shape inference tests
Added error test for >2 inputs
Fixed failed CI tests

* Updates after rebase
+ Try to fix Ninja build

* Fix CI

* Support endianness + potential fix of win32 test fails

* Fix review comment

* Fix review comments

* Fix unit test build

* Fix unit test build #2

* Possible build fix 3

* Simplified reference tests
Observed issue with shuffling Y pixels on little-endian systems, added tests

docs/template_plugin/tests/functional/op_reference/convert_color_nv12.cpp

@@ -0,0 +1,142 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <gtest/gtest.h>
+
+#include <openvino/core/function.hpp>
+#include <tuple>
+#include <openvino/op/nv12_to_rgb.hpp>
+#include <openvino/op/nv12_to_bgr.hpp>
+
+#include "base_reference_test.hpp"
+
+using namespace ov;
+using namespace InferenceEngine;
+using namespace reference_tests;
+
+class ReferenceConvertColorNV12LayerTest : public testing::Test, public CommonReferenceTest {
+public:
+    void SetUp() override {
+    }
+
+public:
+    template <typename T>
+    static std::shared_ptr<Function> CreateFunction(const Tensor& input) {
+        const auto in = std::make_shared<op::v0::Parameter>(input.type, input.shape);
+        std::shared_ptr<Node> conv;
+        conv = std::make_shared<T>(in);
+        auto res = std::make_shared<op::v0::Result>(conv);
+        return std::make_shared<Function>(ResultVector{res}, ParameterVector {in});
+    }
+
+    template <typename T>
+    static std::shared_ptr<Function> CreateFunction2(const Tensor& input1, const Tensor& input2) {
+        const auto in1 = std::make_shared<op::v0::Parameter>(input1.type, input1.shape);
+        const auto in2 = std::make_shared<op::v0::Parameter>(input2.type, input2.shape);
+        std::shared_ptr<Node> conv;
+        conv = std::make_shared<T>(in1, in2);
+        auto res = std::make_shared<op::v0::Result>(conv);
+        return std::make_shared<Function>(ResultVector{res}, ParameterVector {in1, in2});
+    }
+};
+
+TEST_F(ReferenceConvertColorNV12LayerTest, CompareWithHardcodedRefs_r_u8_single_rgb) {
+    auto input = std::vector<uint8_t> {0x51, 0x51, 0x51, 0x51, 0xf0, 0x5a};
+    auto input_shape = Shape{1, 3, 2, 1};
+    auto exp_out = std::vector<uint8_t> {0xff, 0, 0, 0xff, 0, 0, 0xff, 0, 0, 0xff, 0, 0};
+    auto out_shape = Shape{1, 2, 2, 3};
+    Tensor inp_tensor(input_shape, element::u8, input);
+    inputData = {inp_tensor.data};
+    function = CreateFunction<op::v8::NV12toRGB>(inp_tensor);
+    Tensor exp_tensor_u8(out_shape, element::u8, exp_out);
+    refOutData = {exp_tensor_u8.data};
+    Exec();
+}
+
+TEST_F(ReferenceConvertColorNV12LayerTest, CompareWithHardcodedRefs_color_u8_single_bgr) {
+    auto input = std::vector<uint8_t> {0xeb, 0x51, 0xeb, 0x51, 0xb8, 0x6d};
+    auto input_shape = Shape{1, 3, 2, 1};
+    auto exp_out = std::vector<uint8_t> {37, 37, 164, 215, 216, 255, 37, 37, 164, 215, 216, 255};
+    auto out_shape = Shape{1, 2, 2, 3};
+
+    Tensor inp_tensor(input_shape, element::u8, input);
+    inputData = {inp_tensor.data};
+
+    Tensor exp_tensor_u8(out_shape, element::u8, exp_out);
+    refOutData = {exp_tensor_u8.data};
+
+    function = CreateFunction<op::v8::NV12toBGR>(inp_tensor);
+
+    Exec();
+}
+
+TEST_F(ReferenceConvertColorNV12LayerTest, CompareWithHardcodedRefs_g_fp32_single_rgb) {
+    threshold = 2.f;
+    auto input = std::vector<float> {145.f, 145.f, 145.f, 145.f, 34.f, 54.f};
+    auto input_shape = Shape{1, 3, 2, 1};
+    auto exp_out = std::vector<float> {0, 255.f, 0, 0, 255.f, 0, 0, 255.f, 0, 0, 255.f, 0};
+    auto out_shape = Shape{1, 2, 2, 3};
+
+    Tensor inp_tensor(input_shape, element::f32, input);
+    inputData = {inp_tensor.data};
+
+    Tensor exp_tensor(out_shape, element::f32, exp_out);
+    refOutData = {exp_tensor.data};
+
+    function = CreateFunction<op::v8::NV12toRGB>(inp_tensor);
+
+    Exec();
+}
+
+TEST_F(ReferenceConvertColorNV12LayerTest, CompareWithHardcodedRefs_batch_fp32_two_bgr) {
+    threshold = 2.f;
+    auto input_y = std::vector<float> {81.f, 81.f, 81.f, 81.f,
+                                       145.f, 145.f, 145.f, 145.f,
+                                       41.f, 41.f, 41.f, 41.f};
+    auto input_shape_y = Shape{3, 2, 2, 1};
+
+    auto input_uv = std::vector<float> {240., 90.,
+                                        34., 54.,
+                                        110., 240.};
+    auto input_shape_uv = Shape{3, 1, 1, 2};
+
+    auto exp_out = std::vector<float> {0, 0, 255., 0, 0, 255., 0, 0, 255., 0, 0, 255.,
+                                       0, 255., 0, 0, 255., 0, 0, 255., 0, 0, 255., 0,
+                                       255., 0, 0, 255., 0, 0, 255., 0, 0, 255., 0, 0};
+    auto out_shape = Shape{3, 2, 2, 3};
+
+    Tensor inp_tensor_y(input_shape_y, element::f32, input_y);
+    Tensor inp_tensor_uv(input_shape_uv, element::f32, input_uv);
+    inputData = {inp_tensor_y.data, inp_tensor_uv.data};
+
+    Tensor exp_tensor(out_shape, element::f32, exp_out);
+    refOutData = {exp_tensor.data};
+
+    function = CreateFunction2<op::v8::NV12toBGR>(inp_tensor_y, inp_tensor_uv);
+
+    Exec();
+}
+
+TEST_F(ReferenceConvertColorNV12LayerTest, CompareWithHardcodedRefs_color2x2_f32_two_rgb) {
+    threshold = 2.f;
+    auto input_y = std::vector<float> {235, 81, 235, 81};
+    auto input_shape_y = Shape{1, 2, 2, 1};
+
+    auto input_uv = std::vector<float> {184, 109};
+    auto input_shape_uv = Shape{1, 1, 1, 2};
+
+    auto exp_out = std::vector<float> {164, 37, 37, 216, 215, 255, 164, 37, 37, 216, 215, 255};
+    auto out_shape = Shape{1, 2, 2, 3};
+
+    Tensor inp_tensor_y(input_shape_y, element::f32, input_y);
+    Tensor inp_tensor_uv(input_shape_uv, element::f32, input_uv);
+    inputData = {inp_tensor_y.data, inp_tensor_uv.data};
+
+    Tensor exp_tensor(out_shape, element::f32, exp_out);
+    refOutData = {exp_tensor.data};
+
+    function = CreateFunction2<op::v8::NV12toRGB>(inp_tensor_y, inp_tensor_uv);
+
+    Exec();
+}

inference-engine/tests/functional/inference_engine/serialization/single_layer/convert_color_nv12.cpp

@@ -0,0 +1,33 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "shared_test_classes/single_layer/convert_color_nv12.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+
+TEST_P(ConvertColorNV12LayerTest, Serialize) {
+        Serialize();
+    }
+
+const std::vector<ov::Shape> inShapes_nhwc = {
+        {1, 10, 10, 1}
+};
+
+const std::vector<ov::element::Type> inTypes = {
+        ov::element::u8, ov::element::f32
+};
+
+const auto testCase_values = ::testing::Combine(
+        ::testing::ValuesIn(inShapes_nhwc),
+        ::testing::ValuesIn(inTypes),
+        ::testing::Bool(),
+        ::testing::Bool(),
+        ::testing::Values(CommonTestUtils::DEVICE_CPU)
+);
+
+INSTANTIATE_TEST_SUITE_P(smoke_CompareWithRefs, ConvertColorNV12LayerTest, testCase_values, ConvertColorNV12LayerTest::getTestCaseName);
+
+} // namespace

inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/convert_color_nv12.cpp

@@ -0,0 +1,33 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <vector>
+
+#include "single_layer_tests/convert_color_nv12.hpp"
+#include "common_test_utils/test_constants.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+
+const std::vector<ov::Shape> inShapes_nhwc = {
+    {1, 10, 10, 1}
+};
+
+const std::vector<ov::element::Type> inTypes = {
+        ov::element::u8, ov::element::f32
+};
+
+const auto testCase_values = ::testing::Combine(
+    ::testing::ValuesIn(inShapes_nhwc),
+    ::testing::ValuesIn(inTypes),
+    ::testing::Bool(),
+    ::testing::Bool(),
+    ::testing::Values(CommonTestUtils::DEVICE_CPU)
+);
+
+
+INSTANTIATE_TEST_SUITE_P(smoke_TestsConvertColorNV12, ConvertColorNV12LayerTest, testCase_values, ConvertColorNV12LayerTest::getTestCaseName);
+
+}  // namespace

inference-engine/tests/functional/plugin/shared/include/single_layer_tests/convert_color_nv12.hpp

@@ -0,0 +1,15 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "shared_test_classes/single_layer/convert_color_nv12.hpp"
+
+namespace LayerTestsDefinitions {
+
+TEST_P(ConvertColorNV12LayerTest, CompareWithRefs) {
+    Run();
+};
+
+} // namespace LayerTestsDefinitions

inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/convert_color_nv12.hpp

@@ -0,0 +1,33 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <tuple>
+#include <string>
+#include <vector>
+
+#include "shared_test_classes/base/layer_test_utils.hpp"
+#include "ngraph_functions/builders.hpp"
+#include "ngraph_functions/utils/ngraph_helpers.hpp"
+
+namespace LayerTestsDefinitions {
+
+using ConvertColorNV12ParamsTuple = std::tuple<
+        ov::Shape,                                     // Input Shape
+        ov::element::Type,                             // Element type
+        bool,                                          // Conversion type
+        bool,                                          // 1 or 2 planes
+        std::string>;                                  // Device name
+
+class ConvertColorNV12LayerTest : public testing::WithParamInterface<ConvertColorNV12ParamsTuple>,
+                            virtual public LayerTestsUtils::LayerTestsCommon {
+public:
+    static std::string getTestCaseName(const testing::TestParamInfo<ConvertColorNV12ParamsTuple> &obj);
+
+protected:
+    void SetUp() override;
+};
+
+} // namespace LayerTestsDefinitions

inference-engine/tests/functional/shared_test_classes/src/single_layer/convert_color_nv12.cpp

@@ -0,0 +1,58 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "shared_test_classes/single_layer/convert_color_nv12.hpp"
+#include "openvino/op/nv12_to_rgb.hpp"
+#include "openvino/op/nv12_to_bgr.hpp"
+
+namespace LayerTestsDefinitions {
+
+std::string ConvertColorNV12LayerTest::getTestCaseName(const testing::TestParamInfo<ConvertColorNV12ParamsTuple> &obj) {
+    ov::Shape inputShape;
+    ov::element::Type type;
+    bool conversion, singlePlane;
+    std::string targetName;
+    std::tie(inputShape, type, conversion, singlePlane, targetName) = obj.param;
+    std::ostringstream result;
+    result << "IS=" << CommonTestUtils::vec2str(inputShape) << "_";
+    result << "netPRC=" << type.c_type_string() << "_";
+    result << "convRGB=" << conversion << "_";
+    result << "singlePlane=" << singlePlane << "_";
+    result << "targetDevice=" << targetName;
+    return result.str();
+}
+
+void ConvertColorNV12LayerTest::SetUp() {
+    ov::Shape inputShape;
+    ov::element::Type ngPrc;
+    bool conversionToRGB, singlePlane;
+    threshold = 2.0f; // NV12 color conversion can use various of algorithms, thus some deviation is allowed
+    std::tie(inputShape, ngPrc, conversionToRGB, singlePlane, targetDevice) = GetParam();
+    if (singlePlane) {
+        inputShape[1] = inputShape[1] * 3 / 2;
+        auto param = std::make_shared<ov::op::v0::Parameter>(ngPrc, inputShape);
+        std::shared_ptr<ov::Node> convert_color;
+        if (conversionToRGB) {
+            convert_color = std::make_shared<ov::op::v8::NV12toRGB>(param);
+        } else {
+            convert_color = std::make_shared<ov::op::v8::NV12toBGR>(param);
+        }
+        function = std::make_shared<ov::Function>(std::make_shared<ov::op::v0::Result>(convert_color),
+                                                      ov::ParameterVector{param}, "ConvertColorNV12");
+    } else {
+        auto uvShape = ov::Shape{inputShape[0], inputShape[1] / 2, inputShape[2] / 2, 2};
+        auto param_y = std::make_shared<ov::op::v0::Parameter>(ngPrc, inputShape);
+        auto param_uv = std::make_shared<ov::op::v0::Parameter>(ngPrc, uvShape);
+        std::shared_ptr<ov::Node> convert_color;
+        if (conversionToRGB) {
+            convert_color = std::make_shared<ov::op::v8::NV12toRGB>(param_y, param_uv);
+        } else {
+            convert_color = std::make_shared<ov::op::v8::NV12toBGR>(param_y, param_uv);
+        }
+        function = std::make_shared<ov::Function>(std::make_shared<ov::op::v0::Result>(convert_color),
+                                                      ov::ParameterVector{param_y, param_uv}, "ConvertColorNV12");
+    }
+}
+
+} // namespace LayerTestsDefinitions

ngraph/core/include/ngraph/op/nv12_to_bgr.hpp

@@ -0,0 +1,15 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/op/nv12_to_bgr.hpp"
+
+namespace ngraph {
+namespace op {
+namespace v8 {
+using ov::op::v8::NV12toBGR;
+}  // namespace v8
+}  // namespace op
+}  // namespace ngraph

ngraph/core/include/ngraph/op/nv12_to_rgb.hpp

@@ -0,0 +1,15 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/op/nv12_to_rgb.hpp"
+
+namespace ngraph {
+namespace op {
+namespace v8 {
+using ov::op::v8::NV12toRGB;
+}  // namespace v8
+}  // namespace op
+}  // namespace ngraph

ngraph/core/include/ngraph/ops.hpp

@@ -103,6 +103,8 @@
 #include "ngraph/op/normalize_l2.hpp"
 #include "ngraph/op/not.hpp"
 #include "ngraph/op/not_equal.hpp"
+#include "ngraph/op/nv12_to_bgr.hpp"
+#include "ngraph/op/nv12_to_rgb.hpp"
 #include "ngraph/op/one_hot.hpp"
 #include "ngraph/op/or.hpp"
 #include "ngraph/op/pad.hpp"