[OV20] Convert NV12 to RGB operation + preprocessing (openvinotoolkit…

…#7508)

* # Conflicts:
#	docs/template_plugin/tests/functional/op_reference/convert_color_nv12.cpp
#	inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/convert_color_nv12.cpp
#	inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/convert_color_nv12.hpp
#	inference-engine/tests/functional/shared_test_classes/src/single_layer/convert_color_nv12.cpp
#	ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp
#	ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp
#	ngraph/core/include/openvino/op/nv12_to_bgr.hpp
#	ngraph/core/include/openvino/op/nv12_to_rgb.hpp
#	ngraph/core/src/op/nv12_to_bgr.cpp
#	ngraph/core/src/op/nv12_to_rgb.cpp
#	ngraph/core/src/preprocess/pre_post_process.cpp
#	ngraph/core/src/preprocess/preprocess_steps_impl.hpp
#	ngraph/test/CMakeLists.txt

* Added more test to cover 100% of code
Allow convert element type for 'multi-plane' color format

* Inherit tensor names for 'convert_color'

* Clang

* Fix tests

* Disable 'int8' preprocessing resize test

* Fix review comments

* Add more restrictions and tests for planes sub-names

* 1) Added check for uniqueness of tensor names generated for nodes
Raise error if user's plane sub-name conflicts with some node in a function
2) Added exception safety to preprocess build. Before, when input #2 fail, only one preprocess will be applied to function and it will be corrupted
Exception guard will restore function to original state if exception occurs

* Fix clang-format

docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp

@@ -24,6 +24,8 @@ struct RefPreprocessParams {
         std::function<std::shared_ptr<ov::Function>()> function;
         std::vector<Tensor> inputs;
         std::vector<Tensor> expected;
+        float abs_threshold = 0.01f;
+        float rel_threshold = 0.01f;
         std::string name;
 };
 
@@ -39,6 +41,8 @@ class ReferencePreprocessTest : public testing::TestWithParam<RefPreprocessParam
         for (const auto& exp : params.expected) {
             refOutData.push_back(exp.data);
         }
+        abs_threshold = params.abs_threshold;
+        threshold = params.rel_threshold;
     }
     static std::string getTestCaseName(const testing::TestParamInfo<RefPreprocessParams>& obj) {
         const auto& param = obj.param;
@@ -327,6 +331,26 @@ static RefPreprocessParams resize_from_spatial_dims() {
     return res;
 }
 
+static RefPreprocessParams resize_i8() {
+    RefPreprocessParams res("resize_i8");
+    res.function = []() {
+        auto f = create_simple_function(element::i8, PartialShape{1, 3, 1, 1});
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo()
+                                    .set_spatial_dynamic_shape())
+                               .preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
+                               .network(InputNetworkInfo().set_layout("NCHW")))
+                .build(f);
+        return f;
+    };
+    res.inputs.emplace_back(element::i8, Shape{1, 3, 2, 2}, std::vector<int8_t>{0, 0, 0, 0,
+                                                                                1, 1, 1, 1,
+                                                                                2, 2, 2, 2});
+    res.expected.emplace_back(Shape{1, 3, 1, 1}, element::i8, std::vector<int8_t>{0, 1, 2});
+    return res;
+}
+
 static RefPreprocessParams resize_to_network_width_height() {
     RefPreprocessParams res("resize_to_network_width_height");
     res.function = []() {
@@ -505,6 +529,152 @@ static RefPreprocessParams resize_and_convert_layout() {
     return res;
 }
 
+static RefPreprocessParams convert_color_nv12_to_bgr_two_planes() {
+    RefPreprocessParams res("convert_color_nv12_to_bgr_two_planes");
+    res.abs_threshold = 2.f; // Allow small color conversion deviations
+    res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
+    res.function = []() {
+        auto f = create_simple_function(element::u8, PartialShape{1, 4, 4, 3});
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo()
+                                               .set_color_format(ColorFormat::NV12_TWO_PLANES))
+                               .preprocess(PreProcessSteps()
+                                                   .convert_color(ColorFormat::BGR)))
+                .build(f);
+        return f;
+    };
+
+    // clang-format off
+    auto input_y = std::vector<uint8_t> {81, 81, 145, 145,      // RRGG
+                                         81, 81, 145, 145,      // RRGG
+                                         41, 41, 81, 81,        // BBRR
+                                         41, 41, 81, 81};       // BBRR
+    auto input_shape_y = Shape{1, 4, 4, 1};
+    auto input_uv = std::vector<uint8_t> {240, 90,      // R (2x2)
+                                          34, 54,       // G (2x2)
+                                          110, 240,     // B (2x2)
+                                          240, 90};     // R (2x2)
+    auto input_shape_uv = Shape{1, 2, 2, 2};
+    auto exp_out = std::vector<uint8_t> {0, 0, 255,  0, 0, 255,  0, 255, 0,  0, 255, 0,
+                                         0, 0, 255,  0, 0, 255,  0, 255, 0,  0, 255, 0,
+                                         255, 0, 0,  255, 0, 0,  0, 0, 255,  0, 0, 255,
+                                         255, 0, 0,  255, 0, 0,  0, 0, 255,  0, 0, 255};
+    auto out_shape = Shape{1, 4, 4, 3};
+    // clang-format on
+    res.inputs.emplace_back(element::u8, input_shape_y, input_y);
+    res.inputs.emplace_back(element::u8, input_shape_uv, input_uv);
+    res.expected.emplace_back(out_shape, element::u8, exp_out);
+    return res;
+}
+
+static RefPreprocessParams convert_color_nv12_single_plane() {
+    RefPreprocessParams res("convert_color_nv12_single_plane");
+    res.abs_threshold = 2.f; // Allow small color conversion deviations
+    res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
+    res.function = []() {
+        auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3});
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo()
+                                               .set_color_format(ColorFormat::NV12_SINGLE_PLANE))
+                               .preprocess(PreProcessSteps()
+                                                   .convert_color(ColorFormat::RGB)))
+                .build(f);
+        return f;
+    };
+
+    // clang-format off
+    auto input = std::vector<float> {  81, 81, 145, 145,      // RRGG
+                                       81, 81, 145, 145,      // RRGG
+                                       41, 41, 81, 81,        // BBRR
+                                       41, 41, 81, 81,        // BBRR
+                                       240, 90, 34, 54, 110, 240, 240, 90};     // UV (RGBR)
+    auto input_shape = Shape{1, 6, 4, 1};
+    auto exp_out = std::vector<float> {255, 0, 0,  255, 0, 0,  0, 255, 0,  0, 255, 0,    // RRGG
+                                       255, 0, 0,  255, 0, 0,  0, 255, 0,  0, 255, 0,    // RRGG
+                                       0, 0, 255,  0, 0, 255,  255, 0, 0,  255, 0, 0,    // BBRR
+                                       0, 0, 255,  0, 0, 255,  255, 0, 0,  255, 0, 0,    // BBRR
+                                       };
+    auto out_shape = Shape{1, 4, 4, 3};
+    // clang-format on
+    res.inputs.emplace_back(element::f32, input_shape, input);
+    res.expected.emplace_back(out_shape, element::f32, exp_out);
+    return res;
+}
+
+static RefPreprocessParams convert_color_nv12_layout_resize() {
+    RefPreprocessParams res("convert_color_nv12_layout_resize");
+    res.abs_threshold = 2.f; // Allow small color conversion deviations
+    res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
+    res.function = []() {
+        auto f = create_simple_function(element::f32, PartialShape{1, 3, 2, 2});
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo()
+                                               .set_color_format(ColorFormat::NV12_SINGLE_PLANE)
+                                               .set_element_type(element::u8)
+                                               .set_spatial_dynamic_shape())
+                               .preprocess(PreProcessSteps()
+                                                   .convert_color(ColorFormat::RGB)
+                                                   .convert_layout()
+                                                   .convert_element_type(element::f32)
+                                                   .resize(ResizeAlgorithm::RESIZE_NEAREST))
+                               .network(InputNetworkInfo().set_layout("NCHW")))
+                .build(f);
+        return f;
+    };
+
+    auto result = std::make_shared<HostTensor>();
+    // clang-format off
+    auto input = std::vector<uint8_t> {81, 81, 145, 145,      // RRGG
+                                       81, 81, 145, 145,      // RRGG
+                                       41, 41, 81, 81,        // BBRR
+                                       41, 41, 81, 81,        // BBRR
+                                       240, 90, 34, 54, 110, 240, 240, 90};     // UV (RGBR)
+    auto input_shape = Shape{1, 6, 4, 1};
+    auto exp_out = std::vector<float> {255, 0, 0, 255,     // R channel
+                                       0, 255, 0, 0,       // G channel
+                                       0, 0, 255, 0};      // B channel
+    auto out_shape = Shape{1, 2, 2, 3};
+    // clang-format on
+    res.inputs.emplace_back(element::u8, input_shape, input);
+    res.expected.emplace_back(out_shape, element::f32, exp_out);
+    return res;
+}
+
+static RefPreprocessParams element_type_before_convert_color_nv12() {
+    RefPreprocessParams res("element_type_before_convert_color_nv12");
+    res.abs_threshold = 2.f; // Allow small color conversion deviations
+    res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
+    res.function = []() {
+        auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 3});
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo()
+                                               .set_element_type(element::u8)
+                                               .set_color_format(ColorFormat::NV12_TWO_PLANES))
+                               .preprocess(PreProcessSteps()
+                                                   .convert_element_type(element::f32)
+                                                   .convert_color(ColorFormat::RGB))
+                               .network(InputNetworkInfo().set_layout("NHWC")))
+                .build(f);
+        return f;
+    };
+
+    // clang-format off
+    auto input_y = std::vector<uint8_t> {81, 81, 81, 81};
+    auto input_shape_y = Shape{1, 2, 2, 1};
+    auto input_uv = std::vector<uint8_t> {240, 90};
+    auto input_shape_uv = Shape{1, 1, 1, 2};
+    auto exp_out = std::vector<float> {255, 0, 0,  255, 0, 0,  255, 0, 0,  255, 0,  0};
+    auto out_shape = Shape{1, 2, 2, 3};
+    // clang-format on
+    res.inputs.emplace_back(element::u8, input_shape_y, input_y);
+    res.inputs.emplace_back(element::u8, input_shape_uv, input_uv);
+    res.expected.emplace_back(out_shape, element::f32, exp_out);
+    return res;
+}
 
 std::vector<RefPreprocessParams> allPreprocessTests() {
     return std::vector<RefPreprocessParams> {
@@ -521,12 +691,17 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
         resize_to_network_height(),
         resize_to_network_width(),
         resize_from_spatial_dims(),
+        resize_i8(),
         resize_to_network_width_height(),
         resize_to_specified_width_height(),
         resize_lvalues(),
         convert_layout_nhwc_to_nchw_lvalue(),
         convert_layout_nhwc_to_net_no_tensor_shape(),
-        resize_and_convert_layout()
+        resize_and_convert_layout(),
+        convert_color_nv12_to_bgr_two_planes(),
+        convert_color_nv12_single_plane(),
+        convert_color_nv12_layout_resize(),
+        element_type_before_convert_color_nv12(),
              };
 }
 

inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp

@@ -111,6 +111,8 @@ std::vector<std::string> disabledTestPatterns() {
 
         // Issue 66685
         R"(smoke_PrePostProcess.*resize_linear_nhwc.*)",
+        // Issue 67214
+        R"(smoke_PrePostProcess.*resize_and_convert_layout_i8.*)",
     };
 
 #define FIX_62820 0

inference-engine/tests/functional/shared_test_classes/src/subgraph/preprocess.cpp

@@ -30,6 +30,7 @@ void PrePostProcessTest::SetUp() {
     std::tie(func, targetDevice) = GetParam();
     function = (std::get<0>(func))();
     threshold = std::get<2>(func);
+    abs_threshold = std::get<2>(func);
 }
 
 TEST_P(PrePostProcessTest, CompareWithRefs) {

inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp

@@ -270,6 +270,63 @@ inline std::shared_ptr<Function> resize_and_convert_layout() {
     return function;
 }
 
+inline std::shared_ptr<Function> resize_and_convert_layout_i8() {
+    using namespace ov::preprocess;
+    auto function = create_preprocess_1input(element::i8, PartialShape{1, 30, 20, 3});
+    function = PrePostProcessor()
+            .input(InputInfo()
+                           .tensor(InputTensorInfo()
+                                           .set_layout("NHWC")
+                                           .set_spatial_static_shape(40, 30))
+                           .preprocess(PreProcessSteps()
+                                               .convert_layout()
+                                               .resize(ResizeAlgorithm::RESIZE_LINEAR))
+                           .network(InputNetworkInfo().set_layout("NCHW")))
+            .build(function);
+    return function;
+}
+
+inline std::shared_ptr<Function> cvt_color_nv12_to_rgb_single_plane() {
+    using namespace ov::preprocess;
+    auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3});
+    function = PrePostProcessor()
+            .input(InputInfo()
+                           .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE))
+                           .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
+            .build(function);
+    return function;
+}
+
+inline std::shared_ptr<Function> cvt_color_nv12_to_bgr_two_planes() {
+    using namespace ov::preprocess;
+    auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3});
+    function = PrePostProcessor()
+            .input(InputInfo()
+                           .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
+                           .preprocess(PreProcessSteps().convert_color(ColorFormat::BGR)))
+            .build(function);
+    return function;
+}
+
+inline std::shared_ptr<Function> cvt_color_nv12_cvt_layout_resize() {
+    using namespace ov::preprocess;
+    auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 10, 10});
+    function = PrePostProcessor()
+            .input(InputInfo()
+                           .tensor(InputTensorInfo()
+                                .set_color_format(ColorFormat::NV12_TWO_PLANES)
+                                .set_element_type(element::u8)
+                                .set_spatial_static_shape(20, 20))
+                           .preprocess(PreProcessSteps()
+                                .convert_color(ColorFormat::RGB)
+                                .convert_layout()
+                                .convert_element_type(element::f32)
+                                .resize(ResizeAlgorithm::RESIZE_LINEAR))
+                           .network(InputNetworkInfo().set_layout("NCHW")))
+            .build(function);
+    return function;
+}
+
 inline std::vector<preprocess_func> generic_preprocess_functions() {
     return std::vector<preprocess_func> {
             preprocess_func(mean_only, "mean_only", 0.01f),
@@ -290,6 +347,10 @@ inline std::vector<preprocess_func> generic_preprocess_functions() {
             preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f),
             preprocess_func(resize_cubic, "resize_cubic", 0.01f),
             preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f),
+            preprocess_func(resize_and_convert_layout_i8, "resize_and_convert_layout_i8", 0.01f),
+            preprocess_func(cvt_color_nv12_to_rgb_single_plane, "cvt_color_nv12_to_rgb_single_plane", 2.f),
+            preprocess_func(cvt_color_nv12_to_bgr_two_planes, "cvt_color_nv12_to_bgr_two_planes", 2.f),
+            preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 2.f),
     };
 }
 

ngraph/core/include/openvino/core/preprocess/color_format.hpp

@@ -0,0 +1,20 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+namespace ov {
+namespace preprocess {
+
+/// \brief Color format enumeration for conversion
+enum class ColorFormat {
+    UNDEFINED,
+    NV12_SINGLE_PLANE,  // Image in NV12 format as single tensor
+    NV12_TWO_PLANES,    // Image in NV12 format represented as separate tensors for Y and UV planes
+    RGB,
+    BGR
+};
+
+}  // namespace preprocess
+}  // namespace ov

ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp

@@ -6,6 +6,7 @@
 
 #include "openvino/core/core_visibility.hpp"
 #include "openvino/core/layout.hpp"
+#include "openvino/core/preprocess/color_format.hpp"
 #include "openvino/core/type/element_type.hpp"
 
 namespace ov {
@@ -117,6 +118,44 @@ class OPENVINO_API InputTensorInfo final {
     ///
     /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
     InputTensorInfo&& set_spatial_static_shape(size_t height, size_t width) &&;
+
+    /// \brief Set color format for user's input tensor.
+    ///
+    /// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors
+    /// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for
+    /// convenient usage of plane parameters.
+    ///
+    /// This version allows chaining for Lvalue objects.
+    ///
+    /// \param format Color format of input image.
+    ///
+    /// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified,
+    /// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color
+    /// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane
+    /// inputs, also is specified, number of sub-names shall match with number of planes.
+    ///
+    /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
+    InputTensorInfo& set_color_format(const ov::preprocess::ColorFormat& format,
+                                      const std::vector<std::string>& sub_names = {}) &;
+
+    /// \brief Set color format for user's input tensor.
+    ///
+    /// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors
+    /// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for
+    /// convenient usage of plane parameters.
+    ///
+    /// This version allows chaining for Rvalue objects.
+    ///
+    /// \param format Color format of input image.
+    ///
+    /// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified,
+    /// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color
+    /// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane
+    /// inputs, also is specified, number of sub-names shall match with number of planes.
+    ///
+    /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
+    InputTensorInfo&& set_color_format(const ov::preprocess::ColorFormat& format,
+                                       const std::vector<std::string>& sub_names = {}) &&;
 };
 
 }  // namespace preprocess