diff --git a/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp b/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp index 0c0b6087c19cd1..db98119e53e18e 100644 --- a/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp +++ b/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp @@ -24,6 +24,8 @@ struct RefPreprocessParams { std::function()> function; std::vector inputs; std::vector expected; + float abs_threshold = 0.01f; + float rel_threshold = 0.01f; std::string name; }; @@ -39,6 +41,8 @@ class ReferencePreprocessTest : public testing::TestWithParam& obj) { const auto& param = obj.param; @@ -327,6 +331,26 @@ static RefPreprocessParams resize_from_spatial_dims() { return res; } +static RefPreprocessParams resize_i8() { + RefPreprocessParams res("resize_i8"); + res.function = []() { + auto f = create_simple_function(element::i8, PartialShape{1, 3, 1, 1}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_spatial_dynamic_shape()) + .preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR)) + .network(InputNetworkInfo().set_layout("NCHW"))) + .build(f); + return f; + }; + res.inputs.emplace_back(element::i8, Shape{1, 3, 2, 2}, std::vector{0, 0, 0, 0, + 1, 1, 1, 1, + 2, 2, 2, 2}); + res.expected.emplace_back(Shape{1, 3, 1, 1}, element::i8, std::vector{0, 1, 2}); + return res; +} + static RefPreprocessParams resize_to_network_width_height() { RefPreprocessParams res("resize_to_network_width_height"); res.function = []() { @@ -505,6 +529,152 @@ static RefPreprocessParams resize_and_convert_layout() { return res; } +static RefPreprocessParams convert_color_nv12_to_bgr_two_planes() { + RefPreprocessParams res("convert_color_nv12_to_bgr_two_planes"); + res.abs_threshold = 2.f; // Allow small color conversion deviations + res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%) + res.function = []() { + auto f = create_simple_function(element::u8, PartialShape{1, 4, 4, 3}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps() + .convert_color(ColorFormat::BGR))) + .build(f); + return f; + }; + + // clang-format off + auto input_y = std::vector {81, 81, 145, 145, // RRGG + 81, 81, 145, 145, // RRGG + 41, 41, 81, 81, // BBRR + 41, 41, 81, 81}; // BBRR + auto input_shape_y = Shape{1, 4, 4, 1}; + auto input_uv = std::vector {240, 90, // R (2x2) + 34, 54, // G (2x2) + 110, 240, // B (2x2) + 240, 90}; // R (2x2) + auto input_shape_uv = Shape{1, 2, 2, 2}; + auto exp_out = std::vector {0, 0, 255, 0, 0, 255, 0, 255, 0, 0, 255, 0, + 0, 0, 255, 0, 0, 255, 0, 255, 0, 0, 255, 0, + 255, 0, 0, 255, 0, 0, 0, 0, 255, 0, 0, 255, + 255, 0, 0, 255, 0, 0, 0, 0, 255, 0, 0, 255}; + auto out_shape = Shape{1, 4, 4, 3}; + // clang-format on + res.inputs.emplace_back(element::u8, input_shape_y, input_y); + res.inputs.emplace_back(element::u8, input_shape_uv, input_uv); + res.expected.emplace_back(out_shape, element::u8, exp_out); + return res; +} + +static RefPreprocessParams convert_color_nv12_single_plane() { + RefPreprocessParams res("convert_color_nv12_single_plane"); + res.abs_threshold = 2.f; // Allow small color conversion deviations + res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%) + res.function = []() { + auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_color_format(ColorFormat::NV12_SINGLE_PLANE)) + .preprocess(PreProcessSteps() + .convert_color(ColorFormat::RGB))) + .build(f); + return f; + }; + + // clang-format off + auto input = std::vector { 81, 81, 145, 145, // RRGG + 81, 81, 145, 145, // RRGG + 41, 41, 81, 81, // BBRR + 41, 41, 81, 81, // BBRR + 240, 90, 34, 54, 110, 240, 240, 90}; // UV (RGBR) + auto input_shape = Shape{1, 6, 4, 1}; + auto exp_out = std::vector {255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255, 0, // RRGG + 255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255, 0, // RRGG + 0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, // BBRR + 0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, // BBRR + }; + auto out_shape = Shape{1, 4, 4, 3}; + // clang-format on + res.inputs.emplace_back(element::f32, input_shape, input); + res.expected.emplace_back(out_shape, element::f32, exp_out); + return res; +} + +static RefPreprocessParams convert_color_nv12_layout_resize() { + RefPreprocessParams res("convert_color_nv12_layout_resize"); + res.abs_threshold = 2.f; // Allow small color conversion deviations + res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%) + res.function = []() { + auto f = create_simple_function(element::f32, PartialShape{1, 3, 2, 2}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_color_format(ColorFormat::NV12_SINGLE_PLANE) + .set_element_type(element::u8) + .set_spatial_dynamic_shape()) + .preprocess(PreProcessSteps() + .convert_color(ColorFormat::RGB) + .convert_layout() + .convert_element_type(element::f32) + .resize(ResizeAlgorithm::RESIZE_NEAREST)) + .network(InputNetworkInfo().set_layout("NCHW"))) + .build(f); + return f; + }; + + auto result = std::make_shared(); + // clang-format off + auto input = std::vector {81, 81, 145, 145, // RRGG + 81, 81, 145, 145, // RRGG + 41, 41, 81, 81, // BBRR + 41, 41, 81, 81, // BBRR + 240, 90, 34, 54, 110, 240, 240, 90}; // UV (RGBR) + auto input_shape = Shape{1, 6, 4, 1}; + auto exp_out = std::vector {255, 0, 0, 255, // R channel + 0, 255, 0, 0, // G channel + 0, 0, 255, 0}; // B channel + auto out_shape = Shape{1, 2, 2, 3}; + // clang-format on + res.inputs.emplace_back(element::u8, input_shape, input); + res.expected.emplace_back(out_shape, element::f32, exp_out); + return res; +} + +static RefPreprocessParams element_type_before_convert_color_nv12() { + RefPreprocessParams res("element_type_before_convert_color_nv12"); + res.abs_threshold = 2.f; // Allow small color conversion deviations + res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%) + res.function = []() { + auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 3}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_element_type(element::u8) + .set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps() + .convert_element_type(element::f32) + .convert_color(ColorFormat::RGB)) + .network(InputNetworkInfo().set_layout("NHWC"))) + .build(f); + return f; + }; + + // clang-format off + auto input_y = std::vector {81, 81, 81, 81}; + auto input_shape_y = Shape{1, 2, 2, 1}; + auto input_uv = std::vector {240, 90}; + auto input_shape_uv = Shape{1, 1, 1, 2}; + auto exp_out = std::vector {255, 0, 0, 255, 0, 0, 255, 0, 0, 255, 0, 0}; + auto out_shape = Shape{1, 2, 2, 3}; + // clang-format on + res.inputs.emplace_back(element::u8, input_shape_y, input_y); + res.inputs.emplace_back(element::u8, input_shape_uv, input_uv); + res.expected.emplace_back(out_shape, element::f32, exp_out); + return res; +} std::vector allPreprocessTests() { return std::vector { @@ -521,12 +691,17 @@ std::vector allPreprocessTests() { resize_to_network_height(), resize_to_network_width(), resize_from_spatial_dims(), + resize_i8(), resize_to_network_width_height(), resize_to_specified_width_height(), resize_lvalues(), convert_layout_nhwc_to_nchw_lvalue(), convert_layout_nhwc_to_net_no_tensor_shape(), - resize_and_convert_layout() + resize_and_convert_layout(), + convert_color_nv12_to_bgr_two_planes(), + convert_color_nv12_single_plane(), + convert_color_nv12_layout_resize(), + element_type_before_convert_color_nv12(), }; } diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp index 81264f99b95d05..e5670b491c04d4 100644 --- a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp @@ -111,6 +111,8 @@ std::vector disabledTestPatterns() { // Issue 66685 R"(smoke_PrePostProcess.*resize_linear_nhwc.*)", + // Issue 67214 + R"(smoke_PrePostProcess.*resize_and_convert_layout_i8.*)", }; #define FIX_62820 0 diff --git a/inference-engine/tests/functional/shared_test_classes/src/subgraph/preprocess.cpp b/inference-engine/tests/functional/shared_test_classes/src/subgraph/preprocess.cpp index 78e37cee17a5d8..74584005c8a69f 100644 --- a/inference-engine/tests/functional/shared_test_classes/src/subgraph/preprocess.cpp +++ b/inference-engine/tests/functional/shared_test_classes/src/subgraph/preprocess.cpp @@ -30,6 +30,7 @@ void PrePostProcessTest::SetUp() { std::tie(func, targetDevice) = GetParam(); function = (std::get<0>(func))(); threshold = std::get<2>(func); + abs_threshold = std::get<2>(func); } TEST_P(PrePostProcessTest, CompareWithRefs) { diff --git a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp index 0ec6abc7c5c16c..96521f7d2c20d9 100644 --- a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp +++ b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp @@ -270,6 +270,63 @@ inline std::shared_ptr resize_and_convert_layout() { return function; } +inline std::shared_ptr resize_and_convert_layout_i8() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::i8, PartialShape{1, 30, 20, 3}); + function = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_layout("NHWC") + .set_spatial_static_shape(40, 30)) + .preprocess(PreProcessSteps() + .convert_layout() + .resize(ResizeAlgorithm::RESIZE_LINEAR)) + .network(InputNetworkInfo().set_layout("NCHW"))) + .build(function); + return function; +} + +inline std::shared_ptr cvt_color_nv12_to_rgb_single_plane() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3}); + function = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))) + .build(function); + return function; +} + +inline std::shared_ptr cvt_color_nv12_to_bgr_two_planes() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3}); + function = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::BGR))) + .build(function); + return function; +} + +inline std::shared_ptr cvt_color_nv12_cvt_layout_resize() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 10, 10}); + function = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_color_format(ColorFormat::NV12_TWO_PLANES) + .set_element_type(element::u8) + .set_spatial_static_shape(20, 20)) + .preprocess(PreProcessSteps() + .convert_color(ColorFormat::RGB) + .convert_layout() + .convert_element_type(element::f32) + .resize(ResizeAlgorithm::RESIZE_LINEAR)) + .network(InputNetworkInfo().set_layout("NCHW"))) + .build(function); + return function; +} + inline std::vector generic_preprocess_functions() { return std::vector { preprocess_func(mean_only, "mean_only", 0.01f), @@ -290,6 +347,10 @@ inline std::vector generic_preprocess_functions() { preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f), preprocess_func(resize_cubic, "resize_cubic", 0.01f), preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f), + preprocess_func(resize_and_convert_layout_i8, "resize_and_convert_layout_i8", 0.01f), + preprocess_func(cvt_color_nv12_to_rgb_single_plane, "cvt_color_nv12_to_rgb_single_plane", 2.f), + preprocess_func(cvt_color_nv12_to_bgr_two_planes, "cvt_color_nv12_to_bgr_two_planes", 2.f), + preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 2.f), }; } diff --git a/ngraph/core/include/openvino/core/preprocess/color_format.hpp b/ngraph/core/include/openvino/core/preprocess/color_format.hpp new file mode 100644 index 00000000000000..d8963ac14ce818 --- /dev/null +++ b/ngraph/core/include/openvino/core/preprocess/color_format.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +namespace ov { +namespace preprocess { + +/// \brief Color format enumeration for conversion +enum class ColorFormat { + UNDEFINED, + NV12_SINGLE_PLANE, // Image in NV12 format as single tensor + NV12_TWO_PLANES, // Image in NV12 format represented as separate tensors for Y and UV planes + RGB, + BGR +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp index e064e38800d548..357ac3f37c822e 100644 --- a/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp +++ b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp @@ -6,6 +6,7 @@ #include "openvino/core/core_visibility.hpp" #include "openvino/core/layout.hpp" +#include "openvino/core/preprocess/color_format.hpp" #include "openvino/core/type/element_type.hpp" namespace ov { @@ -117,6 +118,44 @@ class OPENVINO_API InputTensorInfo final { /// /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner. InputTensorInfo&& set_spatial_static_shape(size_t height, size_t width) &&; + + /// \brief Set color format for user's input tensor. + /// + /// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors + /// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for + /// convenient usage of plane parameters. + /// + /// This version allows chaining for Lvalue objects. + /// + /// \param format Color format of input image. + /// + /// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified, + /// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color + /// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane + /// inputs, also is specified, number of sub-names shall match with number of planes. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner. + InputTensorInfo& set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names = {}) &; + + /// \brief Set color format for user's input tensor. + /// + /// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors + /// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for + /// convenient usage of plane parameters. + /// + /// This version allows chaining for Rvalue objects. + /// + /// \param format Color format of input image. + /// + /// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified, + /// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color + /// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane + /// inputs, also is specified, number of sub-names shall match with number of planes. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner. + InputTensorInfo&& set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names = {}) &&; }; } // namespace preprocess diff --git a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp index 8ef191dba00a36..d005e9a78cf608 100644 --- a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp +++ b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp @@ -5,6 +5,7 @@ #pragma once #include "openvino/core/core_visibility.hpp" +#include "openvino/core/preprocess/color_format.hpp" #include "openvino/core/preprocess/resize_algorithm.hpp" #include "openvino/core/type/element_type.hpp" @@ -56,6 +57,26 @@ class OPENVINO_API PreProcessSteps final { /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner PreProcessSteps&& convert_element_type(const ov::element::Type& type) &&; + /// \brief Converts color format for user's input tensor. Requires source color format to be specified by + /// InputTensorInfo::set_color_format. + /// + /// This version allows chaining for Lvalue objects + /// + /// \param dst_format Destination color format of input image + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps& convert_color(const ov::preprocess::ColorFormat& dst_format) &; + + /// \brief Converts color format for user's input tensor. Requires source color format to be specified by + /// InputTensorInfo::set_color_format. + /// + /// This version allows chaining for Rvalue objects. + /// + /// \param dst_format Color format of input image. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps&& convert_color(const ov::preprocess::ColorFormat& dst_format) &&; + /// \brief Add scale preprocess operation - Lvalue version /// Divide each element of input by specified value /// diff --git a/ngraph/core/src/preprocess/color_utils.cpp b/ngraph/core/src/preprocess/color_utils.cpp new file mode 100644 index 00000000000000..60eda3cbb00ada --- /dev/null +++ b/ngraph/core/src/preprocess/color_utils.cpp @@ -0,0 +1,23 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "color_utils.hpp" + +using namespace ov::preprocess; + +std::unique_ptr ColorFormatInfo::get(ColorFormat format) { + std::unique_ptr res; + switch (format) { + case ColorFormat::NV12_SINGLE_PLANE: + res.reset(new ColorFormatInfoNV12_Single(format)); + break; + case ColorFormat::NV12_TWO_PLANES: + res.reset(new ColorFormatInfoNV12_TwoPlanes(format)); + break; + default: + res.reset(new ColorFormatInfo(format)); + break; + } + return res; +} diff --git a/ngraph/core/src/preprocess/color_utils.hpp b/ngraph/core/src/preprocess/color_utils.hpp new file mode 100644 index 00000000000000..3758f849c5f165 --- /dev/null +++ b/ngraph/core/src/preprocess/color_utils.hpp @@ -0,0 +1,143 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/layout.hpp" +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/preprocess/color_format.hpp" + +namespace ov { +namespace preprocess { + +/// \brief Helper function to check if color format represents RGB family +inline bool is_rgb_family(const ColorFormat& format) { + return format == ColorFormat::RGB || format == ColorFormat::BGR; +} + +inline std::string color_format_name(ColorFormat format) { + std::string name; + switch (format) { + case ColorFormat::RGB: + name = "RGB"; + break; + case ColorFormat::BGR: + name = "BGR"; + break; + case ColorFormat::NV12_TWO_PLANES: + name = "NV12 (multi-plane)"; + break; + case ColorFormat::NV12_SINGLE_PLANE: + name = "NV12 (single plane)"; + break; + default: + name = "Unknown"; + break; + } + return name; +} + +/// \brief Internal helper class to get information depending on color format +class ColorFormatInfo { +public: + static std::unique_ptr get(ColorFormat format); + + virtual ~ColorFormatInfo() = default; + + virtual size_t planes_count() const { + return 1; + } + + virtual Layout default_layout() const { + return {}; + } + + // Calculate shape of plane based image shape in NHWC format + PartialShape shape(size_t plane_num, const PartialShape& image_src_shape) const { + OPENVINO_ASSERT(plane_num < planes_count(), + "Internal error: incorrect plane number specified for color format"); + return calculate_shape(plane_num, image_src_shape); + } + + std::string friendly_suffix(size_t plane_num) const { + OPENVINO_ASSERT(plane_num < planes_count(), + "Internal error: incorrect plane number specified for color format"); + return calc_name_suffix(plane_num); + } + +protected: + virtual PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const { + return image_shape; + } + virtual std::string calc_name_suffix(size_t plane_num) const { + return {}; + } + explicit ColorFormatInfo(ColorFormat format) : m_format(format) {} + ColorFormat m_format; +}; + +// --- Derived classes --- +class ColorFormatInfoNV12_Single : public ColorFormatInfo { +public: + explicit ColorFormatInfoNV12_Single(ColorFormat format) : ColorFormatInfo(format) {} + +protected: + PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override { + PartialShape result = image_shape; + if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) { + result[3] = 1; + if (result[1].is_static()) { + result[1] = result[1].get_length() * 3 / 2; + } + } + return result; + } + + Layout default_layout() const override { + return "NHWC"; + } +}; + +class ColorFormatInfoNV12_TwoPlanes : public ColorFormatInfo { +public: + explicit ColorFormatInfoNV12_TwoPlanes(ColorFormat format) : ColorFormatInfo(format) {} + + size_t planes_count() const override { + return 2; + } + +protected: + PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override { + PartialShape result = image_shape; + if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) { + if (plane_num == 0) { + result[3] = 1; + return result; + } else { + // UV plane has half or width and half of height. Number of channels is 2 + if (result[1].is_static()) { + result[1] = result[1].get_length() / 2; + } + if (result[2].is_static()) { + result[2] = result[2].get_length() / 2; + } + result[3] = 2; + } + } + return result; + } + std::string calc_name_suffix(size_t plane_num) const override { + if (plane_num == 0) { + return "/Y"; + } + return "/UV"; + } + + Layout default_layout() const override { + return "NHWC"; + } +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/src/preprocess/function_guard.hpp b/ngraph/core/src/preprocess/function_guard.hpp new file mode 100644 index 00000000000000..c7bad81c923966 --- /dev/null +++ b/ngraph/core/src/preprocess/function_guard.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/function.hpp" + +namespace ov { +namespace preprocess { + +/// \brief Internal guard to make preprocess builder exception-safe +class FunctionGuard { + std::shared_ptr m_function; + ParameterVector m_parameters; + std::map, std::set>> m_backup; + bool m_done = false; + +public: + FunctionGuard(const std::shared_ptr& f) : m_function(f) { + m_parameters = f->get_parameters(); + for (const auto& param : f->get_parameters()) { + m_backup.insert({param, param->output(0).get_target_inputs()}); + } + } + virtual ~FunctionGuard() { + if (!m_done) { + try { + auto params = m_function->get_parameters(); + // Remove parameters added by preprocessing + for (const auto& param : params) { + m_function->remove_parameter(param); + } + // Insert old parameters and update consumers + for (const auto& item : m_backup) { + // Replace consumers + for (auto consumer : item.second) { + consumer.replace_source_output(item.first); + } + } + m_function->add_parameters(m_parameters); + } catch (std::exception& ex) { + // Stress condition, can't recover function to original state + std::cerr << "Unrecoverable error occurred during preprocessing. Function is corrupted, exiting\n"; + exit(EXIT_FAILURE); + } + } + } + void reset() noexcept { + m_done = true; + } +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/src/preprocess/pre_post_process.cpp b/ngraph/core/src/preprocess/pre_post_process.cpp index 0445bc90bfddf4..082a7ae4c7fa91 100644 --- a/ngraph/core/src/preprocess/pre_post_process.cpp +++ b/ngraph/core/src/preprocess/pre_post_process.cpp @@ -4,6 +4,8 @@ #include "openvino/core/preprocess/pre_post_process.hpp" +#include "color_utils.hpp" +#include "function_guard.hpp" #include "ngraph/opsets/opset1.hpp" #include "openvino/core/function.hpp" #include "preprocess_steps_impl.hpp" @@ -66,7 +68,39 @@ class InputTensorInfo::InputTensorInfoImpl { m_spatial_width = static_cast(width); } + const ColorFormat& get_color_format() const { + return m_color_format; + } + + void set_color_format(ColorFormat format, const std::vector& sub_names) { + auto info = ColorFormatInfo::get(format); + if (info->planes_count() == 1) { + OPENVINO_ASSERT(sub_names.empty(), + "Plane names are not allowed for single plane color format '", + color_format_name(format), + "'"); + } else if (!sub_names.empty()) { + OPENVINO_ASSERT(sub_names.size() == info->planes_count(), + "Number of sub-names (", + sub_names.size(), + ") shall match with number of planes for '", + color_format_name(format), + "' color format (", + info->planes_count(), + ")"); + } + m_planes_sub_names = sub_names; + m_color_format = format; + } + + const std::vector& planes_sub_names() const { + return m_planes_sub_names; + } + private: + ColorFormat m_color_format = ColorFormat::UNDEFINED; + std::vector m_planes_sub_names; + element::Type m_type = element::dynamic; bool m_type_set = false; @@ -120,6 +154,7 @@ struct InputInfo::InputInfoImpl { std::unique_ptr m_tensor_data; std::unique_ptr m_preprocess; std::unique_ptr m_network_data; + std::shared_ptr m_resolved_param; }; //-------------- InputInfo ------------------ @@ -181,6 +216,7 @@ PrePostProcessor&& PrePostProcessor::input(InputInfo&& builder) && { } std::shared_ptr PrePostProcessor::build(const std::shared_ptr& function) { + FunctionGuard guard(function); bool tensor_data_updated = false; for (const auto& input : m_impl->in_contexts) { std::shared_ptr param; @@ -200,19 +236,30 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_network_data && input->m_network_data->is_layout_set() && param->get_layout().empty()) { param->set_layout(input->m_network_data->get_layout()); } + input->m_resolved_param = param; + } + + for (const auto& input : m_impl->in_contexts) { + auto param = input->m_resolved_param; auto consumers = param->output(0).get_target_inputs(); if (!input->m_tensor_data) { input->create_tensor_data(param->get_element_type(), param->get_layout()); } - if (!input->m_tensor_data->is_layout_set() && param->get_layout() != Layout()) { - input->m_tensor_data->set_layout(param->get_layout()); - } if (!input->m_tensor_data->is_element_type_set()) { input->m_tensor_data->set_element_type(param->get_element_type()); } + auto color_info = ColorFormatInfo::get(input->m_tensor_data->get_color_format()); + if (!input->m_tensor_data->is_layout_set()) { + if (!color_info->default_layout().empty()) { + input->m_tensor_data->set_layout(color_info->default_layout()); + } else if (!param->get_layout().empty()) { + input->m_tensor_data->set_layout(param->get_layout()); + } + } + auto net_shape = param->get_partial_shape(); auto new_param_shape = net_shape; - if (input->m_tensor_data->is_layout_set() && param->get_layout() != Layout() && + if (input->m_tensor_data->is_layout_set() && !param->get_layout().empty() && param->get_layout() != input->m_tensor_data->get_layout()) { // Find transpose between network and tensor layouts and update tensor shape auto net_to_tensor = @@ -236,26 +283,54 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_tensor_data->get_spatial_width(); } } - auto new_param = std::make_shared(input->m_tensor_data->get_element_type(), new_param_shape); - if (input->m_tensor_data->is_layout_set()) { - new_param->set_layout(input->m_tensor_data->get_layout()); - } - // Old param will be removed, so friendly name can be reused - new_param->set_friendly_name(param->get_friendly_name()); - // Also reuse names of original tensor - new_param->get_output_tensor(0).set_names(param->get_output_tensor(0).get_names()); + std::vector> nodes; + std::vector> new_params; + + // Create separate parameter for each plane. Shape and friendly name is based on color format + for (size_t plane = 0; plane < color_info->planes_count(); plane++) { + auto plane_shape = color_info->shape(plane, new_param_shape); + auto plane_param = + std::make_shared(input->m_tensor_data->get_element_type(), plane_shape); + if (plane < input->m_tensor_data->planes_sub_names().size()) { + auto sub_name = std::string("/") + input->m_tensor_data->planes_sub_names()[plane]; + inherit_friendly_names(function, param, plane_param, sub_name, false); + } else { + auto sub_name = color_info->friendly_suffix(plane); + inherit_friendly_names(function, param, plane_param, sub_name); + } + if (!input->m_tensor_data->get_layout().empty()) { + plane_param->set_layout(input->m_tensor_data->get_layout()); + } + new_params.push_back(plane_param); + nodes.push_back(plane_param); + } - std::shared_ptr node = new_param; - PreprocessingContext context(new_param->get_layout()); + PreprocessingContext context(input->m_tensor_data->get_layout()); + context.color_format() = input->m_tensor_data->get_color_format(); context.network_layout() = param->get_layout(); context.network_shape() = param->get_partial_shape(); + // 2. Apply preprocessing - for (const auto& action : input->m_preprocess->actions()) { - node = std::get<0>(action)({node}, context); - tensor_data_updated |= std::get<1>(action); + if (input->m_preprocess) { + for (const auto& action : input->m_preprocess->actions()) { + auto node = std::get<0>(action)(nodes, function, context); + nodes = {node}; + tensor_data_updated |= std::get<1>(action); + } } + OPENVINO_ASSERT(nodes.size() == 1, + "Multiple plane input is not allowed as network input. Consider using of convert_color " + "preprocessing operation. Current format is '", + color_format_name(context.color_format()), + "'"); + OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::UNDEFINED, + "Network shall have RGB/BGR color format. Consider add 'convert_color' preprocessing operation " + "to convert current color format '", + color_format_name(context.color_format()), + "'to RGB/BGR"); + auto node = nodes[0]; // Check final type OPENVINO_ASSERT(node->get_element_type() == param->get_element_type(), std::string("Element type after preprocessing {") + node->get_element_type().c_type_string() + @@ -267,13 +342,14 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptradd_parameters({new_param}); + function->add_parameters(new_params); // remove old parameter function->remove_parameter(param); } if (tensor_data_updated) { function->validate_nodes_and_infer_types(); } + guard.reset(); return function; } @@ -339,6 +415,18 @@ InputNetworkInfo&& InputNetworkInfo::set_layout(const Layout& layout) && { return std::move(*this); } +InputTensorInfo& InputTensorInfo::set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names) & { + m_impl->set_color_format(format, sub_names); + return *this; +} + +InputTensorInfo&& InputTensorInfo::set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names) && { + m_impl->set_color_format(format, sub_names); + return std::move(*this); +} + // --------------------- PreProcessSteps ------------------ PreProcessSteps::PreProcessSteps() : m_impl(std::unique_ptr(new PreProcessStepsImpl())) {} @@ -432,14 +520,26 @@ PreProcessSteps&& PreProcessSteps::convert_layout(const Layout& dst_layout) && { return std::move(*this); } +PreProcessSteps& PreProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) & { + m_impl->add_convert_color_impl(dst_format); + return *this; +} + +PreProcessSteps&& PreProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) && { + m_impl->add_convert_color_impl(dst_format); + return std::move(*this); +} + PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & { // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types m_impl->actions().emplace_back(std::make_tuple( - [preprocess_cb](const std::vector>& nodes, PreprocessingContext&) { + [preprocess_cb](const std::vector>& nodes, + const std::shared_ptr&, + PreprocessingContext&) -> std::vector> { OPENVINO_ASSERT(nodes.size() == 1, "Can't apply custom preprocessing step for multi-plane input. Suggesting to convert " "current image to RGB/BGR color format using 'convert_color'"); - return preprocess_cb(nodes[0]); + return {preprocess_cb(nodes[0])}; }, true)); return *this; @@ -448,11 +548,13 @@ PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb PreProcessSteps&& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) && { // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types m_impl->actions().emplace_back(std::make_tuple( - [preprocess_cb](const std::vector>& nodes, PreprocessingContext&) { + [preprocess_cb](const std::vector>& nodes, + const std::shared_ptr&, + PreprocessingContext&) -> std::vector> { OPENVINO_ASSERT(nodes.size() == 1, "Can't apply custom preprocessing step for multi-plane input. Suggesting to convert " "current image to RGB/BGR color format using 'convert_color'"); - return preprocess_cb(nodes[0]); + return {preprocess_cb(nodes[0])}; }, true)); return std::move(*this); diff --git a/ngraph/core/src/preprocess/preprocess_steps_impl.cpp b/ngraph/core/src/preprocess/preprocess_steps_impl.cpp index 93c3b22df32b32..7f6cf2eb727fb4 100644 --- a/ngraph/core/src/preprocess/preprocess_steps_impl.cpp +++ b/ngraph/core/src/preprocess/preprocess_steps_impl.cpp @@ -4,9 +4,12 @@ #include "preprocess_steps_impl.hpp" +#include "color_utils.hpp" #include "ngraph/opsets/opset1.hpp" #include "openvino/core/node.hpp" #include "openvino/core/shape.hpp" +#include "openvino/op/nv12_to_bgr.hpp" +#include "openvino/op/nv12_to_rgb.hpp" namespace ov { namespace preprocess { @@ -31,7 +34,9 @@ static Shape construct_mean_scale_shape(const std::shared_ptr& node, void PreProcessSteps::PreProcessStepsImpl::add_scale_impl(const std::vector& values) { m_actions.emplace_back(std::make_tuple( - [values](const std::vector>& nodes, PreprocessingContext& context) { + [values](const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext& context) -> std::vector> { OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply scale preprocessing for empty input."); OPENVINO_ASSERT(nodes.size() == 1, "Can't apply scale preprocessing for multi-plane input. Suggesting to convert current " @@ -43,18 +48,20 @@ void PreProcessSteps::PreProcessStepsImpl::add_scale_impl(const std::vectorset_friendly_name(nodes[0]->get_friendly_name() + "/scale/Divide_Factor"); + inherit_friendly_names(function, nodes[0], constant, "/scale/Divide_Factor"); auto new_op = std::make_shared(nodes[0], constant); - new_op->set_friendly_name(nodes[0]->get_friendly_name() + "/scale/Divide"); - return new_op; + inherit_friendly_names(function, nodes[0], new_op, "/scale/Divide"); + return {new_op}; }, false)); } void PreProcessSteps::PreProcessStepsImpl::add_mean_impl(const std::vector& values) { m_actions.emplace_back(std::make_tuple( - [values](const std::vector>& nodes, PreprocessingContext& context) { + [values](const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext& context) -> std::vector> { OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply mean preprocessing for empty input."); OPENVINO_ASSERT(nodes.size() == 1, "Can't apply scale preprocessing for multi-plane input. Suggesting to convert current " @@ -66,27 +73,30 @@ void PreProcessSteps::PreProcessStepsImpl::add_mean_impl(const std::vectorset_friendly_name(nodes[0]->get_friendly_name() + "/mean/Mean_Const"); + inherit_friendly_names(function, nodes[0], constant, "/mean/Mean_Const"); auto new_op = std::make_shared(nodes[0], constant); - new_op->set_friendly_name(nodes[0]->get_friendly_name() + "/mean/Subtract"); - return new_op; + inherit_friendly_names(function, nodes[0], new_op, "/mean/Subtract"); + return {new_op}; }, false)); } void PreProcessSteps::PreProcessStepsImpl::add_convert_impl(const ov::element::Type& type) { m_actions.emplace_back(std::make_tuple( - [type](const std::vector>& nodes, PreprocessingContext&) { + [type](const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext&) -> std::vector> { OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't set element type for empty input."); - OPENVINO_ASSERT(nodes.size() == 1, - "Can't set element type for multi-plane input. Suggesting to convert current image to " - "RGB/BGR color format using 'convert_color'"); - OPENVINO_ASSERT(nodes[0]->get_element_type().is_static(), - "Can't insert 'convert_element_type' for dynamic source tensor type."); - auto convert = std::make_shared(nodes[0], type); - convert->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_element_type"); - return convert; + std::vector> res; + for (const auto& node : nodes) { + OPENVINO_ASSERT(node->get_element_type().is_static(), + "Can't insert 'convert_element_type' for dynamic source tensor type."); + auto convert = std::make_shared(node, type); + inherit_friendly_names(function, node, convert, "/convert_element_type"); + res.emplace_back(convert); + } + return res; }, true)); } @@ -94,7 +104,9 @@ void PreProcessSteps::PreProcessStepsImpl::add_convert_impl(const ov::element::T void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width) { using InterpolateMode = op::v4::Interpolate::InterpolateMode; m_actions.emplace_back(std::make_tuple( - [alg, dst_width, dst_height](const std::vector>& nodes, PreprocessingContext& ctxt) { + [alg, dst_width, dst_height](const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext& ctxt) -> std::vector> { OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't add resize for empty input."); OPENVINO_ASSERT(nodes.size() == 1, "Can't resize multi-plane input. Suggesting to convert current image to " @@ -140,28 +152,88 @@ void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg, {0, 0}); auto interp = std::make_shared(node, target_spatial_shape, scales, axes, attrs); - interp->set_friendly_name(nodes[0]->get_friendly_name() + "/resize"); - return interp; + inherit_friendly_names(function, nodes[0], interp, "/resize"); + return {interp}; }, true)); } void PreProcessSteps::PreProcessStepsImpl::add_convert_layout_impl(const Layout& layout) { m_actions.emplace_back(std::make_tuple( - [layout](const std::vector>& nodes, PreprocessingContext& context) { + [layout](const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext& context) -> std::vector> { OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input."); OPENVINO_ASSERT(nodes.size() == 1, "Can't convert layout for multi-plane input. Suggesting to convert current image to " - "RGB/BGR color format using 'convert_color'"); + "RGB/BGR color format using 'convert_color'. Current format is '", + color_format_name(context.color_format()), + "'"); Layout dst_layout = layout.empty() ? context.network_layout() : layout; auto permutation = layout::find_permutation(context.layout(), nodes[0]->get_output_partial_shape(0).rank(), dst_layout); auto perm_constant = op::v0::Constant::create(element::i64, Shape{permutation.size()}, permutation); auto transpose = std::make_shared(nodes[0], perm_constant); - transpose->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_layout"); + inherit_friendly_names(function, nodes[0], transpose, "/convert_layout"); context.layout() = dst_layout; // Update context's current layout - return transpose; + return {transpose}; + }, + true)); +} + +void PreProcessSteps::PreProcessStepsImpl::add_convert_color_impl(const ColorFormat& dst_format) { + m_actions.emplace_back(std::make_tuple( + [&, dst_format](const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext& context) -> std::vector> { + if (context.color_format() == dst_format) { + return nodes; + } + if (context.color_format() == ColorFormat::NV12_SINGLE_PLANE) { + OPENVINO_ASSERT(nodes.size() == 1, + "Internal error: single plane NV12 image can't have multiple inputs"); + std::shared_ptr convert; + switch (dst_format) { + case ColorFormat::RGB: + convert = std::make_shared(nodes[0]); + break; + case ColorFormat::BGR: + convert = std::make_shared(nodes[0]); + break; + default: + OPENVINO_ASSERT(false, + "Unsupported conversion from NV12 to '", + color_format_name(dst_format), + "' format:"); + } + inherit_friendly_names(function, nodes[0], convert, "/convert_color_nv12_single"); + context.color_format() = dst_format; + return {convert}; + } else if (context.color_format() == ColorFormat::NV12_TWO_PLANES) { + OPENVINO_ASSERT(nodes.size() == 2, "Internal error: two-plane NV12 image must have exactly two inputs"); + std::shared_ptr convert; + switch (dst_format) { + case ColorFormat::RGB: + convert = std::make_shared(nodes[0], nodes[1]); + break; + case ColorFormat::BGR: + convert = std::make_shared(nodes[0], nodes[1]); + break; + default: + OPENVINO_ASSERT(false, + "Unsupported conversion from NV12 to '", + color_format_name(dst_format), + "' format:"); + } + inherit_friendly_names(function, nodes[0], convert, "/convert_color_nv12_two_planes"); + context.color_format() = dst_format; + return {convert}; + } + OPENVINO_ASSERT(false, + "Source color format '", + color_format_name(context.color_format()), + "' is not convertible to any other"); }, true)); } diff --git a/ngraph/core/src/preprocess/preprocess_steps_impl.hpp b/ngraph/core/src/preprocess/preprocess_steps_impl.hpp index 120c6849fd97a4..e4017af7ad5215 100644 --- a/ngraph/core/src/preprocess/preprocess_steps_impl.hpp +++ b/ngraph/core/src/preprocess/preprocess_steps_impl.hpp @@ -7,8 +7,11 @@ #include #include "openvino/core/layout.hpp" +#include "openvino/core/node.hpp" #include "openvino/core/partial_shape.hpp" +#include "openvino/core/preprocess/color_format.hpp" #include "openvino/core/preprocess/preprocess_steps.hpp" +#include "tensor_name_util.hpp" namespace ov { namespace preprocess { @@ -55,11 +58,37 @@ inline size_t get_and_check_channels_idx(const Layout& layout, const PartialShap return idx; } +inline void inherit_friendly_names(const std::shared_ptr& function, + const std::shared_ptr& src_node, + const std::shared_ptr& dst_node, + const std::string& suffix, + bool search_for_available_name = true) { + OPENVINO_ASSERT(src_node->get_output_size() == 1 && dst_node->get_output_size() == 1, + "Internal error. Preprocessing steps must contain nodes with one output"); + dst_node->set_friendly_name(src_node->get_friendly_name() + suffix); + std::unordered_set new_names; + for (const auto& tensor_name : src_node->output(0).get_tensor().get_names()) { + auto new_tensor_name = tensor_name + suffix; + if (!suffix.empty()) { + // Verify that new names are unique for a function + if (!is_tensor_name_available(new_tensor_name, function) && search_for_available_name) { + // Search for available name + size_t idx = 0; + do { + new_tensor_name = tensor_name + suffix + std::to_string(idx++); + } while (!is_tensor_name_available(new_tensor_name, function)); + } + } + new_names.emplace(new_tensor_name); + } + dst_node->output(0).get_tensor().set_names(new_names); +} + /// \brief Preprocessing context passed to each preprocessing operation. /// This is internal structure which is not shared to custom operations yet. class PreprocessingContext { public: - explicit PreprocessingContext(const Layout& layout) : m_layout(layout) {} + explicit PreprocessingContext(Layout layout) : m_layout(std::move(layout)) {} const Layout& layout() const { return m_layout; @@ -99,15 +128,25 @@ class PreprocessingContext { return network_shape()[network_width_idx].get_length(); } + const ColorFormat& color_format() const { + return m_color_format; + } + + ColorFormat& color_format() { + return m_color_format; + } + private: Layout m_layout; PartialShape m_network_shape; Layout m_network_layout; + ColorFormat m_color_format = ColorFormat::UNDEFINED; }; using InternalPreprocessOp = - std::function(const std::vector>& nodes, - PreprocessingContext& context)>; + std::function>(const std::vector>& nodes, + const std::shared_ptr& function, + PreprocessingContext& context)>; /// \brief PreProcessStepsImpl - internal data structure class PreProcessSteps::PreProcessStepsImpl { @@ -117,6 +156,7 @@ class PreProcessSteps::PreProcessStepsImpl { void add_convert_impl(const element::Type& type); void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width); void add_convert_layout_impl(const Layout& layout); + void add_convert_color_impl(const ColorFormat& dst_format); const std::list>& actions() const { return m_actions; diff --git a/ngraph/core/src/tensor_name_util.hpp b/ngraph/core/src/tensor_name_util.hpp new file mode 100644 index 00000000000000..b77c248561d605 --- /dev/null +++ b/ngraph/core/src/tensor_name_util.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/except.hpp" +#include "openvino/core/function.hpp" + +namespace ov { + +/// \brief Check that specified tensor name is unique for a given function. +/// +/// \param tensor_name Name to check across all tensors in a function. +/// \param function Function. +/// \return False if tensor name is already used in some function's node, True otherwise +inline bool is_tensor_name_available(const std::string& tensor_name, const std::shared_ptr& function) { + for (const auto& node : function->get_ordered_ops()) { + for (const auto& output : node->outputs()) { + const auto& tensor = output.get_tensor(); + if (tensor.get_names().count(tensor_name)) { + return false; + } + } + } + return true; +} + +} // namespace ov diff --git a/ngraph/test/preprocess.cpp b/ngraph/test/preprocess.cpp index 94d6863eeda590..dc1669425d12a8 100644 --- a/ngraph/test/preprocess.cpp +++ b/ngraph/test/preprocess.cpp @@ -57,6 +57,13 @@ TEST(pre_post_process, convert_element_type_and_scale) { EXPECT_EQ(f->get_output_element_type(0), element::i8); } +TEST(pre_post_process, empty_preprocess) { + auto f = create_simple_function(element::i8, Shape{1, 3, 2, 2}); + f = PrePostProcessor().input(InputInfo().tensor(InputTensorInfo().set_element_type(element::i8))).build(f); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i8); + EXPECT_EQ(f->get_output_element_type(0), element::i8); +} + TEST(pre_post_process, convert_element_type_from_unknown) { auto f = create_simple_function(element::i32, Shape{1, 3, 224, 224}); ASSERT_THROW( @@ -107,6 +114,265 @@ TEST(pre_post_process, tensor_element_type_and_scale) { EXPECT_EQ(f->get_parameters().front()->get_layout(), Layout()); } +TEST(pre_post_process, convert_color_nv12_rgb_single) { + auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 2, 2, 3}); + auto name = f->get_parameters()[0]->get_friendly_name(); + auto tensor_names = f->get_parameters().front()->get_output_tensor(0).get_names(); + f = PrePostProcessor() + .input( + InputInfo() + .tensor(InputTensorInfo() + .set_element_type(element::u8) + .set_color_format(ColorFormat::NV12_SINGLE_PLANE)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB).convert_element_type(element::f32))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 1); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::u8); + EXPECT_EQ(f->get_parameters().front()->get_layout(), "NHWC"); + EXPECT_EQ(f->get_parameters().front()->get_partial_shape(), (PartialShape{Dimension::dynamic(), 3, 2, 1})); + EXPECT_EQ(f->get_parameters().front()->get_friendly_name(), name); + EXPECT_EQ(f->get_parameters().front()->get_output_tensor(0).get_names(), tensor_names); +} + +TEST(pre_post_process, convert_color_nv12_bgr_single) { + auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 2, 2, 3}); + auto name = f->get_parameters()[0]->get_friendly_name(); + auto tensor_names = f->get_parameters().front()->get_output_tensor(0).get_names(); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::BGR))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 1); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters().front()->get_layout(), "NHWC"); + EXPECT_EQ(f->get_parameters().front()->get_partial_shape(), (PartialShape{Dimension::dynamic(), 3, 2, 1})); + EXPECT_EQ(f->get_parameters().front()->get_friendly_name(), name); + EXPECT_EQ(f->get_parameters().front()->get_output_tensor(0).get_names(), tensor_names); +} + +TEST(pre_post_process, convert_color_nv12_bgr_2_planes) { + auto f = create_simple_function(element::f32, Shape{5, 2, 2, 3}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"TestY", "TestUV"})) + .preprocess(PreProcessSteps().convert_color(ColorFormat::BGR))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 2); + EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), "input1/TestY"); + EXPECT_EQ(*f->get_parameters()[0]->output(0).get_tensor().get_names().begin(), "tensor_input1/TestY"); + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{5, 2, 2, 1})); + + EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), "input1/TestUV"); + EXPECT_EQ(*f->get_parameters()[1]->output(0).get_tensor().get_names().begin(), "tensor_input1/TestUV"); + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{5, 1, 1, 2})); +} + +TEST(pre_post_process, convert_color_nv12_rgb_2_planes) { + auto f = create_simple_function(element::f32, Shape{5, 2, 2, 3}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 2); + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{5, 2, 2, 1})); + EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{5, 1, 1, 2})); + + EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), "input1/Y"); + EXPECT_EQ(*f->get_parameters()[0]->output(0).get_tensor().get_names().begin(), "tensor_input1/Y"); + + EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), "input1/UV"); + EXPECT_EQ(*f->get_parameters()[1]->output(0).get_tensor().get_names().begin(), "tensor_input1/UV"); +} + +TEST(pre_post_process, convert_color_nv12_bgr_2_planes_u8_lvalue) { + auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3}); + auto input_tensor_info = InputTensorInfo(); + input_tensor_info.set_color_format(ColorFormat::NV12_TWO_PLANES); + auto steps = PreProcessSteps(); + steps.convert_color(ColorFormat::BGR); + f = PrePostProcessor() + .input(InputInfo().tensor(std::move(input_tensor_info)).preprocess(std::move(steps))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 2); + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::u8); + EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{1, 2, 2, 1})); + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::u8); + EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{1, 1, 1, 2})); +} + +TEST(pre_post_process, convert_color_nv12_bgr_2_planes_el_type) { + auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3}); + EXPECT_NO_THROW( + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_element_type(element::f32) + .set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess( + PreProcessSteps().convert_element_type(element::u8).convert_color(ColorFormat::BGR))) + .build(f)); + + EXPECT_EQ(f->get_parameters().size(), 2); + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32); +} + +TEST(pre_post_process, convert_color_same_type) { + auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3}); + EXPECT_NO_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::RGB)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))) + .build(f)); + + EXPECT_EQ(f->get_parameters().size(), 1); + EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{1, 2, 2, 3})); +} + +TEST(pre_post_process, convert_color_unsupported) { + // Feel free to update this test when more color conversions are supported in future + auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3}); + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::UNDEFINED))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::UNDEFINED))) + .build(f), + ov::AssertFailure); + + auto colors = {ColorFormat::NV12_TWO_PLANES, ColorFormat::NV12_SINGLE_PLANE, ColorFormat::RGB, ColorFormat::BGR}; + for (const auto& color : colors) { + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::UNDEFINED)) + .preprocess(PreProcessSteps().convert_color(color))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(color)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::UNDEFINED))) + .build(f), + ov::AssertFailure); + } +} + +TEST(pre_post_process, convert_color_incorrect_subnames) { + auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 2, 2, 3}); + auto name = f->get_parameters()[0]->get_friendly_name(); + auto tensor_names = f->get_parameters().front()->get_output_tensor(0).get_names(); + EXPECT_THROW( + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE, {"Test"})) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW( + f = PrePostProcessor() + .input(InputInfo().tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"Test"}))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo().tensor( + InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"1", "2", "3"}))) + .build(f), + ov::AssertFailure); +} + +TEST(pre_post_process, convert_color_duplicate_subnames) { + auto f = create_2inputs(element::f32, PartialShape{1, 2, 2, 3}); + f->get_parameters()[0]->get_output_tensor(0).set_names({"tensor_input1"}); + f->get_parameters()[1]->get_output_tensor(0).set_names({"tensor_input1/CustomUV"}); + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE, + {"CustomY", "CustomUV"})) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB))) + .build(f), + ov::AssertFailure); +} + +TEST(pre_post_process, convert_color_duplicate_internal_subnames_mean) { + auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 3}); + for (int i = 0; i < 10; i++) { + // Create preprocessing step several times (try to duplicate internal node names this way) + EXPECT_NO_THROW(f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().mean(0.1f))).build(f)); + EXPECT_NO_THROW(f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().scale(1.1f))).build(f)); + EXPECT_NO_THROW( + f = PrePostProcessor() + .input(InputInfo().preprocess( + PreProcessSteps().convert_element_type(element::u8).convert_element_type(element::f32))) + .build(f)); + EXPECT_NO_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_layout("NHWC")) + .preprocess(PreProcessSteps().convert_layout("NCHW"))) + .build(f)); + EXPECT_NO_THROW( + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_layout("NHWC").set_spatial_static_shape(480, 640)) + .preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))) + .build(f)); + } +} + +TEST(pre_post_process, unsupported_network_color_format) { + auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3}); + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo().tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo().tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW( + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().convert_layout("NCHW").convert_color(ColorFormat::RGB))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().mean(0.1f).convert_color(ColorFormat::RGB))) + .build(f), + ov::AssertFailure); + + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().scale(2.1f).convert_color(ColorFormat::RGB))) + .build(f), + ov::AssertFailure); +} + TEST(pre_post_process, custom_preprocessing) { auto f = create_simple_function(element::i32, Shape{1, 3, 1, 1}); f = PrePostProcessor() @@ -327,3 +593,34 @@ TEST(pre_post_process, resize_no_tensor_width) { .build(f), ov::AssertFailure); } + +TEST(pre_post_process, exception_safety) { + auto f = create_2inputs(element::f32, Shape{1, 3, 224, 224}); + auto name0 = f->get_parameters()[0]->get_friendly_name(); + auto tensor_names0 = f->get_parameters()[0]->get_output_tensor(0).get_names(); + auto name1 = f->get_parameters()[1]->get_friendly_name(); + auto tensor_names1 = f->get_parameters()[1]->get_output_tensor(0).get_names(); + EXPECT_THROW(f = PrePostProcessor() + .input(InputInfo(0) // this one is correct + .tensor(InputTensorInfo().set_element_type(element::u8)) + .preprocess(PreProcessSteps().convert_element_type(element::f32))) + .input(InputInfo(1) // This one is not + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)) + .preprocess(PreProcessSteps().custom( + [](const std::shared_ptr& node) -> std::shared_ptr { + throw ngraph::ngraph_error("test error"); + }))) + .build(f), + ov::AssertFailure); + EXPECT_EQ(f->get_parameters().size(), 2); + + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{1, 3, 224, 224})); + EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), name0); + EXPECT_EQ(f->get_parameters()[0]->get_output_tensor(0).get_names(), tensor_names0); + + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{1, 3, 224, 224})); + EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), name1); + EXPECT_EQ(f->get_parameters()[1]->get_output_tensor(0).get_names(), tensor_names1); +}