diff --git a/ngraph/core/include/openvino/core/preprocess/color_format.hpp b/ngraph/core/include/openvino/core/preprocess/color_format.hpp new file mode 100644 index 00000000000000..d8963ac14ce818 --- /dev/null +++ b/ngraph/core/include/openvino/core/preprocess/color_format.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +namespace ov { +namespace preprocess { + +/// \brief Color format enumeration for conversion +enum class ColorFormat { + UNDEFINED, + NV12_SINGLE_PLANE, // Image in NV12 format as single tensor + NV12_TWO_PLANES, // Image in NV12 format represented as separate tensors for Y and UV planes + RGB, + BGR +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp index e064e38800d548..5974ccad41b48b 100644 --- a/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp +++ b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp @@ -6,6 +6,7 @@ #include "openvino/core/core_visibility.hpp" #include "openvino/core/layout.hpp" +#include "openvino/core/preprocess/color_format.hpp" #include "openvino/core/type/element_type.hpp" namespace ov { @@ -117,6 +118,42 @@ class OPENVINO_API InputTensorInfo final { /// /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner. InputTensorInfo&& set_spatial_static_shape(size_t height, size_t width) &&; + + /// \brief Set color format for user's input tensor + /// + /// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors + /// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for + /// convenient usage TBD: example + /// + /// This version allows chaining for Lvalue objects + /// + /// \param format Color format of input image + /// + /// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified, + /// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color + /// format, and client's code shall not rely on these rules. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + InputTensorInfo& set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names = {}) &; + + /// \brief Set color format for user's input tensor + /// + /// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors + /// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for + /// convenient usage TBD: example + /// + /// This version allows chaining for Rvalue objects + /// + /// \param format Color format of input image + /// + /// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified, + /// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color + /// format, and client's code shall not rely on these rules. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + InputTensorInfo&& set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names = {}) &&; }; } // namespace preprocess diff --git a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp index 8ef191dba00a36..d005e9a78cf608 100644 --- a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp +++ b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp @@ -5,6 +5,7 @@ #pragma once #include "openvino/core/core_visibility.hpp" +#include "openvino/core/preprocess/color_format.hpp" #include "openvino/core/preprocess/resize_algorithm.hpp" #include "openvino/core/type/element_type.hpp" @@ -56,6 +57,26 @@ class OPENVINO_API PreProcessSteps final { /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner PreProcessSteps&& convert_element_type(const ov::element::Type& type) &&; + /// \brief Converts color format for user's input tensor. Requires source color format to be specified by + /// InputTensorInfo::set_color_format. + /// + /// This version allows chaining for Lvalue objects + /// + /// \param dst_format Destination color format of input image + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps& convert_color(const ov::preprocess::ColorFormat& dst_format) &; + + /// \brief Converts color format for user's input tensor. Requires source color format to be specified by + /// InputTensorInfo::set_color_format. + /// + /// This version allows chaining for Rvalue objects. + /// + /// \param dst_format Color format of input image. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps&& convert_color(const ov::preprocess::ColorFormat& dst_format) &&; + /// \brief Add scale preprocess operation - Lvalue version /// Divide each element of input by specified value /// diff --git a/ngraph/core/src/preprocess/color_utils.cpp b/ngraph/core/src/preprocess/color_utils.cpp new file mode 100644 index 00000000000000..60eda3cbb00ada --- /dev/null +++ b/ngraph/core/src/preprocess/color_utils.cpp @@ -0,0 +1,23 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "color_utils.hpp" + +using namespace ov::preprocess; + +std::unique_ptr ColorFormatInfo::get(ColorFormat format) { + std::unique_ptr res; + switch (format) { + case ColorFormat::NV12_SINGLE_PLANE: + res.reset(new ColorFormatInfoNV12_Single(format)); + break; + case ColorFormat::NV12_TWO_PLANES: + res.reset(new ColorFormatInfoNV12_TwoPlanes(format)); + break; + default: + res.reset(new ColorFormatInfo(format)); + break; + } + return res; +} diff --git a/ngraph/core/src/preprocess/color_utils.hpp b/ngraph/core/src/preprocess/color_utils.hpp new file mode 100644 index 00000000000000..d5a825612bde08 --- /dev/null +++ b/ngraph/core/src/preprocess/color_utils.hpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/preprocess/color_format.hpp" + +namespace ov { +namespace preprocess { + +/// \brief Helper function to check if color format represents RGB family +inline bool is_rgb_family(const ColorFormat& format) { + return format == ColorFormat::RGB || format == ColorFormat::BGR; +} + +/// \brief Internal helper class to get information depending on color format +class ColorFormatInfo { +public: + static std::unique_ptr get(ColorFormat format); + + virtual ~ColorFormatInfo() = default; + + virtual size_t planes_count() const { + return 1; + } + // Calculate shape of plane based image shape in NHWC format + PartialShape shape(size_t plane_num, const PartialShape& image_src_shape) const { + OPENVINO_ASSERT(plane_num < planes_count(), + "Internal error: incorrect plane number specified for color format"); + return calculate_shape(plane_num, image_src_shape); + } + + std::string friendly_name(size_t plane_num, const std::string& original_name) const { + OPENVINO_ASSERT(plane_num < planes_count(), + "Internal error: incorrect plane number specified for color format"); + return calc_friendly_name(plane_num, original_name); + } + +protected: + virtual PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const { + return image_shape; + } + virtual std::string calc_friendly_name(size_t plane_num, const std::string& original_name) const { + return original_name; + } + explicit ColorFormatInfo(ColorFormat format) : m_format(format) {} + ColorFormat m_format; +}; + +// --- Derived classes --- +class ColorFormatInfoNV12_Single : public ColorFormatInfo { +public: + explicit ColorFormatInfoNV12_Single(ColorFormat format) : ColorFormatInfo(format) {} + +protected: + PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override { + PartialShape result = image_shape; + if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) { + result[3] = 1; + if (result[1].is_static()) { + result[1] = result[1].get_length() * 3 / 2; + } + } + return result; + } +}; + +class ColorFormatInfoNV12_TwoPlanes : public ColorFormatInfo { +public: + explicit ColorFormatInfoNV12_TwoPlanes(ColorFormat format) : ColorFormatInfo(format) {} + + size_t planes_count() const override { + return 2; + } + +protected: + PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override { + PartialShape result = image_shape; + if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) { + if (plane_num == 0) { + result[3] = 1; + return result; + } else { + // UV plane has half or width and half of height. Number of channels is 2 + if (result[1].is_static()) { + result[1] = result[1].get_length() / 2; + } + if (result[2].is_static()) { + result[2] = result[2].get_length() / 2; + } + result[3] = 2; + } + } + return result; + } + std::string calc_friendly_name(size_t plane_num, const std::string& original_name) const override { + if (plane_num == 0) { + return original_name + "/Y"; + } + return original_name + "/UV"; + } +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/src/preprocess/pre_post_process.cpp b/ngraph/core/src/preprocess/pre_post_process.cpp index 0445bc90bfddf4..e476a2e16d698e 100644 --- a/ngraph/core/src/preprocess/pre_post_process.cpp +++ b/ngraph/core/src/preprocess/pre_post_process.cpp @@ -4,6 +4,7 @@ #include "openvino/core/preprocess/pre_post_process.hpp" +#include "color_utils.hpp" #include "ngraph/opsets/opset1.hpp" #include "openvino/core/function.hpp" #include "preprocess_steps_impl.hpp" @@ -66,7 +67,26 @@ class InputTensorInfo::InputTensorInfoImpl { m_spatial_width = static_cast(width); } + const ColorFormat& get_color_format() const { + return m_color_format; + } + + void set_color_format(ColorFormat format) noexcept { + m_color_format = format; + } + + const std::vector& planes_sub_names() const { + return m_planes_sub_names; + } + + void set_planes_sub_names(const std::vector& names) { + m_planes_sub_names = names; + } + private: + ColorFormat m_color_format = ColorFormat::UNDEFINED; + std::vector m_planes_sub_names; + element::Type m_type = element::dynamic; bool m_type_set = false; @@ -120,6 +140,7 @@ struct InputInfo::InputInfoImpl { std::unique_ptr m_tensor_data; std::unique_ptr m_preprocess; std::unique_ptr m_network_data; + std::shared_ptr m_resolved_param; }; //-------------- InputInfo ------------------ @@ -200,6 +221,11 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_network_data && input->m_network_data->is_layout_set() && param->get_layout().empty()) { param->set_layout(input->m_network_data->get_layout()); } + input->m_resolved_param = param; + } + + for (const auto& input : m_impl->in_contexts) { + auto param = input->m_resolved_param; auto consumers = param->output(0).get_target_inputs(); if (!input->m_tensor_data) { input->create_tensor_data(param->get_element_type(), param->get_layout()); @@ -210,6 +236,7 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_tensor_data->is_element_type_set()) { input->m_tensor_data->set_element_type(param->get_element_type()); } + auto net_shape = param->get_partial_shape(); auto new_param_shape = net_shape; if (input->m_tensor_data->is_layout_set() && param->get_layout() != Layout() && @@ -236,26 +263,53 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_tensor_data->get_spatial_width(); } } - auto new_param = std::make_shared(input->m_tensor_data->get_element_type(), new_param_shape); - if (input->m_tensor_data->is_layout_set()) { - new_param->set_layout(input->m_tensor_data->get_layout()); - } - // Old param will be removed, so friendly name can be reused - new_param->set_friendly_name(param->get_friendly_name()); - // Also reuse names of original tensor - new_param->get_output_tensor(0).set_names(param->get_output_tensor(0).get_names()); + std::vector> nodes; + std::vector> new_params; + + // Create separate parameter for each plane. Shape and friendly name is based on color format + auto color_info = ColorFormatInfo::get(input->m_tensor_data->get_color_format()); + for (size_t plane = 0; plane < color_info->planes_count(); plane++) { + auto plane_shape = color_info->shape(plane, new_param_shape); + auto plane_param = + std::make_shared(input->m_tensor_data->get_element_type(), plane_shape); + if (plane < input->m_tensor_data->planes_sub_names().size()) { + auto sub_name = std::string("/") + input->m_tensor_data->planes_sub_names()[plane]; + plane_param->set_friendly_name(param->get_friendly_name() + sub_name); + std::unordered_set plane_names; + for (const auto& name : param->get_output_tensor(0).get_names()) { + plane_names.insert(name + sub_name); + } + plane_param->get_output_tensor(0).set_names(plane_names); + } else { + plane_param->set_friendly_name(color_info->friendly_name(plane, param->get_friendly_name())); + plane_param->get_output_tensor(0).set_names(param->get_output_tensor(0).get_names()); + } + if (input->m_tensor_data->get_layout() != Layout()) { + plane_param->set_layout(input->m_tensor_data->get_layout()); + } + new_params.push_back(plane_param); + nodes.push_back(plane_param); + } - std::shared_ptr node = new_param; - PreprocessingContext context(new_param->get_layout()); + PreprocessingContext context(input->m_tensor_data->get_layout()); + context.color_format() = input->m_tensor_data->get_color_format(); context.network_layout() = param->get_layout(); context.network_shape() = param->get_partial_shape(); + // 2. Apply preprocessing for (const auto& action : input->m_preprocess->actions()) { - node = std::get<0>(action)({node}, context); + auto node = std::get<0>(action)(nodes, context); + nodes = {node}; tensor_data_updated |= std::get<1>(action); } + OPENVINO_ASSERT(nodes.size() == 1, + "Multiple plane input is not allowed as network input. Consider using of convert_color " + "preprocessing operation"); + OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::UNDEFINED, + "Final input color format is incorrect"); + auto node = nodes[0]; // Check final type OPENVINO_ASSERT(node->get_element_type() == param->get_element_type(), std::string("Element type after preprocessing {") + node->get_element_type().c_type_string() + @@ -267,7 +321,7 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptradd_parameters({new_param}); + function->add_parameters(new_params); // remove old parameter function->remove_parameter(param); } @@ -339,6 +393,20 @@ InputNetworkInfo&& InputNetworkInfo::set_layout(const Layout& layout) && { return std::move(*this); } +InputTensorInfo& InputTensorInfo::set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names) & { + m_impl->set_planes_sub_names(sub_names); + m_impl->set_color_format(format); // noexcept + return *this; +} + +InputTensorInfo&& InputTensorInfo::set_color_format(const ov::preprocess::ColorFormat& format, + const std::vector& sub_names) && { + m_impl->set_planes_sub_names(sub_names); + m_impl->set_color_format(format); // noexcept + return std::move(*this); +} + // --------------------- PreProcessSteps ------------------ PreProcessSteps::PreProcessSteps() : m_impl(std::unique_ptr(new PreProcessStepsImpl())) {} @@ -432,6 +500,16 @@ PreProcessSteps&& PreProcessSteps::convert_layout(const Layout& dst_layout) && { return std::move(*this); } +PreProcessSteps& PreProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) & { + m_impl->add_convert_color_impl(dst_format); + return *this; +} + +PreProcessSteps&& PreProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) && { + m_impl->add_convert_color_impl(dst_format); + return std::move(*this); +} + PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & { // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types m_impl->actions().emplace_back(std::make_tuple( diff --git a/ngraph/core/src/preprocess/preprocess_steps_impl.cpp b/ngraph/core/src/preprocess/preprocess_steps_impl.cpp index 93c3b22df32b32..bded5e15c4d23d 100644 --- a/ngraph/core/src/preprocess/preprocess_steps_impl.cpp +++ b/ngraph/core/src/preprocess/preprocess_steps_impl.cpp @@ -7,6 +7,8 @@ #include "ngraph/opsets/opset1.hpp" #include "openvino/core/node.hpp" #include "openvino/core/shape.hpp" +#include "openvino/op/nv12_to_bgr.hpp" +#include "openvino/op/nv12_to_rgb.hpp" namespace ov { namespace preprocess { @@ -166,5 +168,49 @@ void PreProcessSteps::PreProcessStepsImpl::add_convert_layout_impl(const Layout& true)); } +void PreProcessSteps::PreProcessStepsImpl::add_convert_color_impl(const ColorFormat& dst_format) { + m_actions.emplace_back(std::make_tuple( + [&, dst_format](const std::vector>& nodes, PreprocessingContext& context) { + if (context.color_format() == ColorFormat::NV12_SINGLE_PLANE) { + OPENVINO_ASSERT(nodes.size() == 1, + "Internal error: single plane NV12 image can't have multiple inputs"); + std::shared_ptr convert; + switch (dst_format) { + case ColorFormat::RGB: + convert = std::make_shared(nodes[0]); + break; + case ColorFormat::BGR: + convert = std::make_shared(nodes[0]); + break; + default: + OPENVINO_ASSERT(false, "Unsupported NV12 conversion format"); + } + convert->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_color_nv12_single"); + context.color_format() = dst_format; + return convert; + } else if (context.color_format() == ColorFormat::NV12_TWO_PLANES) { + OPENVINO_ASSERT(nodes.size() == 2, "Internal error: two-plane NV12 image must have exactly two inputs"); + std::shared_ptr convert; + switch (dst_format) { + case ColorFormat::RGB: + convert = std::make_shared(nodes[0], nodes[1]); + break; + case ColorFormat::BGR: + convert = std::make_shared(nodes[0], nodes[1]); + break; + default: + OPENVINO_ASSERT(false, "Unsupported NV12 conversion format"); + } + convert->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_color_nv12_two_planes"); + context.color_format() = dst_format; + return convert; + } + // Throw even if source_format == dst_format, because we don't want to deal with multiple output nodes + // here + OPENVINO_ASSERT(false, "Source color format is not convertible to any other"); + }, + true)); +} + } // namespace preprocess } // namespace ov diff --git a/ngraph/core/src/preprocess/preprocess_steps_impl.hpp b/ngraph/core/src/preprocess/preprocess_steps_impl.hpp index 120c6849fd97a4..e581f200a57343 100644 --- a/ngraph/core/src/preprocess/preprocess_steps_impl.hpp +++ b/ngraph/core/src/preprocess/preprocess_steps_impl.hpp @@ -8,6 +8,7 @@ #include "openvino/core/layout.hpp" #include "openvino/core/partial_shape.hpp" +#include "openvino/core/preprocess/color_format.hpp" #include "openvino/core/preprocess/preprocess_steps.hpp" namespace ov { @@ -59,7 +60,7 @@ inline size_t get_and_check_channels_idx(const Layout& layout, const PartialShap /// This is internal structure which is not shared to custom operations yet. class PreprocessingContext { public: - explicit PreprocessingContext(const Layout& layout) : m_layout(layout) {} + explicit PreprocessingContext(Layout layout) : m_layout(std::move(layout)) {} const Layout& layout() const { return m_layout; @@ -99,10 +100,19 @@ class PreprocessingContext { return network_shape()[network_width_idx].get_length(); } + const ColorFormat& color_format() const { + return m_color_format; + } + + ColorFormat& color_format() { + return m_color_format; + } + private: Layout m_layout; PartialShape m_network_shape; Layout m_network_layout; + ColorFormat m_color_format = ColorFormat::UNDEFINED; }; using InternalPreprocessOp = @@ -117,6 +127,7 @@ class PreProcessSteps::PreProcessStepsImpl { void add_convert_impl(const element::Type& type); void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width); void add_convert_layout_impl(const Layout& layout); + void add_convert_color_impl(const ColorFormat& dst_format); const std::list>& actions() const { return m_actions; diff --git a/ngraph/test/preprocess.cpp b/ngraph/test/preprocess.cpp index 94d6863eeda590..8d85d1f51154d6 100644 --- a/ngraph/test/preprocess.cpp +++ b/ngraph/test/preprocess.cpp @@ -107,6 +107,52 @@ TEST(pre_post_process, tensor_element_type_and_scale) { EXPECT_EQ(f->get_parameters().front()->get_layout(), Layout()); } +TEST(pre_post_process, convert_color_nv12_rgb_single) { + auto f = create_simple_function(element::f32, Shape{1, 2, 2, 3}); + f = PrePostProcessor() + .input( + InputInfo() + .tensor(InputTensorInfo() + .set_element_type(element::u8) + .set_color_format(ColorFormat::NV12_SINGLE_PLANE)) + .preprocess(PreProcessSteps().convert_color(ColorFormat::RGB).convert_element_type(element::f32))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 1); + auto result = std::make_shared(); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::u8); +} + +TEST(pre_post_process, convert_color_nv12_bgr_2_planes) { + auto f = create_simple_function(element::f32, Shape{1, 2, 2, 3}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"TestY", "TestUV"})) + .preprocess(PreProcessSteps().convert_color(ColorFormat::BGR))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 2); + EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), "input1/TestY"); + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), "input1/TestUV"); + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32); +} + +TEST(pre_post_process, convert_color_nv12_bgr_2_planes_u8_lvalue) { + auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3}); + auto input_tensor_info = InputTensorInfo(); + input_tensor_info.set_color_format(ColorFormat::NV12_TWO_PLANES); + auto steps = PreProcessSteps(); + steps.convert_color(ColorFormat::BGR); + f = PrePostProcessor() + .input(InputInfo().tensor(std::move(input_tensor_info)).preprocess(std::move(steps))) + .build(f); + + EXPECT_EQ(f->get_parameters().size(), 2); + EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::u8); + EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::u8); +} + TEST(pre_post_process, custom_preprocessing) { auto f = create_simple_function(element::i32, Shape{1, 3, 1, 1}); f = PrePostProcessor()