From 9218954e93fd7e1ca202a6685fc98762979dd3b9 Mon Sep 17 00:00:00 2001 From: Michael Nosov Date: Mon, 6 Sep 2021 20:11:45 +0300 Subject: [PATCH 1/4] Preprocessing API - base classes Includes API definition for trivial mean/scale operations (which don't require layout) Mean/scale with 'layout' support will be done under separate task together with Layout Current test code coverage: 100% --- .../core/pre_post_process/input_info.hpp | 73 +++++ .../pre_post_process/input_tensor_info.hpp | 63 +++++ .../pre_post_process/pre_post_process.hpp | 62 +++++ .../pre_post_process/preprocess_steps.hpp | 113 ++++++++ .../src/pre_post_process/pre_post_process.cpp | 251 ++++++++++++++++++ ngraph/test/CMakeLists.txt | 1 + ngraph/test/pre_post_process.cpp | 192 ++++++++++++++ 7 files changed, 755 insertions(+) create mode 100644 ngraph/core/include/openvino/core/pre_post_process/input_info.hpp create mode 100644 ngraph/core/include/openvino/core/pre_post_process/input_tensor_info.hpp create mode 100644 ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp create mode 100644 ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp create mode 100644 ngraph/core/src/pre_post_process/pre_post_process.cpp create mode 100644 ngraph/test/pre_post_process.cpp diff --git a/ngraph/core/include/openvino/core/pre_post_process/input_info.hpp b/ngraph/core/include/openvino/core/pre_post_process/input_info.hpp new file mode 100644 index 00000000000000..c808e955e90388 --- /dev/null +++ b/ngraph/core/include/openvino/core/pre_post_process/input_info.hpp @@ -0,0 +1,73 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/core_visibility.hpp" +#include "openvino/core/pre_post_process/input_tensor_info.hpp" +#include "openvino/core/pre_post_process/preprocess_steps.hpp" + +namespace ov { +namespace preprocess { + +class InputInfoImpl; + +/// \brief Class holding preprocessing information for one input +/// API has Builder-like style to allow chaining calls in client's code, like +/// \code{.cpp} +/// auto proc = PrePostProcessor().input(InputInfo().tensor(...).preprocess(...); +/// \endcode +class OPENVINO_API InputInfo final { + std::unique_ptr m_impl; + friend class PrePostProcessor; + +public: + /// \brief Empty constructor. Should be used only if network will have only one input + InputInfo(); + + /// \brief Information about info for particular input index of model + /// + /// \param input_index Index to address specified input parameter of model + InputInfo(size_t input_index); + + /// \brief Default move constructor + InputInfo(InputInfo&&) noexcept; + + /// \brief Default move assignment operator + InputInfo& operator=(InputInfo&&) noexcept; + + /// \brief Default destructor + ~InputInfo(); + + /// \brief Set input tensor information for input - Lvalue version + /// + /// \param builder Input tensor information. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + InputInfo& tensor(InputTensorInfo&& builder) &; + + /// \brief Set input tensor information for input - Rvalue version + /// + /// \param builder Input tensor information. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + InputInfo&& tensor(InputTensorInfo&& builder) &&; + + /// \brief Set preprocessing operations for input - Lvalue version + /// + /// \param builder Preprocessing operations. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + InputInfo& preprocess(PreProcessSteps&& builder) &; + + /// \brief Set preprocessing operations for input - Rvalue version + /// + /// \param builder Preprocessing operations. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + InputInfo&& preprocess(PreProcessSteps&& builder) &&; +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/include/openvino/core/pre_post_process/input_tensor_info.hpp b/ngraph/core/include/openvino/core/pre_post_process/input_tensor_info.hpp new file mode 100644 index 00000000000000..5308336e3db791 --- /dev/null +++ b/ngraph/core/include/openvino/core/pre_post_process/input_tensor_info.hpp @@ -0,0 +1,63 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/core_visibility.hpp" +#include "openvino/core/type/element_type.hpp" + +namespace ov { +namespace preprocess { + +class InputTensorInfoImpl; + +/// \brief Information about user's input tensor. By default, it will be initialized to same data (type/shape/etc) as +/// network's input parameter User application can override particular parameters (like 'element_type') according to +/// application's data and specify appropriate conversions in pre-processing steps +/// +/// \code{.cpp} +/// auto proc = +/// PrePostProcessor() +/// .input(InputInfo() +/// .tensor(InputTensorInfo() +/// .set_element_type(ov::element::u8)) +/// .preprocess() +/// ); +/// \endcode +class OPENVINO_API InputTensorInfo final { + std::unique_ptr m_impl; + friend class InputInfo; + +public: + /// \brief Default empty constructor + InputTensorInfo(); + + /// \brief Default move constructor + InputTensorInfo(InputTensorInfo&&) noexcept; + + /// \brief Default move assignment + InputTensorInfo& operator=(InputTensorInfo&&) noexcept; + + /// \brief Default destructor + ~InputTensorInfo(); + + /// \brief Set element type for user's input tensor + /// This version allows chaining for Lvalue objects + /// + /// \param type Element type for user's input tensor. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + InputTensorInfo& set_element_type(const ov::element::Type& type) &; + + /// \brief Set element type for user's input tensor + /// This version allows chaining for Rvalue objects + /// + /// \param builder Pre-processing data for input tensor of model. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + InputTensorInfo&& set_element_type(const ov::element::Type& type) &&; +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp b/ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp new file mode 100644 index 00000000000000..9ea806a5efe642 --- /dev/null +++ b/ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp @@ -0,0 +1,62 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/core_visibility.hpp" +#include "openvino/core/function.hpp" +#include "openvino/core/pre_post_process/input_info.hpp" + +namespace ov { +namespace preprocess { + +class PrePostProcessorImpl; + +/// \brief Main class for adding pre- and post- processing steps to existing ov::Function +/// API has Builder-like style to allow chaining calls in client's code, like +/// \code{.cpp} +/// auto proc = PrePostProcessor().input().input(); +/// \endcode +class OPENVINO_API PrePostProcessor final { + std::unique_ptr m_impl; + +public: + /// \brief Default constructor + PrePostProcessor(); + + /// \brief Default move constructor + PrePostProcessor(PrePostProcessor&&) noexcept; + + /// \brief Default move assignment operator + PrePostProcessor& operator=(PrePostProcessor&&) noexcept; + + /// \brief Default destructor + ~PrePostProcessor(); + + /// \brief Adds pre-processing information and steps to input of model. This method can be used only if ov::Function + /// passed on `build` has only one input + /// + /// \param builder Pre-processing data for input tensor of model. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PrePostProcessor& input(InputInfo&& builder) &; + + /// \brief Adds pre-processing information and steps to input of model. This method can be used only if ov::Function + /// passed on `build` has only one input This version allows chaining if object represents Rvalue + /// + /// \param builder Pre-processing data for input tensor of model. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PrePostProcessor&& input(InputInfo&& builder) &&; + + /// \brief Adds pre/post-processing operations to existing function + /// + /// \param function Existing function representing loaded model + /// + /// \return Function with added pre/post-processing operations + std::shared_ptr build(const std::shared_ptr& function); +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp b/ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp new file mode 100644 index 00000000000000..6fb35f5e056f57 --- /dev/null +++ b/ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp @@ -0,0 +1,113 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/core_visibility.hpp" +#include "openvino/core/type/element_type.hpp" + +namespace ov { +namespace preprocess { + +class PreProcessStepsImpl; + +/// \brief Preprocessing steps. Each step typically intends adding of some operation to input parameter +/// User application can specify sequence of preprocessing steps in a builder-like manner +/// \code{.cpp} +/// auto proc = PrePostProcessor() +/// .input(InputInfo() +/// .preprocess(PreProcessSteps() +/// .mean(0.2f) // Subtract 0.2 from each element +/// .scale(2.3f)) // then divide each element to 2.3 +/// ); +/// \endcode +class OPENVINO_API PreProcessSteps final { + std::unique_ptr m_impl; + friend class InputInfo; + +public: + /// \brief Default empty constructor + PreProcessSteps(); + + /// \brief Default move constructor + PreProcessSteps(PreProcessSteps&&) noexcept; + + /// \brief Default move assignment operator + PreProcessSteps& operator=(PreProcessSteps&&) noexcept; + + /// \brief Default destructor + ~PreProcessSteps(); + + /// \brief Add convert element type preprocess operation - Lvalue version + /// + /// \param type Desired type of input. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps& convert_element_type(const ov::element::Type& type) &; + + /// \brief Add convert element type preprocess operation - Rvalue version + /// + /// \param type Desired type of input. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps&& convert_element_type(const ov::element::Type& type) &&; + + /// \brief Add scale preprocess operation - Lvalue version + /// Divide each element of input by specified value + /// + /// \param value Scaling value. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps& scale(float value) &; + + /// \brief Add scale preprocess operation - Rvalue version + /// Divide each element of input by specified value + /// + /// \param value Scaling value. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps&& scale(float value) &&; + + /// \brief Add mean preprocess operation - Lvalue version + /// Subtract specified value from each element of input + /// + /// \param value Value to subtract from each element. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps& mean(float value) &; + + /// \brief Add mean preprocess operation - Rvalue version + /// Subtract specified value from each element of input + /// + /// \param value Value to subtract from each element. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps&& mean(float value) &&; + + /// \brief Signature for custom preprocessing operation + /// + /// \param node Input node for custom preprocessing operation + /// + /// \return New node after applying custom preprocessing operation + using CustomPreprocessOp = std::function(const std::shared_ptr& node)>; + + /// \brief Add custom preprocess operation - Lvalue version + /// Client application can specify callback function for custom action + /// + /// \param preprocess_cb Client's custom preprocess operation. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps& custom(const CustomPreprocessOp& preprocess_cb) &; + + /// \brief Add custom preprocess operation - Rvalue version + /// Client application can specify callback function for custom action + /// + /// \param preprocess_cb Client's custom preprocess operation. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner + PreProcessSteps&& custom(const CustomPreprocessOp& preprocess_cb) &&; +}; + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/core/src/pre_post_process/pre_post_process.cpp b/ngraph/core/src/pre_post_process/pre_post_process.cpp new file mode 100644 index 00000000000000..dc0e1aeae8b175 --- /dev/null +++ b/ngraph/core/src/pre_post_process/pre_post_process.cpp @@ -0,0 +1,251 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/core/pre_post_process/pre_post_process.hpp" + +#include "ngraph/opsets/opset.hpp" +#include "ngraph/opsets/opset7.hpp" + +namespace ov { +namespace preprocess { + +/// \brief InputTensorInfoImpl - internal data structure +struct InputTensorInfoImpl { + InputTensorInfoImpl() = default; + explicit InputTensorInfoImpl(const ov::element::Type& type) : m_type(type) {} + + ov::element::Type m_type = ov::element::dynamic; +}; + +/// \brief PreProcessStepsImpl - internal data structure +struct PreProcessStepsImpl { + void add_scale_impl(const std::vector& values) { + m_actions.emplace_back([values](const std::shared_ptr& node) { + ngraph::Shape shape; + if (values.size() == 1) { + shape = ngraph::Shape{1}; + } else { + // TODO: implement when Layout API is available + } + auto constant = ngraph::op::v0::Constant::create(ngraph::element::f32, shape, values); + constant->set_friendly_name(node->get_friendly_name() + "/scale/Divide_Factor"); + + auto new_op = std::make_shared(node, constant); + new_op->set_friendly_name(node->get_friendly_name() + "/scale/Divide"); + return new_op; + }); + } + + void add_mean_impl(const std::vector& values) { + m_actions.emplace_back([values](const std::shared_ptr& node) { + ngraph::Shape shape; + if (values.size() == 1) { + shape = ngraph::Shape{1}; + } else { + // TODO: implement when Layout API is available + } + auto constant = ngraph::op::v0::Constant::create(ngraph::element::f32, shape, values); + constant->set_friendly_name(node->get_friendly_name() + "/mean/Mean_Const"); + + auto new_op = std::make_shared(node, constant); + new_op->set_friendly_name(node->get_friendly_name() + "/mean/Subtract"); + return new_op; + }); + } + + void add_convert_impl(const ov::element::Type& type) { + m_actions.emplace_back([type](const std::shared_ptr& node) { + if (node->get_element_type().is_dynamic()) { + throw ngraph::ngraph_error("Can't insert 'convert_element_type' for dynamic source tensor type."); + } + auto convert = std::make_shared(node, type); + convert->set_friendly_name(node->get_friendly_name() + "/convert_element_type"); + return convert; + }); + } + std::vector m_actions; +}; + +/// \brief InputInfoImpl - internal data structure +struct InputInfoImpl { + InputInfoImpl() = default; + explicit InputInfoImpl(size_t idx) : m_has_index(true), m_index(idx) {} + + bool has_index() const { + return m_has_index; + } + + bool m_has_index = false; + size_t m_index = 0; + std::unique_ptr m_tensor_data; + std::unique_ptr m_preprocess; +}; + +//-------------- InputInfo ------------------ +InputInfo::InputInfo() : m_impl(std::unique_ptr(new InputInfoImpl)) {} +InputInfo::InputInfo(size_t input_index) : m_impl(std::unique_ptr(new InputInfoImpl(input_index))) {} +InputInfo::InputInfo(InputInfo&&) noexcept = default; +InputInfo& InputInfo::operator=(InputInfo&&) noexcept = default; +InputInfo::~InputInfo() = default; + +InputInfo& InputInfo::tensor(InputTensorInfo&& builder) & { + m_impl->m_tensor_data = std::move(builder.m_impl); + return *this; +} + +InputInfo&& InputInfo::tensor(InputTensorInfo&& builder) && { + m_impl->m_tensor_data = std::move(builder.m_impl); + return std::move(*this); +} + +InputInfo&& InputInfo::preprocess(PreProcessSteps&& builder) && { + m_impl->m_preprocess = std::move(builder.m_impl); + return std::move(*this); +} + +InputInfo& InputInfo::preprocess(PreProcessSteps&& builder) & { + m_impl->m_preprocess = std::move(builder.m_impl); + return *this; +} + +// ------------------------ PrePostProcessor -------------------- +struct PrePostProcessorImpl { +public: + std::vector> in_contexts; +}; + +PrePostProcessor::PrePostProcessor() : m_impl(std::unique_ptr(new PrePostProcessorImpl())) {} +PrePostProcessor::PrePostProcessor(PrePostProcessor&&) noexcept = default; +PrePostProcessor& PrePostProcessor::operator=(PrePostProcessor&&) noexcept = default; +PrePostProcessor::~PrePostProcessor() = default; + +PrePostProcessor& PrePostProcessor::input(InputInfo&& builder) & { + m_impl->in_contexts.push_back(std::move(builder.m_impl)); + return *this; +} + +PrePostProcessor&& PrePostProcessor::input(InputInfo&& builder) && { + m_impl->in_contexts.push_back(std::move(builder.m_impl)); + return std::move(*this); +} + +std::shared_ptr PrePostProcessor::build(const std::shared_ptr& function) { + for (const auto& input : m_impl->in_contexts) { + std::shared_ptr param; + OV_CHECK(input, "Internal error: Invalid preprocessing input, please report a problem"); + if (input->has_index()) { + param = function->get_parameters().at(input->m_index); + } else { + // Default case + OV_CHECK(function->get_parameters().size() == 1, + std::string("Preprocessing info expects having 1 input, however function has ") + + std::to_string(function->get_parameters().size()) + + " inputs. Please use ov::preprocess::InputInfo constructor specifying " + "particular input instead of default one"); + param = function->get_parameters().front(); + } + auto consumers = param->output(0).get_target_inputs(); + if (!input->m_tensor_data) { + input->m_tensor_data = + std::unique_ptr(new InputTensorInfoImpl(param->get_element_type())); + } + auto new_param_shape = param->get_partial_shape(); + auto new_param = std::make_shared(input->m_tensor_data->m_type, new_param_shape); + // Old param will be removed, so friendly name can be reused + new_param->set_friendly_name(param->get_friendly_name()); + std::shared_ptr node = new_param; + + // 2. Apply preprocessing + for (const auto& action : input->m_preprocess->m_actions) { + node = action(node); + } + + // 3. Apply 'network()' data + // Check final type + if (node->get_element_type() != param->get_element_type()) { + throw ngraph::ngraph_error( + std::string("Element type after preprocessing {") + node->get_element_type().c_type_string() + + std::string("} doesn't match with network element type {") + param->get_element_type().c_type_string() + + "}. Please add 'convert_element_type' explicitly"); + } + + // Replace parameter + for (auto consumer : consumers) { + consumer.replace_source_output(node); + } + if (input->has_index()) { + function->replace_parameter(input->m_index, new_param); + } else { + function->replace_parameter(0, new_param); + } + } + function->validate_nodes_and_infer_types(); + return function; +} + +// --------------------- InputTensorInfo ------------------ +InputTensorInfo::InputTensorInfo() : m_impl(std::unique_ptr(new InputTensorInfoImpl())) {} +InputTensorInfo::InputTensorInfo(InputTensorInfo&&) noexcept = default; +InputTensorInfo& InputTensorInfo::operator=(InputTensorInfo&&) noexcept = default; +InputTensorInfo::~InputTensorInfo() = default; + +InputTensorInfo& InputTensorInfo::set_element_type(const ov::element::Type& type) & { + m_impl->m_type = type; + return *this; +} + +InputTensorInfo&& InputTensorInfo::set_element_type(const ov::element::Type& type) && { + m_impl->m_type = type; + return std::move(*this); +} + +// --------------------- PreProcessSteps ------------------ + +PreProcessSteps::PreProcessSteps() : m_impl(std::unique_ptr(new PreProcessStepsImpl())) {} +PreProcessSteps::PreProcessSteps(PreProcessSteps&&) noexcept = default; +PreProcessSteps& PreProcessSteps::operator=(PreProcessSteps&&) noexcept = default; +PreProcessSteps::~PreProcessSteps() = default; + +PreProcessSteps& PreProcessSteps::scale(float value) & { + m_impl->add_scale_impl(std::vector{value}); + return *this; +} + +PreProcessSteps&& PreProcessSteps::scale(float value) && { + m_impl->add_scale_impl(std::vector{value}); + return std::move(*this); +} + +PreProcessSteps& PreProcessSteps::mean(float value) & { + m_impl->add_mean_impl(std::vector{value}); + return *this; +} + +PreProcessSteps&& PreProcessSteps::mean(float value) && { + m_impl->add_mean_impl(std::vector{value}); + return std::move(*this); +} + +PreProcessSteps& PreProcessSteps::convert_element_type(const ov::element::Type& type) & { + m_impl->add_convert_impl(type); + return *this; +} + +PreProcessSteps&& PreProcessSteps::convert_element_type(const ov::element::Type& type) && { + m_impl->add_convert_impl(type); + return std::move(*this); +} + +PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & { + m_impl->m_actions.emplace_back(preprocess_cb); + return *this; +} + +PreProcessSteps&& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) && { + m_impl->m_actions.emplace_back(preprocess_cb); + return std::move(*this); +} + +} // namespace preprocess +} // namespace ov diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 1ac9ee9fcbad84..f85e636b662d09 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -76,6 +76,7 @@ set(SRC pass_config.cpp pass_manager.cpp pattern.cpp + pre_post_process.cpp provenance.cpp replace_node.cpp reshape_opt_kernel.cpp diff --git a/ngraph/test/pre_post_process.cpp b/ngraph/test/pre_post_process.cpp new file mode 100644 index 00000000000000..5d292e3d58ced2 --- /dev/null +++ b/ngraph/test/pre_post_process.cpp @@ -0,0 +1,192 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/core/pre_post_process/pre_post_process.hpp" + +#include + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "ngraph/ops.hpp" +#include "util/all_close.hpp" +#include "util/all_close_f.hpp" +#include "util/test_tools.hpp" + +using namespace ov; +using namespace ov::preprocess; +using namespace ngraph::test; + +static std::shared_ptr create_simple_function(element::Type type, const PartialShape& shape) { + auto data1 = std::make_shared(type, shape); + data1->set_friendly_name("input1"); + auto res = std::make_shared(data1); + res->set_friendly_name("Result"); + return std::make_shared(ngraph::ResultVector{res}, ngraph::ParameterVector{data1}); +} + +static std::shared_ptr create_2inputs(element::Type type, const PartialShape& shape) { + auto data1 = std::make_shared(type, shape); + data1->set_friendly_name("input1"); + auto data2 = std::make_shared(type, shape); + data2->set_friendly_name("input2"); + auto res1 = std::make_shared(data1); + res1->set_friendly_name("Result"); + auto res2 = std::make_shared(data2); + res2->set_friendly_name("Result"); + return std::make_shared(ngraph::ResultVector{res1, res2}, ngraph::ParameterVector{data1, data2}); +} + +TEST(pre_post_process, simple_mean_scale) { + auto f = create_simple_function(ngraph::element::f32, ngraph::Shape{1, 3, 2, 2}); + f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().mean(1.f).scale(2.f))).build(f); + + auto result = std::make_shared(); + f->evaluate({result}, + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, + {1., 3., 5., 7., 9., 11., 13., 15., 17., 19., 21., 23.})}); + auto result_val = read_vector(result); + EXPECT_TRUE(all_close_f(std::vector{0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.}, result_val)); +} + +TEST(pre_post_process, scale_then_mean) { + auto f = create_simple_function(ngraph::element::f32, ngraph::Shape{1, 3, 2, 2}); + f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().scale(2.0f).mean(1.0f))).build(f); + + auto result = std::make_shared(); + f->evaluate({result}, + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, + {2., 4., 6., 8., 10., 12., 14., 16., 18., 20., 100., 200.})}); + auto result_val = read_vector(result); + EXPECT_TRUE(all_close_f(std::vector{0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 49., 99.}, result_val)); +} + +TEST(pre_post_process, convert_element_type_and_scale) { + auto f = create_simple_function(element::i8, ngraph::Shape{1, 3, 2, 2}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_element_type(element::i16)) + .preprocess(PreProcessSteps() + .convert_element_type(element::f32) + .scale(2.f) + .convert_element_type(element::i8))) + .build(f); + + auto result = std::make_shared(); + f->evaluate({result}, + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, + {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 1000000, 200})}); + auto result_val = read_vector(result); + EXPECT_TRUE(all_close(std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, (int8_t)500000, 100}, result_val)); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i16); + + ASSERT_EQ(f->get_output_element_type(0), element::i8); +} + +TEST(pre_post_process, convert_element_type_from_unknown) { + auto f = create_simple_function(element::i32, ngraph::Shape{1, 3, 224, 224}); + ASSERT_ANY_THROW( + f = PrePostProcessor() + .input(InputInfo().preprocess( + PreProcessSteps().convert_element_type(element::dynamic).convert_element_type(element::i32))) + .build(f);); +} + +TEST(pre_post_process, convert_element_type_no_match) { + auto f = create_simple_function(element::i32, ngraph::Shape{1, 3, 224, 224}); + ASSERT_ANY_THROW(f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_element_type(element::i32)) + .preprocess(PreProcessSteps().convert_element_type(element::f32).scale(2.0f))) + .build(f);); +} + +TEST(pre_post_process, tensor_element_type_and_scale) { + auto f = create_simple_function(element::i8, ngraph::Shape{1, 3, 1, 1}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_element_type(element::f32)) + .preprocess(PreProcessSteps().scale(2.0f).convert_element_type(element::i8))) + .build(f); + + auto result = std::make_shared(); + f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {2., 4., 6.})}); + auto result_val = read_vector(result); + EXPECT_TRUE(all_close(std::vector{1, 2, 3}, result_val)); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32); + + ASSERT_EQ(f->get_output_element_type(0), element::i8); +} + +TEST(pre_post_process, custom_preprocessing) { + auto f = create_simple_function(element::i32, ngraph::Shape{1, 3, 1, 1}); + f = PrePostProcessor() + .input(InputInfo().preprocess(PreProcessSteps().custom([](const std::shared_ptr& node) { + auto abs = std::make_shared(node); + abs->set_friendly_name(node->get_friendly_name() + "/abs"); + return abs; + }))) + .build(f); + + auto result = std::make_shared(); + f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {0, 4, -6})}); + auto result_val = read_vector(result); + EXPECT_TRUE(all_close(std::vector{0, 4, 6}, result_val)); +} + +TEST(pre_post_process, test_lvalue) { + auto f = create_simple_function(element::i8, ngraph::Shape{1, 3, 1, 1}); + auto p = PrePostProcessor(); + auto p1 = std::move(p); + p = std::move(p1); + auto inputInfo = InputInfo(); + auto inputInfo2 = std::move(inputInfo); + inputInfo = std::move(inputInfo2); + { + auto inputTensorInfo = InputTensorInfo(); + auto inputTensorInfo2 = std::move(inputTensorInfo); + inputTensorInfo = std::move(inputTensorInfo2); + auto& same = inputTensorInfo.set_element_type(element::f32); + inputInfo.tensor(std::move(same)); + } + { + auto preprocessSteps = PreProcessSteps(); + auto preprocessSteps2 = std::move(preprocessSteps); + preprocessSteps = std::move(preprocessSteps2); + preprocessSteps.mean(1.f); + preprocessSteps.scale(2.f); + preprocessSteps.custom([](const std::shared_ptr& node) { + auto abs = std::make_shared(node); + abs->set_friendly_name(node->get_friendly_name() + "/abs"); + return abs; + }); + auto& same = preprocessSteps.convert_element_type(element::i8); + inputInfo.preprocess(std::move(same)); + } + p.input(std::move(inputInfo)); + f = p.build(f); + + auto result = std::make_shared(); + f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {-3., 5., 7.})}); + auto result_val = read_vector(result); + EXPECT_TRUE(all_close(std::vector{2, 2, 3}, result_val)); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32); + + ASSERT_EQ(f->get_output_element_type(0), element::i8); +} + +TEST(pre_post_process, test_2_inputs_basic) { + auto f = create_2inputs(element::f32, ngraph::Shape{1, 3, 1, 1}); + { f = PrePostProcessor().input(InputInfo(1).preprocess(PreProcessSteps().mean(1.f).scale(2.0f))).build(f); } + auto result1 = std::make_shared(); + auto result2 = std::make_shared(); + auto input1 = make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {3., 5., 7.}); + auto input2 = make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {3., 5., 7.}); + f->evaluate({result1, result2}, {input1, input2}); + + auto result1_val = read_vector(result1); + EXPECT_TRUE(all_close_f(std::vector{3, 5, 7}, result1_val)); + + auto result2_val = read_vector(result2); + EXPECT_TRUE(all_close_f(std::vector{1, 2, 3}, result2_val)); +} From d04b6573dd04066067bf171df34442df9c1ef700 Mon Sep 17 00:00:00 2001 From: Michael Nosov Date: Tue, 7 Sep 2021 13:52:22 +0300 Subject: [PATCH 2/4] Rename 'pre_post_process' folder to 'preprocess', aligned with namespace Fix 32-bit tests --- .../{pre_post_process => preprocess}/input_info.hpp | 4 ++-- .../input_tensor_info.hpp | 0 .../pre_post_process.hpp | 13 ++++++++++--- .../preprocess_steps.hpp | 4 +++- .../pre_post_process.cpp | 5 ++--- ngraph/test/CMakeLists.txt | 2 +- .../test/{pre_post_process.cpp => preprocess.cpp} | 8 ++++---- 7 files changed, 22 insertions(+), 14 deletions(-) rename ngraph/core/include/openvino/core/{pre_post_process => preprocess}/input_info.hpp (94%) rename ngraph/core/include/openvino/core/{pre_post_process => preprocess}/input_tensor_info.hpp (100%) rename ngraph/core/include/openvino/core/{pre_post_process => preprocess}/pre_post_process.hpp (78%) rename ngraph/core/include/openvino/core/{pre_post_process => preprocess}/preprocess_steps.hpp (94%) rename ngraph/core/src/{pre_post_process => preprocess}/pre_post_process.cpp (98%) rename ngraph/test/{pre_post_process.cpp => preprocess.cpp} (97%) diff --git a/ngraph/core/include/openvino/core/pre_post_process/input_info.hpp b/ngraph/core/include/openvino/core/preprocess/input_info.hpp similarity index 94% rename from ngraph/core/include/openvino/core/pre_post_process/input_info.hpp rename to ngraph/core/include/openvino/core/preprocess/input_info.hpp index c808e955e90388..056ff37b0fed9f 100644 --- a/ngraph/core/include/openvino/core/pre_post_process/input_info.hpp +++ b/ngraph/core/include/openvino/core/preprocess/input_info.hpp @@ -5,8 +5,8 @@ #pragma once #include "openvino/core/core_visibility.hpp" -#include "openvino/core/pre_post_process/input_tensor_info.hpp" -#include "openvino/core/pre_post_process/preprocess_steps.hpp" +#include "openvino/core/preprocess/input_tensor_info.hpp" +#include "openvino/core/preprocess/preprocess_steps.hpp" namespace ov { namespace preprocess { diff --git a/ngraph/core/include/openvino/core/pre_post_process/input_tensor_info.hpp b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp similarity index 100% rename from ngraph/core/include/openvino/core/pre_post_process/input_tensor_info.hpp rename to ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp diff --git a/ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp b/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp similarity index 78% rename from ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp rename to ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp index 9ea806a5efe642..800a053eb500c1 100644 --- a/ngraph/core/include/openvino/core/pre_post_process/pre_post_process.hpp +++ b/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp @@ -6,7 +6,7 @@ #include "openvino/core/core_visibility.hpp" #include "openvino/core/function.hpp" -#include "openvino/core/pre_post_process/input_info.hpp" +#include "openvino/core/preprocess/input_info.hpp" namespace ov { namespace preprocess { @@ -18,6 +18,13 @@ class PrePostProcessorImpl; /// \code{.cpp} /// auto proc = PrePostProcessor().input().input(); /// \endcode +/// +/// This is a helper class for writing easy pre- and post- processing operations on ov::Function object assuming that +/// any preprocess operation takes one input and produces one output. +/// +/// For advanced preprocessing scenarios, like combining several functions with multiple inputs/outputs into one, +/// client's code can use transformation passes over ov::Function +/// class OPENVINO_API PrePostProcessor final { std::unique_ptr m_impl; @@ -42,8 +49,8 @@ class OPENVINO_API PrePostProcessor final { /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner PrePostProcessor& input(InputInfo&& builder) &; - /// \brief Adds pre-processing information and steps to input of model. This method can be used only if ov::Function - /// passed on `build` has only one input This version allows chaining if object represents Rvalue + /// \brief Adds pre-processing information and steps to input of model - Rvalue version. This method can be used + /// only if ov::Function passed on `build` has only one input. /// /// \param builder Pre-processing data for input tensor of model. /// diff --git a/ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp similarity index 94% rename from ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp rename to ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp index 6fb35f5e056f57..2741aac88e95ac 100644 --- a/ngraph/core/include/openvino/core/pre_post_process/preprocess_steps.hpp +++ b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp @@ -85,7 +85,9 @@ class OPENVINO_API PreProcessSteps final { /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner PreProcessSteps&& mean(float value) &&; - /// \brief Signature for custom preprocessing operation + /// \brief Signature for custom preprocessing operation. Custom preprocessing operation takes one input node and + /// produces one output node. For more advanced cases, client's code can use transformation passes over ov::Function + /// directly /// /// \param node Input node for custom preprocessing operation /// diff --git a/ngraph/core/src/pre_post_process/pre_post_process.cpp b/ngraph/core/src/preprocess/pre_post_process.cpp similarity index 98% rename from ngraph/core/src/pre_post_process/pre_post_process.cpp rename to ngraph/core/src/preprocess/pre_post_process.cpp index dc0e1aeae8b175..125bf2490828d2 100644 --- a/ngraph/core/src/pre_post_process/pre_post_process.cpp +++ b/ngraph/core/src/preprocess/pre_post_process.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "openvino/core/pre_post_process/pre_post_process.hpp" +#include "openvino/core/preprocess/pre_post_process.hpp" #include "ngraph/opsets/opset.hpp" #include "ngraph/opsets/opset7.hpp" @@ -64,7 +64,7 @@ struct PreProcessStepsImpl { return convert; }); } - std::vector m_actions; + std::list m_actions; }; /// \brief InputInfoImpl - internal data structure @@ -161,7 +161,6 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrget_element_type() != param->get_element_type()) { throw ngraph::ngraph_error( diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index f85e636b662d09..9fb8a70c950960 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -76,7 +76,7 @@ set(SRC pass_config.cpp pass_manager.cpp pattern.cpp - pre_post_process.cpp + preprocess.cpp provenance.cpp replace_node.cpp reshape_opt_kernel.cpp diff --git a/ngraph/test/pre_post_process.cpp b/ngraph/test/preprocess.cpp similarity index 97% rename from ngraph/test/pre_post_process.cpp rename to ngraph/test/preprocess.cpp index 5d292e3d58ced2..553864a91d97ac 100644 --- a/ngraph/test/pre_post_process.cpp +++ b/ngraph/test/preprocess.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "openvino/core/pre_post_process/pre_post_process.hpp" +#include "openvino/core/preprocess/pre_post_process.hpp" #include @@ -74,10 +74,10 @@ TEST(pre_post_process, convert_element_type_and_scale) { auto result = std::make_shared(); f->evaluate({result}, - {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, - {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 1000000, 200})}); + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, + {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 10000, 200})}); auto result_val = read_vector(result); - EXPECT_TRUE(all_close(std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, (int8_t)500000, 100}, result_val)); + EXPECT_TRUE(all_close(std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, (int8_t)5000, 100}, result_val)); EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i16); ASSERT_EQ(f->get_output_element_type(0), element::i8); From 79695428bb0cc31d1a7bdd5ab947a89e88caa124 Mon Sep 17 00:00:00 2001 From: Michael Nosov Date: Tue, 7 Sep 2021 14:29:59 +0300 Subject: [PATCH 3/4] More cleanup --- .../core/preprocess/pre_post_process.hpp | 4 +- .../core/preprocess/preprocess_steps.hpp | 3 + .../core/src/preprocess/pre_post_process.cpp | 36 ++++++------ ngraph/test/preprocess.cpp | 57 +++++++++---------- 4 files changed, 51 insertions(+), 49 deletions(-) diff --git a/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp b/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp index 800a053eb500c1..3e17d015073157 100644 --- a/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp +++ b/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp @@ -5,10 +5,12 @@ #pragma once #include "openvino/core/core_visibility.hpp" -#include "openvino/core/function.hpp" #include "openvino/core/preprocess/input_info.hpp" namespace ov { + +class Function; + namespace preprocess { class PrePostProcessorImpl; diff --git a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp index 2741aac88e95ac..667b9bb8881076 100644 --- a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp +++ b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp @@ -8,6 +8,9 @@ #include "openvino/core/type/element_type.hpp" namespace ov { + +class Node; + namespace preprocess { class PreProcessStepsImpl; diff --git a/ngraph/core/src/preprocess/pre_post_process.cpp b/ngraph/core/src/preprocess/pre_post_process.cpp index 125bf2490828d2..d65245075684e6 100644 --- a/ngraph/core/src/preprocess/pre_post_process.cpp +++ b/ngraph/core/src/preprocess/pre_post_process.cpp @@ -4,8 +4,8 @@ #include "openvino/core/preprocess/pre_post_process.hpp" -#include "ngraph/opsets/opset.hpp" -#include "ngraph/opsets/opset7.hpp" +#include "ngraph/opsets/opset1.hpp" +#include "openvino/core/function.hpp" namespace ov { namespace preprocess { @@ -13,9 +13,9 @@ namespace preprocess { /// \brief InputTensorInfoImpl - internal data structure struct InputTensorInfoImpl { InputTensorInfoImpl() = default; - explicit InputTensorInfoImpl(const ov::element::Type& type) : m_type(type) {} + explicit InputTensorInfoImpl(const element::Type& type) : m_type(type) {} - ov::element::Type m_type = ov::element::dynamic; + element::Type m_type = element::dynamic; }; /// \brief PreProcessStepsImpl - internal data structure @@ -28,10 +28,10 @@ struct PreProcessStepsImpl { } else { // TODO: implement when Layout API is available } - auto constant = ngraph::op::v0::Constant::create(ngraph::element::f32, shape, values); + auto constant = op::v0::Constant::create(element::f32, shape, values); constant->set_friendly_name(node->get_friendly_name() + "/scale/Divide_Factor"); - auto new_op = std::make_shared(node, constant); + auto new_op = std::make_shared(node, constant); new_op->set_friendly_name(node->get_friendly_name() + "/scale/Divide"); return new_op; }); @@ -45,21 +45,21 @@ struct PreProcessStepsImpl { } else { // TODO: implement when Layout API is available } - auto constant = ngraph::op::v0::Constant::create(ngraph::element::f32, shape, values); + auto constant = op::v0::Constant::create(element::f32, shape, values); constant->set_friendly_name(node->get_friendly_name() + "/mean/Mean_Const"); - auto new_op = std::make_shared(node, constant); + auto new_op = std::make_shared(node, constant); new_op->set_friendly_name(node->get_friendly_name() + "/mean/Subtract"); return new_op; }); } - void add_convert_impl(const ov::element::Type& type) { + void add_convert_impl(const element::Type& type) { m_actions.emplace_back([type](const std::shared_ptr& node) { if (node->get_element_type().is_dynamic()) { throw ngraph::ngraph_error("Can't insert 'convert_element_type' for dynamic source tensor type."); } - auto convert = std::make_shared(node, type); + auto convert = std::make_shared(node, type); convert->set_friendly_name(node->get_friendly_name() + "/convert_element_type"); return convert; }); @@ -112,7 +112,7 @@ InputInfo& InputInfo::preprocess(PreProcessSteps&& builder) & { // ------------------------ PrePostProcessor -------------------- struct PrePostProcessorImpl { public: - std::vector> in_contexts; + std::list> in_contexts; }; PrePostProcessor::PrePostProcessor() : m_impl(std::unique_ptr(new PrePostProcessorImpl())) {} @@ -132,7 +132,7 @@ PrePostProcessor&& PrePostProcessor::input(InputInfo&& builder) && { std::shared_ptr PrePostProcessor::build(const std::shared_ptr& function) { for (const auto& input : m_impl->in_contexts) { - std::shared_ptr param; + std::shared_ptr param; OV_CHECK(input, "Internal error: Invalid preprocessing input, please report a problem"); if (input->has_index()) { param = function->get_parameters().at(input->m_index); @@ -151,10 +151,10 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptr(new InputTensorInfoImpl(param->get_element_type())); } auto new_param_shape = param->get_partial_shape(); - auto new_param = std::make_shared(input->m_tensor_data->m_type, new_param_shape); + auto new_param = std::make_shared(input->m_tensor_data->m_type, new_param_shape); // Old param will be removed, so friendly name can be reused new_param->set_friendly_name(param->get_friendly_name()); - std::shared_ptr node = new_param; + std::shared_ptr node = new_param; // 2. Apply preprocessing for (const auto& action : input->m_preprocess->m_actions) { @@ -189,12 +189,12 @@ InputTensorInfo::InputTensorInfo(InputTensorInfo&&) noexcept = default; InputTensorInfo& InputTensorInfo::operator=(InputTensorInfo&&) noexcept = default; InputTensorInfo::~InputTensorInfo() = default; -InputTensorInfo& InputTensorInfo::set_element_type(const ov::element::Type& type) & { +InputTensorInfo& InputTensorInfo::set_element_type(const element::Type& type) & { m_impl->m_type = type; return *this; } -InputTensorInfo&& InputTensorInfo::set_element_type(const ov::element::Type& type) && { +InputTensorInfo&& InputTensorInfo::set_element_type(const element::Type& type) && { m_impl->m_type = type; return std::move(*this); } @@ -226,12 +226,12 @@ PreProcessSteps&& PreProcessSteps::mean(float value) && { return std::move(*this); } -PreProcessSteps& PreProcessSteps::convert_element_type(const ov::element::Type& type) & { +PreProcessSteps& PreProcessSteps::convert_element_type(const element::Type& type) & { m_impl->add_convert_impl(type); return *this; } -PreProcessSteps&& PreProcessSteps::convert_element_type(const ov::element::Type& type) && { +PreProcessSteps&& PreProcessSteps::convert_element_type(const element::Type& type) && { m_impl->add_convert_impl(type); return std::move(*this); } diff --git a/ngraph/test/preprocess.cpp b/ngraph/test/preprocess.cpp index 553864a91d97ac..874c473dbf0b30 100644 --- a/ngraph/test/preprocess.cpp +++ b/ngraph/test/preprocess.cpp @@ -2,13 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "openvino/core/preprocess/pre_post_process.hpp" - -#include - #include "gtest/gtest.h" #include "ngraph/ngraph.hpp" #include "ngraph/ops.hpp" +#include "openvino/core/preprocess/pre_post_process.hpp" #include "util/all_close.hpp" #include "util/all_close_f.hpp" #include "util/test_tools.hpp" @@ -18,47 +15,47 @@ using namespace ov::preprocess; using namespace ngraph::test; static std::shared_ptr create_simple_function(element::Type type, const PartialShape& shape) { - auto data1 = std::make_shared(type, shape); + auto data1 = std::make_shared(type, shape); data1->set_friendly_name("input1"); - auto res = std::make_shared(data1); + auto res = std::make_shared(data1); res->set_friendly_name("Result"); - return std::make_shared(ngraph::ResultVector{res}, ngraph::ParameterVector{data1}); + return std::make_shared(ResultVector{res}, ParameterVector{data1}); } static std::shared_ptr create_2inputs(element::Type type, const PartialShape& shape) { - auto data1 = std::make_shared(type, shape); + auto data1 = std::make_shared(type, shape); data1->set_friendly_name("input1"); - auto data2 = std::make_shared(type, shape); + auto data2 = std::make_shared(type, shape); data2->set_friendly_name("input2"); - auto res1 = std::make_shared(data1); + auto res1 = std::make_shared(data1); res1->set_friendly_name("Result"); - auto res2 = std::make_shared(data2); + auto res2 = std::make_shared(data2); res2->set_friendly_name("Result"); - return std::make_shared(ngraph::ResultVector{res1, res2}, ngraph::ParameterVector{data1, data2}); + return std::make_shared(ResultVector{res1, res2}, ParameterVector{data1, data2}); } TEST(pre_post_process, simple_mean_scale) { - auto f = create_simple_function(ngraph::element::f32, ngraph::Shape{1, 3, 2, 2}); + auto f = create_simple_function(element::f32, ngraph::Shape{1, 3, 2, 2}); f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().mean(1.f).scale(2.f))).build(f); auto result = std::make_shared(); f->evaluate({result}, - {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, - {1., 3., 5., 7., 9., 11., 13., 15., 17., 19., 21., 23.})}); + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, + {1., 3., 5., 7., 9., 11., 13., 15., 17., 19., 21., 23.})}); auto result_val = read_vector(result); EXPECT_TRUE(all_close_f(std::vector{0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.}, result_val)); } TEST(pre_post_process, scale_then_mean) { - auto f = create_simple_function(ngraph::element::f32, ngraph::Shape{1, 3, 2, 2}); - f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().scale(2.0f).mean(1.0f))).build(f); + auto f = create_simple_function(element::f32, ngraph::Shape{1, 3, 2, 2}); + f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().scale(2.0f).mean(2.0f))).build(f); auto result = std::make_shared(); f->evaluate({result}, - {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, - {2., 4., 6., 8., 10., 12., 14., 16., 18., 20., 100., 200.})}); + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, + {2., 4., 6., 8., 10., 12., 14., 16., 18., 20., 100., 200.})}); auto result_val = read_vector(result); - EXPECT_TRUE(all_close_f(std::vector{0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 49., 99.}, result_val)); + EXPECT_TRUE(all_close_f(std::vector{-1., 0, 1., 2., 3., 4., 5., 6., 7., 8., 48., 98.}, result_val)); } TEST(pre_post_process, convert_element_type_and_scale) { @@ -73,9 +70,9 @@ TEST(pre_post_process, convert_element_type_and_scale) { .build(f); auto result = std::make_shared(); - f->evaluate({result}, - {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, - {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 10000, 200})}); + f->evaluate( + {result}, + {make_host_tensor(ngraph::Shape{1, 3, 2, 2}, {2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 10000, 200})}); auto result_val = read_vector(result); EXPECT_TRUE(all_close(std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, (int8_t)5000, 100}, result_val)); EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i16); @@ -110,7 +107,7 @@ TEST(pre_post_process, tensor_element_type_and_scale) { .build(f); auto result = std::make_shared(); - f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {2., 4., 6.})}); + f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {2., 4., 6.})}); auto result_val = read_vector(result); EXPECT_TRUE(all_close(std::vector{1, 2, 3}, result_val)); EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32); @@ -122,14 +119,14 @@ TEST(pre_post_process, custom_preprocessing) { auto f = create_simple_function(element::i32, ngraph::Shape{1, 3, 1, 1}); f = PrePostProcessor() .input(InputInfo().preprocess(PreProcessSteps().custom([](const std::shared_ptr& node) { - auto abs = std::make_shared(node); + auto abs = std::make_shared(node); abs->set_friendly_name(node->get_friendly_name() + "/abs"); return abs; }))) .build(f); auto result = std::make_shared(); - f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {0, 4, -6})}); + f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {0, 4, -6})}); auto result_val = read_vector(result); EXPECT_TRUE(all_close(std::vector{0, 4, 6}, result_val)); } @@ -156,7 +153,7 @@ TEST(pre_post_process, test_lvalue) { preprocessSteps.mean(1.f); preprocessSteps.scale(2.f); preprocessSteps.custom([](const std::shared_ptr& node) { - auto abs = std::make_shared(node); + auto abs = std::make_shared(node); abs->set_friendly_name(node->get_friendly_name() + "/abs"); return abs; }); @@ -167,7 +164,7 @@ TEST(pre_post_process, test_lvalue) { f = p.build(f); auto result = std::make_shared(); - f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {-3., 5., 7.})}); + f->evaluate({result}, {make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {-3., 5., 7.})}); auto result_val = read_vector(result); EXPECT_TRUE(all_close(std::vector{2, 2, 3}, result_val)); EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32); @@ -180,8 +177,8 @@ TEST(pre_post_process, test_2_inputs_basic) { { f = PrePostProcessor().input(InputInfo(1).preprocess(PreProcessSteps().mean(1.f).scale(2.0f))).build(f); } auto result1 = std::make_shared(); auto result2 = std::make_shared(); - auto input1 = make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {3., 5., 7.}); - auto input2 = make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {3., 5., 7.}); + auto input1 = make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {3., 5., 7.}); + auto input2 = make_host_tensor(ngraph::Shape{1, 3, 1, 1}, {3., 5., 7.}); f->evaluate({result1, result2}, {input1, input2}); auto result1_val = read_vector(result1); From 351bca90ccd165d2ef81348b707316bceddebc90 Mon Sep 17 00:00:00 2001 From: Michael Nosov Date: Tue, 7 Sep 2021 23:18:08 +0300 Subject: [PATCH 4/4] Review comments --- .../openvino/core/preprocess/input_info.hpp | 3 +- .../core/preprocess/input_tensor_info.hpp | 3 +- .../core/preprocess/pre_post_process.hpp | 3 +- .../core/preprocess/preprocess_steps.hpp | 3 +- .../core/src/preprocess/pre_post_process.cpp | 116 ++++++++++-------- ngraph/test/preprocess.cpp | 20 ++- 6 files changed, 88 insertions(+), 60 deletions(-) diff --git a/ngraph/core/include/openvino/core/preprocess/input_info.hpp b/ngraph/core/include/openvino/core/preprocess/input_info.hpp index 056ff37b0fed9f..d586a44ca6657e 100644 --- a/ngraph/core/include/openvino/core/preprocess/input_info.hpp +++ b/ngraph/core/include/openvino/core/preprocess/input_info.hpp @@ -11,14 +11,13 @@ namespace ov { namespace preprocess { -class InputInfoImpl; - /// \brief Class holding preprocessing information for one input /// API has Builder-like style to allow chaining calls in client's code, like /// \code{.cpp} /// auto proc = PrePostProcessor().input(InputInfo().tensor(...).preprocess(...); /// \endcode class OPENVINO_API InputInfo final { + class InputInfoImpl; std::unique_ptr m_impl; friend class PrePostProcessor; diff --git a/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp index 5308336e3db791..25f58e7c19266f 100644 --- a/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp +++ b/ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp @@ -10,8 +10,6 @@ namespace ov { namespace preprocess { -class InputTensorInfoImpl; - /// \brief Information about user's input tensor. By default, it will be initialized to same data (type/shape/etc) as /// network's input parameter User application can override particular parameters (like 'element_type') according to /// application's data and specify appropriate conversions in pre-processing steps @@ -26,6 +24,7 @@ class InputTensorInfoImpl; /// ); /// \endcode class OPENVINO_API InputTensorInfo final { + class InputTensorInfoImpl; std::unique_ptr m_impl; friend class InputInfo; diff --git a/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp b/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp index 3e17d015073157..300fd1ab557295 100644 --- a/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp +++ b/ngraph/core/include/openvino/core/preprocess/pre_post_process.hpp @@ -13,8 +13,6 @@ class Function; namespace preprocess { -class PrePostProcessorImpl; - /// \brief Main class for adding pre- and post- processing steps to existing ov::Function /// API has Builder-like style to allow chaining calls in client's code, like /// \code{.cpp} @@ -28,6 +26,7 @@ class PrePostProcessorImpl; /// client's code can use transformation passes over ov::Function /// class OPENVINO_API PrePostProcessor final { + class PrePostProcessorImpl; std::unique_ptr m_impl; public: diff --git a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp index 667b9bb8881076..097ca507329b2f 100644 --- a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp +++ b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp @@ -13,8 +13,6 @@ class Node; namespace preprocess { -class PreProcessStepsImpl; - /// \brief Preprocessing steps. Each step typically intends adding of some operation to input parameter /// User application can specify sequence of preprocessing steps in a builder-like manner /// \code{.cpp} @@ -26,6 +24,7 @@ class PreProcessStepsImpl; /// ); /// \endcode class OPENVINO_API PreProcessSteps final { + class PreProcessStepsImpl; std::unique_ptr m_impl; friend class InputInfo; diff --git a/ngraph/core/src/preprocess/pre_post_process.cpp b/ngraph/core/src/preprocess/pre_post_process.cpp index d65245075684e6..1ff65d66a01070 100644 --- a/ngraph/core/src/preprocess/pre_post_process.cpp +++ b/ngraph/core/src/preprocess/pre_post_process.cpp @@ -11,7 +11,7 @@ namespace ov { namespace preprocess { /// \brief InputTensorInfoImpl - internal data structure -struct InputTensorInfoImpl { +struct InputTensorInfo::InputTensorInfoImpl { InputTensorInfoImpl() = default; explicit InputTensorInfoImpl(const element::Type& type) : m_type(type) {} @@ -19,56 +19,62 @@ struct InputTensorInfoImpl { }; /// \brief PreProcessStepsImpl - internal data structure -struct PreProcessStepsImpl { +struct PreProcessSteps::PreProcessStepsImpl { void add_scale_impl(const std::vector& values) { - m_actions.emplace_back([values](const std::shared_ptr& node) { - ngraph::Shape shape; - if (values.size() == 1) { - shape = ngraph::Shape{1}; - } else { - // TODO: implement when Layout API is available - } - auto constant = op::v0::Constant::create(element::f32, shape, values); - constant->set_friendly_name(node->get_friendly_name() + "/scale/Divide_Factor"); - - auto new_op = std::make_shared(node, constant); - new_op->set_friendly_name(node->get_friendly_name() + "/scale/Divide"); - return new_op; - }); + m_actions.emplace_back(std::make_tuple( + [values](const std::shared_ptr& node) { + ngraph::Shape shape; + if (values.size() == 1) { + shape = ngraph::Shape{1}; + } else { + // TODO: implement when Layout API is available + } + auto constant = op::v0::Constant::create(element::f32, shape, values); + constant->set_friendly_name(node->get_friendly_name() + "/scale/Divide_Factor"); + + auto new_op = std::make_shared(node, constant); + new_op->set_friendly_name(node->get_friendly_name() + "/scale/Divide"); + return new_op; + }, + false)); } void add_mean_impl(const std::vector& values) { - m_actions.emplace_back([values](const std::shared_ptr& node) { - ngraph::Shape shape; - if (values.size() == 1) { - shape = ngraph::Shape{1}; - } else { - // TODO: implement when Layout API is available - } - auto constant = op::v0::Constant::create(element::f32, shape, values); - constant->set_friendly_name(node->get_friendly_name() + "/mean/Mean_Const"); - - auto new_op = std::make_shared(node, constant); - new_op->set_friendly_name(node->get_friendly_name() + "/mean/Subtract"); - return new_op; - }); + m_actions.emplace_back(std::make_tuple( + [values](const std::shared_ptr& node) { + ngraph::Shape shape; + if (values.size() == 1) { + shape = ngraph::Shape{1}; + } else { + // TODO: implement when Layout API is available + } + auto constant = op::v0::Constant::create(element::f32, shape, values); + constant->set_friendly_name(node->get_friendly_name() + "/mean/Mean_Const"); + + auto new_op = std::make_shared(node, constant); + new_op->set_friendly_name(node->get_friendly_name() + "/mean/Subtract"); + return new_op; + }, + false)); } void add_convert_impl(const element::Type& type) { - m_actions.emplace_back([type](const std::shared_ptr& node) { - if (node->get_element_type().is_dynamic()) { - throw ngraph::ngraph_error("Can't insert 'convert_element_type' for dynamic source tensor type."); - } - auto convert = std::make_shared(node, type); - convert->set_friendly_name(node->get_friendly_name() + "/convert_element_type"); - return convert; - }); + m_actions.emplace_back(std::make_tuple( + [type](const std::shared_ptr& node) { + if (node->get_element_type().is_dynamic()) { + throw ngraph::ngraph_error("Can't insert 'convert_element_type' for dynamic source tensor type."); + } + auto convert = std::make_shared(node, type); + convert->set_friendly_name(node->get_friendly_name() + "/convert_element_type"); + return convert; + }, + true)); } - std::list m_actions; + std::list> m_actions; }; /// \brief InputInfoImpl - internal data structure -struct InputInfoImpl { +struct InputInfo::InputInfoImpl { InputInfoImpl() = default; explicit InputInfoImpl(size_t idx) : m_has_index(true), m_index(idx) {} @@ -76,10 +82,15 @@ struct InputInfoImpl { return m_has_index; } + void create_tensor_data(const element::Type& type) { + m_tensor_data = + std::unique_ptr(new InputTensorInfo::InputTensorInfoImpl(type)); + } + bool m_has_index = false; size_t m_index = 0; - std::unique_ptr m_tensor_data; - std::unique_ptr m_preprocess; + std::unique_ptr m_tensor_data; + std::unique_ptr m_preprocess; }; //-------------- InputInfo ------------------ @@ -110,9 +121,9 @@ InputInfo& InputInfo::preprocess(PreProcessSteps&& builder) & { } // ------------------------ PrePostProcessor -------------------- -struct PrePostProcessorImpl { +struct PrePostProcessor::PrePostProcessorImpl { public: - std::list> in_contexts; + std::list> in_contexts; }; PrePostProcessor::PrePostProcessor() : m_impl(std::unique_ptr(new PrePostProcessorImpl())) {} @@ -131,6 +142,7 @@ PrePostProcessor&& PrePostProcessor::input(InputInfo&& builder) && { } std::shared_ptr PrePostProcessor::build(const std::shared_ptr& function) { + bool tensor_data_updated = false; for (const auto& input : m_impl->in_contexts) { std::shared_ptr param; OV_CHECK(input, "Internal error: Invalid preprocessing input, please report a problem"); @@ -147,8 +159,7 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptroutput(0).get_target_inputs(); if (!input->m_tensor_data) { - input->m_tensor_data = - std::unique_ptr(new InputTensorInfoImpl(param->get_element_type())); + input->create_tensor_data(param->get_element_type()); } auto new_param_shape = param->get_partial_shape(); auto new_param = std::make_shared(input->m_tensor_data->m_type, new_param_shape); @@ -158,7 +169,8 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_preprocess->m_actions) { - node = action(node); + node = std::get<0>(action)(node); + tensor_data_updated |= std::get<1>(action); } // Check final type @@ -179,7 +191,9 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrreplace_parameter(0, new_param); } } - function->validate_nodes_and_infer_types(); + if (tensor_data_updated) { + function->validate_nodes_and_infer_types(); + } return function; } @@ -237,12 +251,14 @@ PreProcessSteps&& PreProcessSteps::convert_element_type(const element::Type& typ } PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & { - m_impl->m_actions.emplace_back(preprocess_cb); + // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types + m_impl->m_actions.emplace_back(std::make_tuple(preprocess_cb, true)); return *this; } PreProcessSteps&& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) && { - m_impl->m_actions.emplace_back(preprocess_cb); + // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types + m_impl->m_actions.emplace_back(std::make_tuple(preprocess_cb, true)); return std::move(*this); } diff --git a/ngraph/test/preprocess.cpp b/ngraph/test/preprocess.cpp index 874c473dbf0b30..340a70f09ef19e 100644 --- a/ngraph/test/preprocess.cpp +++ b/ngraph/test/preprocess.cpp @@ -86,7 +86,7 @@ TEST(pre_post_process, convert_element_type_from_unknown) { f = PrePostProcessor() .input(InputInfo().preprocess( PreProcessSteps().convert_element_type(element::dynamic).convert_element_type(element::i32))) - .build(f);); + .build(f)); } TEST(pre_post_process, convert_element_type_no_match) { @@ -95,7 +95,23 @@ TEST(pre_post_process, convert_element_type_no_match) { .input(InputInfo() .tensor(InputTensorInfo().set_element_type(element::i32)) .preprocess(PreProcessSteps().convert_element_type(element::f32).scale(2.0f))) - .build(f);); + .build(f)); +} + +TEST(pre_post_process, scale_not_float) { + auto f = create_simple_function(element::i32, ngraph::Shape{1, 3, 224, 224}); + ASSERT_ANY_THROW( + f = PrePostProcessor() + .input(InputInfo().preprocess(PreProcessSteps().convert_element_type(element::f32).scale(2.0f))) + .build(f)); +} + +TEST(pre_post_process, mean_not_float) { + auto f = create_simple_function(element::i32, ngraph::Shape{1, 3, 224, 224}); + ASSERT_ANY_THROW( + f = PrePostProcessor() + .input(InputInfo().preprocess(PreProcessSteps().convert_element_type(element::f32).mean(2.0f))) + .build(f)); } TEST(pre_post_process, tensor_element_type_and_scale) {