From 11da85b19585bbd3529c7b87067ca7cea68dbd5e Mon Sep 17 00:00:00 2001 From: Yury Gaydaychuk Date: Tue, 1 Jun 2021 17:03:24 +0300 Subject: [PATCH] [CPU] Extended preprocessing for CPU (#5750) --- .../preprocessing/preprocessing.hpp | 4 ++-- .../preprocessing/std_scale.cpp | 6 ++--- .../transformations/preprocessing.cpp | 12 +++++----- .../src/mkldnn_plugin/mkldnn_graph.cpp | 9 +++---- .../src/mkldnn_plugin/mkldnn_graph.h | 8 +++---- .../mkldnn_plugin/mkldnn_infer_request.cpp | 17 +++++++++++-- ...ean_image.cpp => normalize_preprocess.cpp} | 24 +++++++++++++------ .../{mean_image.h => normalize_preprocess.h} | 19 +++++++++++---- .../skip_tests_config.cpp | 5 ---- .../skip_tests_config.cpp | 3 +++ .../include/behavior/set_preprocess.hpp | 9 +++---- 11 files changed, 72 insertions(+), 44 deletions(-) rename inference-engine/src/mkldnn_plugin/{mean_image.cpp => normalize_preprocess.cpp} (77%) rename inference-engine/src/mkldnn_plugin/{mean_image.h => normalize_preprocess.h} (80%) diff --git a/docs/template_plugin/src/transformations/preprocessing/preprocessing.hpp b/docs/template_plugin/src/transformations/preprocessing/preprocessing.hpp index c724f06aa0ea73..1725c04a11b191 100644 --- a/docs/template_plugin/src/transformations/preprocessing/preprocessing.hpp +++ b/docs/template_plugin/src/transformations/preprocessing/preprocessing.hpp @@ -20,10 +20,10 @@ class AddPreprocessing; * @brief Converts the following preprocessing information to ngraph operations: * - InferenceEngine::PreProcessInfo->PreProcessChannel::meanData -> Subtract * - InferenceEngine::PreProcessInfo->PreProcessChannel::meanValue -> Subtract - * - InferenceEngine::PreProcessInfo->PreProcessChannel::stdScale -> Multiply + * - InferenceEngine::PreProcessInfo->PreProcessChannel::stdScale -> Divide * * The order of operations is the following: - * (x - mean) * stdScale + * (x - mean) / stdScale */ class ngraph::pass::AddPreprocessing : public ngraph::pass::FunctionPass { const InferenceEngine::InputsDataMap& m_inputInfoMap; diff --git a/docs/template_plugin/src/transformations/preprocessing/std_scale.cpp b/docs/template_plugin/src/transformations/preprocessing/std_scale.cpp index 90c5163bdf2a1d..b8c144a19f012f 100644 --- a/docs/template_plugin/src/transformations/preprocessing/std_scale.cpp +++ b/docs/template_plugin/src/transformations/preprocessing/std_scale.cpp @@ -31,10 +31,10 @@ ngraph::pass::AddStdScale::AddStdScale(const ScaleMap& inputInfoMap) { NGRAPH_CHECK(scale_const->get_element_type() == ngraph::element::f32, "Scale for ", param->get_friendly_name(), " must have f32 type"); auto copy_param = param->clone_with_new_inputs({}); - auto mul = std::make_shared(copy_param, it->second); + auto div = std::make_shared(copy_param, it->second); - ngraph::replace_node(param, mul); - mul->set_argument(0, param); + ngraph::replace_node(param, div); + div->set_argument(0, param); // Return true as the root node was changed return true; diff --git a/docs/template_plugin/tests/functional/transformations/preprocessing.cpp b/docs/template_plugin/tests/functional/transformations/preprocessing.cpp index b7721e68d489fd..b138e36e97ad25 100644 --- a/docs/template_plugin/tests/functional/transformations/preprocessing.cpp +++ b/docs/template_plugin/tests/functional/transformations/preprocessing.cpp @@ -43,8 +43,8 @@ // auto data = std::make_shared(element::f32, data_shape); // auto scales = opset5::Constant::create(element::f32, scale_shape, // std::vector(shape_size(scale_shape), 2.0f)); -// auto mul = std::make_shared(data, scales); -// auto relu = std::make_shared(mul); +// auto div = std::make_shared(data, scales); +// auto relu = std::make_shared(div); // f_ref = std::make_shared(NodeVector{relu}, ParameterVector{data}); // } @@ -137,8 +137,8 @@ // auto scaleValues = opset5::Constant::create(element::f32, scale_shape, // std::vector(shape_size(scale_shape), 2.0f)); // auto sub = std::make_shared(data, meanValues); -// auto mul = std::make_shared(sub, scaleValues); -// auto relu = std::make_shared(mul); +// auto div = std::make_shared(sub, scaleValues); +// auto relu = std::make_shared(div); // f_ref = std::make_shared(NodeVector{relu}, ParameterVector{data}); // } @@ -173,8 +173,8 @@ // auto scaleValues = opset5::Constant::create(element::f32, scale_shape, // std::vector(shape_size(scale_shape), 2.0f)); // auto sub = std::make_shared(data, meanValues); -// auto mul = std::make_shared(sub, meanValues); -// auto relu = std::make_shared(mul); +// auto div = std::make_shared(sub, meanValues); +// auto relu = std::make_shared(div); // f_ref = std::make_shared(NodeVector{relu}, ParameterVector{data}); // } diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp index 86820c88ce405a..b92afb8a9f0d4f 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp @@ -311,7 +311,7 @@ void MKLDNNGraph::Replicate(const CNNNetwork &network, const MKLDNNExtensionMana } InputInfo::Ptr ii = inputsInfo[input.first]; if (ii && ii->getPreProcess().getNumberOfChannels()) { - _meanImages[input.first].Load(outDims, ii); + _normalizePreprocMap[input.first].Load(outDims, ii); } } } @@ -362,7 +362,7 @@ void MKLDNNGraph::InitDescriptors() { OV_ITT_SCOPE_CHAIN(FIRST_INFERENCE, taskChain, MKLDNNPlugin::itt::domains::MKLDNN_LT, "InitDescriptors", "Prepare"); for (auto &node : graphNodes) { - if (node->getType() == Input && _meanImages.find(node->getName()) != _meanImages.end()) { + if (node->getType() == Input && _normalizePreprocMap.find(node->getName()) != _normalizePreprocMap.end()) { auto *inputNode = dynamic_cast(node.get()); if (inputNode) inputNode->withMeanImage(); @@ -726,9 +726,10 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine:: } // todo: make sure 'name' exists in this map... - if (_meanImages.find(name) != _meanImages.end()) { + if (_normalizePreprocMap.find(name) != _normalizePreprocMap.end()) { if (in->getTensorDesc().getPrecision() == InferenceEngine::Precision::FP32) { - _meanImages[name].Subtract(outDims, reinterpret_cast(inter_data_ptr), in->getTensorDesc().getLayout()); + _normalizePreprocMap[name].NormalizeImage(outDims, reinterpret_cast(inter_data_ptr), + in->getTensorDesc().getLayout()); } else { IE_THROW() << "Mean image of type " << in->getTensorDesc().getPrecision().name() << " is unsupported"; } diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.h b/inference-engine/src/mkldnn_plugin/mkldnn_graph.h index 822cdeb387b6a6..c3fcb0d5c9c635 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.h +++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.h @@ -7,7 +7,7 @@ #include "cpp/ie_cnn_network.h" #include "config.h" #include "mkldnn_memory.h" -#include "mean_image.h" +#include "normalize_preprocess.h" #include "mkldnn_node.h" #include "mkldnn_edge.h" #include @@ -51,7 +51,7 @@ class MKLDNNGraph { MKLDNNWeightsSharing::Ptr &w_cache); bool hasMeanImageFor(const std::string& name) { - return _meanImages.find(name) != _meanImages.end(); + return _normalizePreprocMap.find(name) != _normalizePreprocMap.end(); } void PushInputData(const std::string& name, const InferenceEngine::Blob::Ptr &in); @@ -176,7 +176,7 @@ class MKLDNNGraph { outputNodesMap.clear(); graphNodes.clear(); graphEdges.clear(); - _meanImages.clear(); + _normalizePreprocMap.clear(); } Status status { NotReady }; Config config; @@ -194,7 +194,7 @@ class MKLDNNGraph { std::vector graphNodes; std::vector graphEdges; - std::map _meanImages; + std::map _normalizePreprocMap; std::string _name; bool isQuantizedFlag = false; diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp index 2496ea27fb6913..738604a6f0a6ac 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp @@ -236,12 +236,25 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std:: _inputs[name] = make_blob_with_precision(desc); _inputs[name]->allocate(); if (blobs[name]->getTensorDesc() == desc && - graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) { + graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) { externalPtr[name] = _inputs[name]->buffer(); } } data = _inputs[name]; checkBlob(data, name, true); + // check if preprocess required, but still wasn't set + auto preProcessedInput = std::find_if(std::begin(_networkInputs), std::end(_networkInputs), + [&](const std::pair& pair) + {return pair.first == name;}); + if (preProcessedInput!= std::end(_networkInputs)) { + auto preProcess = preProcessedInput->second->getPreProcess(); + if (preProcess.getColorFormat() != InferenceEngine::ColorFormat::RAW || + preProcess.getResizeAlgorithm() != InferenceEngine::ResizeAlgorithm::NO_RESIZE) { + _preProcData.emplace(name, InferenceEngine::CreatePreprocDataHelper()); + _preProcData[name]->isApplicable(data, _inputs[name]); + _preProcData[name]->setRoiBlob(data); + } + } } if (graph->hasOutputWithName(name)) { @@ -359,7 +372,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In IE_THROW() << "MKLDNN graph doesn't contain input node with name: " << name; if (data->getTensorDesc() == blobs.at(name)->getTensorDesc() && - graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) { + graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) { externalPtr[name] = data->buffer(); } else if (externalPtr.find(name) != externalPtr.end()) { externalPtr.erase(name); diff --git a/inference-engine/src/mkldnn_plugin/mean_image.cpp b/inference-engine/src/mkldnn_plugin/normalize_preprocess.cpp similarity index 77% rename from inference-engine/src/mkldnn_plugin/mean_image.cpp rename to inference-engine/src/mkldnn_plugin/normalize_preprocess.cpp index 056037421e464e..7007c6ad00a13f 100644 --- a/inference-engine/src/mkldnn_plugin/mean_image.cpp +++ b/inference-engine/src/mkldnn_plugin/normalize_preprocess.cpp @@ -2,17 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "mean_image.h" +#include "normalize_preprocess.h" #include "ie_parallel.hpp" #include "nodes/common/cpu_memcpy.h" using namespace MKLDNNPlugin; using namespace InferenceEngine; -MeanImage::MeanImage() : meanBuffer(nullptr) { +NormalizePreprocess::NormalizePreprocess() : meanBuffer(nullptr) { } -void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) { +void NormalizePreprocess::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) { PreProcessInfo &pp = inputInfo->getPreProcess(); size_t inChannels = pp.getNumberOfChannels(); if (inChannels == 0) { @@ -26,11 +26,16 @@ void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) { switch (pp.getMeanVariant()) { case MEAN_VALUE: { - // mean image common value per channel (1x1xC) + // mean and standard deviation image common value per channel (1x1xC) meanValues.resize(inChannels); + stdScales.resize(inChannels); for (unsigned channel = 0; channel < inChannels; channel++) { + if (pp[channel]->stdScale == 0) { + IE_THROW() << "Preprocessing error: stdScale cannot be equal zero"; + } meanValues[channel] = pp[channel]->meanValue; + stdScales[channel] = pp[channel]->stdScale; } } break; @@ -71,7 +76,7 @@ void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) { } } -void MeanImage::Subtract(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout) { +void NormalizePreprocess::NormalizeImage(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout) { IE_ASSERT(input != nullptr); if (inputDims.ndims() != 4) { @@ -91,19 +96,24 @@ void MeanImage::Subtract(const MKLDNNDims &inputDims, float *input, InferenceEng parallel_for2d(MB, srcSize, [&](int mb, int i) { input[srcSize * mb + i] -= meanBufferValues[i]; }); - } else if (!meanValues.empty()) { + } else if (!meanValues.empty() && !stdScales.empty()) { int C = inputDims[1]; srcSize /= inputDims[1]; if (layout == NCHW) { parallel_for3d(MB, C, srcSize, [&](int mb, int c, int i) { input[mb * C * srcSize + c * srcSize + i] -= meanValues[c]; + input[mb * C * srcSize + c * srcSize + i] /= stdScales[c]; }); } else if (layout == NHWC) { parallel_for2d(MB, srcSize, [&](int mb, int i) { - for (int c = 0; c < C; c++) + for (int c = 0; c < C; c++) { input[mb * srcSize * C + i * C + c] -= meanValues[c]; + input[mb * srcSize * C + i * C + c] /= stdScales[c]; + } }); } + } else { + IE_THROW() << "Preprocessing error: meanValues and stdScales arrays are inconsistent."; } } diff --git a/inference-engine/src/mkldnn_plugin/mean_image.h b/inference-engine/src/mkldnn_plugin/normalize_preprocess.h similarity index 80% rename from inference-engine/src/mkldnn_plugin/mean_image.h rename to inference-engine/src/mkldnn_plugin/normalize_preprocess.h index b3264f7250b2de..1bc6d8431957fd 100644 --- a/inference-engine/src/mkldnn_plugin/mean_image.h +++ b/inference-engine/src/mkldnn_plugin/normalize_preprocess.h @@ -13,16 +13,16 @@ namespace MKLDNNPlugin { -class MeanImage { +class NormalizePreprocess { public: - MeanImage(); + NormalizePreprocess(); public: void Load(const MKLDNNDims& inputDims, InferenceEngine::InputInfo::Ptr inputInfo); - void Subtract(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout); + void NormalizeImage(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout); template::value>::type* = nullptr> - void Subtract(const MKLDNNDims &inputDims, T *input, InferenceEngine::Layout layout) { + void NormalizeImage(const MKLDNNDims &inputDims, T *input, InferenceEngine::Layout layout) { IE_ASSERT(input != nullptr); if (inputDims.ndims() != 4) { @@ -46,10 +46,15 @@ class MeanImage { if (buf > (std::numeric_limits::max)()) buf = (std::numeric_limits::max)(); input[srcSize * mb + i] = buf; }); - } else if (!meanValues.empty()) { + } else if (!meanValues.empty() && !stdScales.empty()) { int C = inputDims[1]; srcSize /= inputDims[1]; + for (int c = 0; c < C; c++) { + if (stdScales[c] != 1) + IE_THROW() << "Preprocessing error: fractional normalization is not supported for integer data. "; + } + if (layout == InferenceEngine::NCHW) { InferenceEngine::parallel_for3d(MB, C, srcSize, [&](int mb, int c, int i) { int buf = input[srcSize * mb * C + c * srcSize + i]; @@ -69,12 +74,16 @@ class MeanImage { } }); } + } else { + IE_THROW() << "Preprocessing error: meanValues and stdScales arrays are inconsistent."; } } private: std::vector meanValues; + std::vector stdScales; + InferenceEngine::TBlob::Ptr meanBuffer; }; diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp index 18ca8b7cf73101..a43937cf6c6bb7 100644 --- a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp @@ -24,11 +24,6 @@ std::vector disabledTestPatterns() { R"(.*(RangeAddSubgraphTest).*Start=1.2.*Stop=(5.2|-5.2).*Step=(0.1|-0.1).*netPRC=FP16.*)", R"(.*(RangeNumpyAddSubgraphTest).*netPRC=FP16.*)", // TODO: Issue: 43793 - R"(.*(PreprocessTest).*(SetScalePreProcessSetBlob).*)", - R"(.*(PreprocessTest).*(SetScalePreProcessGetBlob).*)", - R"(.*(PreprocessTest).*(SetMeanValuePreProcessSetBlob).*)", - R"(.*(PreprocessTest).*(SetMeanImagePreProcessSetBlob).*)", - R"(.*(PreprocessTest).*(ReverseInputChannelsPreProcessGetBlob).*)", R"(.*PreprocessDynamicallyInSetBlobTest.*iPRC=0.*_iLT=1.*)", R"(.*PreprocessDynamicallyInSetBlobTest.*oPRC=0.*_oLT=1.*)", // TODO: Issue: 34348 diff --git a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp index 12271e7c66b278..158da0a75906f1 100644 --- a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp @@ -37,6 +37,9 @@ std::vector disabledTestPatterns() { R"(.*CTCGreedyDecoderSeqLen.*?\(1.1.1\).*)", // TODO: Issue 51804 ".*PreprocessConversionTest.*oPRC=U8.*", + // TODO: Issue: 56556 + R"(.*(PreprocessTest).*(SetScalePreProcessSetBlob).*)", + R"(.*(PreprocessTest).*(SetScalePreProcessGetBlob).*)", // TODO: Issue 54163 R"(.*ActivationLayerTest.*SoftPlus.*)", // TODO: Issue 54722 diff --git a/inference-engine/tests/functional/plugin/shared/include/behavior/set_preprocess.hpp b/inference-engine/tests/functional/plugin/shared/include/behavior/set_preprocess.hpp index 8939acbd4ec557..de442f12c215a8 100644 --- a/inference-engine/tests/functional/plugin/shared/include/behavior/set_preprocess.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/behavior/set_preprocess.hpp @@ -323,11 +323,10 @@ TEST_P(PreprocessTest, SetMeanValuePreProcessSetBlob) { const auto* outData = outMem.as(); ASSERT_EQ(inBlob->size(), outBlob->size()); for (size_t i = 0; i < inBlob->size(); i++) - ASSERT_EQ(inData[i]+5, outData[i]); + ASSERT_EQ(inData[i] + 5, outData[i]); } } - TEST_P(PreprocessTest, ReverseInputChannelsPreProcessGetBlob) { // Skip test according to plugin specific disabledTestPatterns() (if any) SKIP_IF_CURRENT_TEST_IS_DISABLED() @@ -391,7 +390,6 @@ TEST_P(PreprocessTest, ReverseInputChannelsPreProcessGetBlob) { } } - TEST_P(PreprocessTest, ReverseInputChannelsPreProcessSetBlob) { // Skip test according to plugin specific disabledTestPatterns() (if any) SKIP_IF_CURRENT_TEST_IS_DISABLED() @@ -515,12 +513,11 @@ TEST_P(PreprocessTest, SetScalePreProcessGetBlob) { const auto* outData = outMem.as(); ASSERT_EQ(inBlob->size(), outBlob->size()); for (size_t i = 0; i < inBlob->size(); i++) { - ASSERT_EQ(inData[i]*2, outData[i]); + ASSERT_EQ(inData[i] / 2, outData[i]); } } } - TEST_P(PreprocessTest, SetScalePreProcessSetBlob) { // Skip test according to plugin specific disabledTestPatterns() (if any) SKIP_IF_CURRENT_TEST_IS_DISABLED() @@ -581,7 +578,7 @@ TEST_P(PreprocessTest, SetScalePreProcessSetBlob) { const auto* outData = outMem.as(); ASSERT_EQ(inBlob->size(), outBlob->size()); for (size_t i = 0; i < inBlob->size(); i++) - ASSERT_EQ(inData[i]*2, outData[i]); + ASSERT_EQ(inData[i] / 2, outData[i]); } }