[CPU] Extended preprocessing for CPU (openvinotoolkit#5750)

docs/template_plugin/src/transformations/preprocessing/preprocessing.hpp

@@ -20,10 +20,10 @@ class AddPreprocessing;
  * @brief Converts the following preprocessing information to ngraph operations:
  *  - InferenceEngine::PreProcessInfo->PreProcessChannel::meanData -> Subtract
  *  - InferenceEngine::PreProcessInfo->PreProcessChannel::meanValue -> Subtract
- *  - InferenceEngine::PreProcessInfo->PreProcessChannel::stdScale -> Multiply
+ *  - InferenceEngine::PreProcessInfo->PreProcessChannel::stdScale -> Divide
  *
  * The order of operations is the following:
- *      (x - mean) * stdScale
+ *      (x - mean) / stdScale
  */
 class ngraph::pass::AddPreprocessing : public ngraph::pass::FunctionPass {
     const InferenceEngine::InputsDataMap& m_inputInfoMap;

docs/template_plugin/src/transformations/preprocessing/std_scale.cpp

@@ -31,10 +31,10 @@ ngraph::pass::AddStdScale::AddStdScale(const ScaleMap& inputInfoMap) {
         NGRAPH_CHECK(scale_const->get_element_type() == ngraph::element::f32, "Scale for ", param->get_friendly_name(), " must have f32 type");
 
         auto copy_param = param->clone_with_new_inputs({});
-        auto mul = std::make_shared<ngraph::opset3::Multiply>(copy_param, it->second);
+        auto div = std::make_shared<ngraph::opset3::Divide>(copy_param, it->second);
 
-        ngraph::replace_node(param, mul);
-        mul->set_argument(0, param);
+        ngraph::replace_node(param, div);
+        div->set_argument(0, param);
 
         // Return true as the root node was changed
         return true;

docs/template_plugin/tests/functional/transformations/preprocessing.cpp

@@ -43,8 +43,8 @@
 //         auto data = std::make_shared<opset5::Parameter>(element::f32, data_shape);
 //         auto scales = opset5::Constant::create(element::f32, scale_shape,
 //             std::vector<float>(shape_size(scale_shape), 2.0f));
-//         auto mul = std::make_shared<opset5::Multiply>(data, scales);
-//         auto relu = std::make_shared<opset5::Relu>(mul);
+//         auto div = std::make_shared<opset5::Divide>(data, scales);
+//         auto relu = std::make_shared<opset5::Relu>(div);
 //         f_ref = std::make_shared<Function>(NodeVector{relu}, ParameterVector{data});
 //     }
 
@@ -137,8 +137,8 @@
 //         auto scaleValues = opset5::Constant::create(element::f32, scale_shape,
 //             std::vector<float>(shape_size(scale_shape), 2.0f));
 //         auto sub = std::make_shared<opset5::Subtract>(data, meanValues);
-//         auto mul = std::make_shared<opset5::Multiply>(sub, scaleValues);
-//         auto relu = std::make_shared<opset5::Relu>(mul);
+//         auto div = std::make_shared<opset5::Divide>(sub, scaleValues);
+//         auto relu = std::make_shared<opset5::Relu>(div);
 //         f_ref = std::make_shared<Function>(NodeVector{relu}, ParameterVector{data});
 //     }
 
@@ -173,8 +173,8 @@
 //         auto scaleValues = opset5::Constant::create(element::f32, scale_shape,
 //             std::vector<float>(shape_size(scale_shape), 2.0f));
 //         auto sub = std::make_shared<opset5::Subtract>(data, meanValues);
-//         auto mul = std::make_shared<opset5::Multiply>(sub, meanValues);
-//         auto relu = std::make_shared<opset5::Relu>(mul);
+//         auto div = std::make_shared<opset5::Divide>(sub, meanValues);
+//         auto relu = std::make_shared<opset5::Relu>(div);
 //         f_ref = std::make_shared<Function>(NodeVector{relu}, ParameterVector{data});
 //     }
 

inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp

@@ -311,7 +311,7 @@ void MKLDNNGraph::Replicate(const CNNNetwork &network, const MKLDNNExtensionMana
         }
         InputInfo::Ptr ii = inputsInfo[input.first];
         if (ii && ii->getPreProcess().getNumberOfChannels()) {
-            _meanImages[input.first].Load(outDims, ii);
+            _normalizePreprocMap[input.first].Load(outDims, ii);
         }
     }
 }
@@ -362,7 +362,7 @@ void MKLDNNGraph::InitDescriptors() {
     OV_ITT_SCOPE_CHAIN(FIRST_INFERENCE, taskChain, MKLDNNPlugin::itt::domains::MKLDNN_LT, "InitDescriptors", "Prepare");
 
     for (auto &node : graphNodes) {
-        if (node->getType() == Input && _meanImages.find(node->getName()) != _meanImages.end()) {
+        if (node->getType() == Input && _normalizePreprocMap.find(node->getName()) != _normalizePreprocMap.end()) {
             auto *inputNode = dynamic_cast<MKLDNNInputNode *>(node.get());
             if (inputNode)
                 inputNode->withMeanImage();
@@ -726,9 +726,10 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine::
         }
 
         // todo: make sure 'name' exists in this map...
-        if (_meanImages.find(name) != _meanImages.end()) {
+        if (_normalizePreprocMap.find(name) != _normalizePreprocMap.end()) {
             if (in->getTensorDesc().getPrecision() == InferenceEngine::Precision::FP32) {
-                _meanImages[name].Subtract(outDims, reinterpret_cast<float *>(inter_data_ptr), in->getTensorDesc().getLayout());
+                _normalizePreprocMap[name].NormalizeImage(outDims, reinterpret_cast<float *>(inter_data_ptr),
+                                                          in->getTensorDesc().getLayout());
             } else {
                 IE_THROW() << "Mean image of type " << in->getTensorDesc().getPrecision().name() << " is unsupported";
             }

inference-engine/src/mkldnn_plugin/mkldnn_graph.h

@@ -7,7 +7,7 @@
 #include "cpp/ie_cnn_network.h"
 #include "config.h"
 #include "mkldnn_memory.h"
-#include "mean_image.h"
+#include "normalize_preprocess.h"
 #include "mkldnn_node.h"
 #include "mkldnn_edge.h"
 #include <map>
@@ -51,7 +51,7 @@ class MKLDNNGraph {
                      MKLDNNWeightsSharing::Ptr &w_cache);
 
     bool hasMeanImageFor(const std::string& name) {
-        return _meanImages.find(name) != _meanImages.end();
+        return _normalizePreprocMap.find(name) != _normalizePreprocMap.end();
     }
 
     void PushInputData(const std::string& name, const InferenceEngine::Blob::Ptr &in);
@@ -176,7 +176,7 @@ class MKLDNNGraph {
         outputNodesMap.clear();
         graphNodes.clear();
         graphEdges.clear();
-        _meanImages.clear();
+        _normalizePreprocMap.clear();
     }
     Status status { NotReady };
     Config config;
@@ -194,7 +194,7 @@ class MKLDNNGraph {
     std::vector<MKLDNNNodePtr> graphNodes;
     std::vector<MKLDNNEdgePtr> graphEdges;
 
-    std::map<std::string, MeanImage> _meanImages;
+    std::map<std::string, NormalizePreprocess> _normalizePreprocMap;
     std::string _name;
 
     bool isQuantizedFlag = false;

inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp

@@ -236,12 +236,25 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std::
             _inputs[name] = make_blob_with_precision(desc);
             _inputs[name]->allocate();
             if (blobs[name]->getTensorDesc() == desc &&
-                graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) {
+                graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
                 externalPtr[name] = _inputs[name]->buffer();
             }
         }
         data = _inputs[name];
         checkBlob(data, name, true);
+        // check if preprocess required, but still wasn't set
+        auto preProcessedInput = std::find_if(std::begin(_networkInputs), std::end(_networkInputs),
+            [&](const std::pair<std::string, InferenceEngine::InputInfo::Ptr>& pair)
+            {return pair.first == name;});
+        if (preProcessedInput!= std::end(_networkInputs)) {
+            auto preProcess = preProcessedInput->second->getPreProcess();
+            if (preProcess.getColorFormat() != InferenceEngine::ColorFormat::RAW ||
+                 preProcess.getResizeAlgorithm() != InferenceEngine::ResizeAlgorithm::NO_RESIZE) {
+                _preProcData.emplace(name, InferenceEngine::CreatePreprocDataHelper());
+                _preProcData[name]->isApplicable(data, _inputs[name]);
+                _preProcData[name]->setRoiBlob(data);
+            }
+        }
     }
 
     if (graph->hasOutputWithName(name)) {
@@ -359,7 +372,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
                 IE_THROW() << "MKLDNN graph doesn't contain input node with name: " << name;
 
             if (data->getTensorDesc() == blobs.at(name)->getTensorDesc() &&
-                graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) {
+                graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
                 externalPtr[name] = data->buffer();
             } else if (externalPtr.find(name) != externalPtr.end()) {
                 externalPtr.erase(name);

inference-engine/src/mkldnn_plugin/mean_image.cpp → inference-engine/src/mkldnn_plugin/normalize_preprocess.cpp

@@ -2,17 +2,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-#include "mean_image.h"
+#include "normalize_preprocess.h"
 #include "ie_parallel.hpp"
 #include "nodes/common/cpu_memcpy.h"
 
 using namespace MKLDNNPlugin;
 using namespace InferenceEngine;
 
-MeanImage::MeanImage() : meanBuffer(nullptr) {
+NormalizePreprocess::NormalizePreprocess() : meanBuffer(nullptr) {
 }
 
-void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
+void NormalizePreprocess::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
     PreProcessInfo &pp = inputInfo->getPreProcess();
     size_t inChannels = pp.getNumberOfChannels();
     if (inChannels == 0) {
@@ -26,11 +26,16 @@ void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
 
     switch (pp.getMeanVariant()) {
         case MEAN_VALUE: {
-            // mean image common value per channel (1x1xC)
+            // mean and standard deviation image common value per channel (1x1xC)
             meanValues.resize(inChannels);
+            stdScales.resize(inChannels);
 
             for (unsigned channel = 0; channel < inChannels; channel++) {
+                if (pp[channel]->stdScale == 0) {
+                    IE_THROW() << "Preprocessing error: stdScale cannot be equal zero";
+                }
                 meanValues[channel] = pp[channel]->meanValue;
+                stdScales[channel] = pp[channel]->stdScale;
             }
         }
         break;
@@ -71,7 +76,7 @@ void MeanImage::Load(const MKLDNNDims& inputDims, InputInfo::Ptr inputInfo) {
     }
 }
 
-void MeanImage::Subtract(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout) {
+void NormalizePreprocess::NormalizeImage(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout) {
     IE_ASSERT(input != nullptr);
 
     if (inputDims.ndims() != 4) {
@@ -91,19 +96,24 @@ void MeanImage::Subtract(const MKLDNNDims &inputDims, float *input, InferenceEng
         parallel_for2d(MB, srcSize, [&](int mb, int i) {
             input[srcSize * mb + i] -= meanBufferValues[i];
         });
-    } else if (!meanValues.empty()) {
+    } else if (!meanValues.empty() && !stdScales.empty()) {
         int C = inputDims[1];
         srcSize /= inputDims[1];
 
         if (layout == NCHW) {
             parallel_for3d(MB, C, srcSize, [&](int mb, int c, int i) {
                 input[mb * C * srcSize + c * srcSize + i] -= meanValues[c];
+                input[mb * C * srcSize + c * srcSize + i] /= stdScales[c];
             });
         } else if (layout == NHWC) {
             parallel_for2d(MB, srcSize, [&](int mb, int i) {
-                for (int c = 0; c < C; c++)
+                for (int c = 0; c < C; c++) {
                     input[mb * srcSize * C + i * C + c] -= meanValues[c];
+                    input[mb * srcSize * C + i * C + c] /= stdScales[c];
+                }
             });
         }
+    } else {
+        IE_THROW() << "Preprocessing error: meanValues and stdScales arrays are inconsistent.";
     }
 }

inference-engine/src/mkldnn_plugin/mean_image.h → inference-engine/src/mkldnn_plugin/normalize_preprocess.h

@@ -13,16 +13,16 @@
 
 namespace MKLDNNPlugin {
 
-class MeanImage {
+class NormalizePreprocess {
 public:
-    MeanImage();
+    NormalizePreprocess();
 
 public:
     void Load(const MKLDNNDims& inputDims, InferenceEngine::InputInfo::Ptr inputInfo);
-    void Subtract(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout);
+    void NormalizeImage(const MKLDNNDims &inputDims, float *input, InferenceEngine::Layout layout);
 
     template<typename T, typename std::enable_if<std::is_integral<T>::value>::type* = nullptr>
-    void Subtract(const MKLDNNDims &inputDims, T *input, InferenceEngine::Layout layout) {
+    void NormalizeImage(const MKLDNNDims &inputDims, T *input, InferenceEngine::Layout layout) {
         IE_ASSERT(input != nullptr);
 
         if (inputDims.ndims() != 4) {
@@ -46,10 +46,15 @@ class MeanImage {
                 if (buf > (std::numeric_limits<T>::max)()) buf = (std::numeric_limits<T>::max)();
                 input[srcSize * mb + i] = buf;
             });
-        } else if (!meanValues.empty()) {
+        } else if (!meanValues.empty() && !stdScales.empty()) {
             int C = inputDims[1];
             srcSize /= inputDims[1];
 
+            for (int c = 0; c < C; c++) {
+                if (stdScales[c] != 1)
+                    IE_THROW() << "Preprocessing error: fractional normalization is not supported for integer data. ";
+            }
+
             if (layout == InferenceEngine::NCHW) {
                 InferenceEngine::parallel_for3d(MB, C, srcSize, [&](int mb, int c, int i) {
                     int buf = input[srcSize * mb * C + c * srcSize + i];
@@ -69,12 +74,16 @@ class MeanImage {
                     }
                 });
             }
+        } else {
+            IE_THROW() << "Preprocessing error: meanValues and stdScales arrays are inconsistent.";
         }
     }
 
 private:
     std::vector<float> meanValues;
 
+    std::vector<float> stdScales;
+
     InferenceEngine::TBlob<float>::Ptr meanBuffer;
 };
 

inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp

@@ -24,11 +24,6 @@ std::vector<std::string> disabledTestPatterns() {
         R"(.*(RangeAddSubgraphTest).*Start=1.2.*Stop=(5.2|-5.2).*Step=(0.1|-0.1).*netPRC=FP16.*)",
         R"(.*(RangeNumpyAddSubgraphTest).*netPRC=FP16.*)",
         // TODO: Issue: 43793
-        R"(.*(PreprocessTest).*(SetScalePreProcessSetBlob).*)",
-        R"(.*(PreprocessTest).*(SetScalePreProcessGetBlob).*)",
-        R"(.*(PreprocessTest).*(SetMeanValuePreProcessSetBlob).*)",
-        R"(.*(PreprocessTest).*(SetMeanImagePreProcessSetBlob).*)",
-        R"(.*(PreprocessTest).*(ReverseInputChannelsPreProcessGetBlob).*)",
         R"(.*PreprocessDynamicallyInSetBlobTest.*iPRC=0.*_iLT=1.*)",
         R"(.*PreprocessDynamicallyInSetBlobTest.*oPRC=0.*_oLT=1.*)",
         // TODO: Issue: 34348

inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp

@@ -37,6 +37,9 @@ std::vector<std::string> disabledTestPatterns() {
         R"(.*CTCGreedyDecoderSeqLen.*?\(1.1.1\).*)",
         // TODO: Issue 51804
         ".*PreprocessConversionTest.*oPRC=U8.*",
+        // TODO: Issue: 56556
+        R"(.*(PreprocessTest).*(SetScalePreProcessSetBlob).*)",
+        R"(.*(PreprocessTest).*(SetScalePreProcessGetBlob).*)",
         // TODO: Issue 54163
         R"(.*ActivationLayerTest.*SoftPlus.*)",
         // TODO: Issue 54722

inference-engine/tests/functional/plugin/shared/include/behavior/set_preprocess.hpp

@@ -323,11 +323,10 @@ TEST_P(PreprocessTest, SetMeanValuePreProcessSetBlob) {
         const auto* outData = outMem.as<const float*>();
         ASSERT_EQ(inBlob->size(), outBlob->size());
         for (size_t i = 0; i < inBlob->size(); i++)
-            ASSERT_EQ(inData[i]+5, outData[i]);
+            ASSERT_EQ(inData[i] + 5, outData[i]);
     }
 }
 
-
 TEST_P(PreprocessTest, ReverseInputChannelsPreProcessGetBlob) {
     // Skip test according to plugin specific disabledTestPatterns() (if any)
     SKIP_IF_CURRENT_TEST_IS_DISABLED()
@@ -391,7 +390,6 @@ TEST_P(PreprocessTest, ReverseInputChannelsPreProcessGetBlob) {
     }
 }
 
-
 TEST_P(PreprocessTest, ReverseInputChannelsPreProcessSetBlob) {
     // Skip test according to plugin specific disabledTestPatterns() (if any)
     SKIP_IF_CURRENT_TEST_IS_DISABLED()
@@ -515,12 +513,11 @@ TEST_P(PreprocessTest, SetScalePreProcessGetBlob) {
         const auto* outData = outMem.as<const float*>();
         ASSERT_EQ(inBlob->size(), outBlob->size());
         for (size_t i = 0; i < inBlob->size(); i++) {
-            ASSERT_EQ(inData[i]*2, outData[i]);
+            ASSERT_EQ(inData[i] / 2, outData[i]);
         }
     }
 }
 
-
 TEST_P(PreprocessTest, SetScalePreProcessSetBlob) {
     // Skip test according to plugin specific disabledTestPatterns() (if any)
     SKIP_IF_CURRENT_TEST_IS_DISABLED()
@@ -581,7 +578,7 @@ TEST_P(PreprocessTest, SetScalePreProcessSetBlob) {
         const auto* outData = outMem.as<const float*>();
         ASSERT_EQ(inBlob->size(), outBlob->size());
         for (size_t i = 0; i < inBlob->size(); i++)
-            ASSERT_EQ(inData[i]*2, outData[i]);
+            ASSERT_EQ(inData[i] / 2, outData[i]);
     }
 }