[CPU] A new transformation that adds a convert layer if there is no reorders that support the data type conversion.

inference-engine/src/mkldnn_plugin/CMakeLists.txt

@@ -46,11 +46,11 @@ set(LAYERS
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_scatter_update_node.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_interpolate_node.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_reduce_node.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_convert_node.cpp
 
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/list.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/batch_to_space.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/broadcast.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/nodes/convert.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/ctc_greedy_decoder.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/ctc_greedy_decoder_seq_len.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/nodes/ctc_loss.cpp

inference-engine/src/mkldnn_plugin/bf16transformer.cpp

@@ -43,6 +43,7 @@ void BF16Transformer::convertToFloat(InferenceEngine::CNNNetwork &network) {
         for (size_t o = 0; o < iter->outData.size(); o++) {
             if (inputs.find(iter->outData[o]->getName()) == inputs.end()
                 && outputs.find(iter->outData[o]->getName()) == outputs.end()
+                && !CaselessEq<std::string>()(iter->type, "const")
                 && iter->outData[o]->getPrecision() == Precision::BF16) {
                 iter->outData[o]->setPrecision(Precision::FP32);
             }

inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp

@@ -23,6 +23,7 @@
 #include "mkldnn_infer_request.h"
 #include <nodes/mkldnn_input_node.h>
 #include <nodes/mkldnn_reorder_node.h>
+#include <nodes/mkldnn_convert_node.h>
 
 #include <legacy/graph_tools.hpp>
 #include <ie_algorithm.hpp>
@@ -457,6 +458,21 @@ void MKLDNNGraph::ExecuteConstantNodesOnly() {
     }
 }
 
+static bool isReorderAvailable(const TensorDesc& parentDesc, const TensorDesc& childDesc, const mkldnn::engine& eng) {
+    memory::desc dstMemDesc = MKLDNNMemoryDesc(childDesc);
+    memory::desc srcMemDesc = MKLDNNMemoryDesc(parentDesc);
+    mkldnn::primitive_attr attr;
+
+    dnnl_primitive_desc_t result = nullptr;
+    auto status = dnnl_reorder_primitive_desc_create(&result, &srcMemDesc.data, eng.get(), &dstMemDesc.data, eng.get(),
+                                                     attr.get());
+    if (result) {
+        mkldnn_primitive_desc_destroy(result);
+    }
+
+    return mkldnn_success == status;
+}
+
 void MKLDNNGraph::InitEdges() {
     OV_ITT_SCOPED_TASK(itt::domains::MKLDNN_LT, "MKLDNNGraph::InitEdges");
 
@@ -470,18 +486,42 @@ void MKLDNNGraph::InitEdges() {
     for (auto i = 0; i < numberOfEdges; i++) {
         if (graphEdges[i]->needReorder()) {
 #if defined (COMPILED_CPU_MKLDNN_REORDER_NODE)
-            auto &edge = graphEdges[i];
-            std::string basicLayerName = edge->getParent()->getName() + "_" +
-                    MKLDNNExtensionUtils::getReorderArgs(edge->getInputDesc(), edge->getOutputDesc()) + "_" +
-                    edge->getChild()->getName();
-            std::string layerName = basicLayerName;
-            int idx = 0;
-            while (uniqueLayerNames.find(layerName) != uniqueLayerNames.end()) {
-                idx++;
-                layerName = basicLayerName + "_" + std::to_string(idx);
+            auto edge = graphEdges[i];
+            bool insertReorder = true;
+
+            // Check if there is a reorder that supports the type conversion
+            if (edge->getInputDesc().getPrecision() != edge->getOutputDesc().getPrecision() &&
+                !isReorderAvailable(edge->getInputDesc(), edge->getOutputDesc(), this->getEngine())) {
+                //If we are here, then we need to insert Convert, because there are no reorders that support such type conversion
+                std::string convertName = edge->getParent()->getName() + "_" +
+                                          edge->getInputDesc().getPrecision().name() + "_" + edge->getOutputDesc().getPrecision().name();
+
+                CNNLayerPtr convert(new CNNLayer(LayerParams{convertName, "Convert", edge->getInputDesc().getPrecision()}));
+                auto convertNode = std::make_shared<MKLDNNConvertNode>(convert, this->getEngine(), this->weightsCache);
+                convertNode->setDescs(edge->getInputDesc(), edge->getOutputDesc());
+                InsertNode(edge, convertNode, true);
+
+                //Check if reorder is still needed
+                if (convertNode->getChildEdgeAt(0)->needReorder()) {
+                    edge = convertNode->getChildEdgeAt(0);
+                } else {
+                    insertReorder = false;
+                }
+            }
+
+            if (insertReorder) {
+                std::string basicLayerName = edge->getParent()->getName() + "_" +
+                                             MKLDNNExtensionUtils::getReorderArgs(edge->getInputDesc(), edge->getOutputDesc()) + "_" +
+                                             edge->getChild()->getName();
+                std::string layerName = basicLayerName;
+                int idx = 0;
+                while (uniqueLayerNames.find(layerName) != uniqueLayerNames.end()) {
+                    idx++;
+                    layerName = basicLayerName + "_" + std::to_string(idx);
+                }
+                uniqueLayerNames.insert(layerName);
+                InsertReorder(edge, layerName, edge->getInputDesc(), edge->getOutputDesc());
             }
-            uniqueLayerNames.insert(layerName);
-            InsertReorder(edge, layerName, edge->getInputDesc(), edge->getOutputDesc());
             graphEdges.erase(graphEdges.begin() + i);
             i--;
             numberOfEdges--;
@@ -1095,44 +1135,17 @@ MKLDNNNodePtr MKLDNNGraph::InsertReorder(MKLDNNEdgePtr edge, std::string layerNa
     }
     reorderPtr->setDescs(inDesc, outDesc);
     reorderPtr->_scales = scales;
-
-    auto oIndex = edge->getOutputNum();
-    auto iIndex = edge->getInputNum();
-    if (iIndex < 0 || oIndex < 0)
-        THROW_IE_EXCEPTION << "Cannot create reorder for nodes: "
-                           << edge->getParent()->getName() << " and "
-                           << edge->getChild()->getName() << ".";
-
-    edge->drop();
-
-    MKLDNNEdgePtr beforeNode(new MKLDNNEdge(edge->getParent(), newReorder, iIndex, 0));
-    MKLDNNEdgePtr afterNode(new MKLDNNEdge(newReorder, edge->getChild(), 0, oIndex));
-
-    // Add edge for beforeNode
-    beforeNode->getChild()->parentEdges.push_back(beforeNode);
-    edge->getParent()->childEdges.push_back(beforeNode);
-
-    // Add edge for afterNode
-    afterNode->getParent()->childEdges.push_back(afterNode);
-    edge->getChild()->parentEdges.push_back(afterNode);
-
     reorderPtr->setOptimized(isOptimized);
 
-    newReorder->getSupportedDescriptors();
-    newReorder->initSupportedPrimitiveDescriptors();
-    newReorder->selectOptimalPrimitiveDescriptor();
-
-    graphEdges.push_back(beforeNode);
-    graphEdges.push_back(afterNode);
+    InsertNode(edge, newReorder, true);
 
     // Using the method MKLDNNEdge::getDesc() we can check that input and output tensor descriptors are equal.
     // Due to the specificity of MKLDNNGraphOptimizer::MergePermuteAndReorder() that isOptimized flag uses, we shouldn't do these checks.
     if (!isOptimized) {
-        beforeNode->getDesc();
-        afterNode->getDesc();
+        newReorder->getParentEdgeAt(0)->getDesc();
+        newReorder->getChildEdgeAt(0)->getDesc();
     }
 
-    graphNodes.push_back(newReorder);
     return newReorder;
 }
 
@@ -1235,3 +1248,42 @@ void MKLDNNGraph::do_after(const std::string &dir, const MKLDNNNodePtr &node) {
 InferenceEngine::CNNNetwork MKLDNNGraph::dump() const {
     return dump_graph_as_ie_ngraph_net(*this);
 }
+
+bool MKLDNNGraph::InsertNode(MKLDNNEdgePtr edge, MKLDNNNodePtr node, bool initNode) {
+    auto oIndex = edge->getOutputNum();
+    auto iIndex = edge->getInputNum();
+    if (iIndex < 0 || oIndex < 0)
+        THROW_IE_EXCEPTION << "Cannot insert node '" << node->getName() << "' between nodes: "
+                           << edge->getParent()->getName() << " and "
+                           << edge->getChild()->getName() << ".";
+
+    edge->drop();
+
+    return InsertNode(edge->getParent(), edge->getChild(), node, iIndex, oIndex, initNode);
+}
+
+bool MKLDNNGraph::InsertNode(MKLDNNNodePtr parent, MKLDNNNodePtr child, MKLDNNNodePtr node, int parentPort, int childPort, bool initNode) {
+    MKLDNNEdgePtr beforeNode(new MKLDNNEdge(parent, node, parentPort, 0));
+    MKLDNNEdgePtr afterNode(new MKLDNNEdge(node, child, 0, childPort));
+
+    // Add edge for beforeNode
+    beforeNode->getChild()->parentEdges.push_back(beforeNode);
+    parent->childEdges.push_back(beforeNode);
+
+    // Add edge for afterNode
+    afterNode->getParent()->childEdges.push_back(afterNode);
+    child->parentEdges.push_back(afterNode);
+
+    if (initNode) {
+        node->getSupportedDescriptors();
+        node->initSupportedPrimitiveDescriptors();
+        node->filterSupportedPrimitiveDescriptors();
+        node->selectOptimalPrimitiveDescriptor();
+        node->initOptimalPrimitiveDescriptor();
+    }
+
+    graphEdges.push_back(beforeNode);
+    graphEdges.push_back(afterNode);
+    graphNodes.push_back(node);
+    return true;
+}

inference-engine/src/mkldnn_plugin/mkldnn_graph.h

@@ -115,6 +115,41 @@ class MKLDNNGraph {
     MKLDNNNodePtr InsertReorder(MKLDNNEdgePtr edge, std::string layerName, const InferenceEngine::TensorDesc& inDesc,
             const InferenceEngine::TensorDesc& outDesc, bool isOptimized = false, InferenceEngine::Blob::Ptr scales = nullptr);
 
+    /**
+     * @brief Insert MKLDNNNode at the edge-specified location.
+     * This method supports two regimes. First, the node is inserted without initialization (i.e. supported descriptors initialization,
+     * supported primitive descriptors selection, etc.), which can be useful after the InitEdges() completes. The second is just inserting the
+     * node without initialization.
+     * @param edge
+     * pointer to the edge in the graph where the node will be inserted
+     * @param node
+     * pointer to the inserted node
+     * @param initNode
+     * parameter that determines whether the node needs to be initialized
+     * @return true in case of success, false otherwise.
+     */
+    bool InsertNode(MKLDNNEdgePtr edge, MKLDNNNodePtr node, bool initNode = false);
+
+    /**
+     * @brief Insert MKLDNNNode between two specified nodes.
+     * This procedure creates two edges that link the parent and child nodes to the inserted one and adds all created objects to the graph.
+     * This method supports two regimes. First, the node is inserted without initialization (i.e. supported descriptors initialization,
+     * supported primitive descriptors selection, etc.), which can be useful after the InitEdges() completes. The second is just inserting the
+     * node without initialization.
+     * @param parent
+     * pointer to the parent node
+     * @param child
+     * pointer to the child node
+     * @param parentPort
+     * port number of the parent node to which the inserted node should be connected
+     * @param childPort
+     * port number of the child node to which the inserted node should be connected
+     * @param initNode
+     * parameter that determines whether the node needs to be initialized
+     * @return true in case of success, false otherwise.
+     */
+    bool InsertNode(MKLDNNNodePtr parent, MKLDNNNodePtr child, MKLDNNNodePtr node, int parentPort, int childPort, bool initNode = false);
+
     InferenceEngine::CNNNetwork dump() const;
 
     template<typename NET>

inference-engine/src/mkldnn_plugin/mkldnn_graph_optimizer.cpp

@@ -55,9 +55,6 @@ void MKLDNNGraphOptimizer::ApplyCommonGraphOptimizations(MKLDNNGraph &graph) {
     MergeTwoEqualScaleShifts(graph);
     graph.RemoveDroppedNodes();
 
-    MergeConversions(graph);
-    graph.RemoveDroppedNodes();
-
     FuseBroadcastAndEltwise(graph);
     graph.RemoveDroppedNodes();
 
@@ -154,51 +151,6 @@ void MKLDNNGraphOptimizer::ApplyImplSpecificGraphOptimizations(MKLDNNGraph &grap
     graph.RemoveDroppedEdges();
 }
 
-void MKLDNNGraphOptimizer::MergeConversions(MKLDNNGraph& graph) {
-    for (auto node : graph.GetNodes()) {
-        // Input with at least 2 Convertions
-        if (!IsOneOf(node->getType(), { Input }) || node->getChildEdges().size() < 2 ||
-            !IsOneOf(node->getChildEdgeAt(0)->getChild()->getType(), { Convert })) {
-            continue;
-        }
-        auto& input = node;
-
-        // Convertions of same the type with Concat as a child
-        for (size_t i = 0; i < input->getChildEdges().size(); i++) {
-            auto convInEdge = input->getChildEdgeAt(i);
-            auto conv = convInEdge->getChild();
-            auto convOutEdge = conv->getChildEdgeAt(i);
-            auto convInDims = convInEdge->getDims();
-            auto convOutDims = convOutEdge->getDims();
-            Precision convOutPrecision = conv->getCnnLayer()->precision;
-
-            for (size_t j = i + 1; j < input->getChildEdges().size();) {
-                auto childEdge = input->getChildEdgeAt(j);
-                auto child = childEdge->getChild();
-
-                if (child->getCnnLayer()->precision != convOutPrecision ||
-                    child->getChildEdgeAt(0)->getDims() != convOutDims ||
-                    childEdge->getDims() != convInDims ||
-                    child->getChildEdges().size() != 1) {
-                    j++;
-                    continue;
-                }
-
-                auto childChildEdge = child->getChildEdgeAt(0);
-                auto childChild = childChildEdge->getChild();
-                int idxChild = childChildEdge->getOutputNum();
-
-                child->remove();
-                graph.DropNode(child);
-
-                MKLDNNEdgePtr newEdge(new MKLDNNEdge(conv, childChild, 0, idxChild));
-                graph.GetEdges().push_back(newEdge);
-                conv->addEdge(newEdge);
-            }
-        }
-    }
-}
-
 void MKLDNNGraphOptimizer::FuseConvolutionAndZeroPoints(MKLDNNGraph &graph) {
     auto& graphNodes = graph.GetNodes();
 
@@ -1844,6 +1796,10 @@ void MKLDNNGraphOptimizer::ChangeConvertToReorder(MKLDNNGraph& graph) {
         if (!InferenceEngine::details::CaselessEq<std::string>()(nodeType, "convert")) {
             continue;
         }
+        if (convertCandidate->getCnnLayer()->insData.empty() ||
+            convertCandidate->getCnnLayer()->outData.empty()) {
+            continue;
+        }
         auto inputPrecision = convertCandidate->getCnnLayer()->insData[0].lock()->getPrecision();
         auto outputPrecision = convertCandidate->getCnnLayer()->outData[0]->getPrecision();
         if (std::find(continuousPrecisions.begin(), continuousPrecisions.end(), inputPrecision) == continuousPrecisions.end() ||
@@ -2313,4 +2269,4 @@ void MKLDNNGraphOptimizer::MergePermuteAndReorder(MKLDNNGraph &graph) {
             mergePermuteAndReorder(parentNode, childNode);
         }
     }
-}
+}

inference-engine/src/mkldnn_plugin/mkldnn_graph_optimizer.h

@@ -19,7 +19,6 @@ class MKLDNNGraphOptimizer {
     void ApplyImplSpecificGraphOptimizations(MKLDNNGraph& graph);
 
 private:
-    void MergeConversions(MKLDNNGraph& graph);
     void MergeGroupConvolution(MKLDNNGraph& graph);
     void MergeTwoEqualScaleShifts(MKLDNNGraph& graph);
     void FuseConvolutionAndActivation(MKLDNNGraph &graph);
@@ -41,6 +40,7 @@ class MKLDNNGraphOptimizer {
     void DropDoubleReorders(MKLDNNGraph& graph);
     void DropConvertReorder(MKLDNNGraph& graph);
     void ChangeConvertToReorder(MKLDNNGraph &graph);
+    void AddConvertToReorder(MKLDNNGraph &graph);
     void FuseConvolutionAndZeroPoints(MKLDNNGraph &graph);
     void FuseBroadcastAndEltwise(MKLDNNGraph &graph);
     void FuseEltwiseAndSimple(MKLDNNGraph &graph);