Skip to content

Commit

Permalink
[CPU] {GRM, LSTM, RNN} Cells and Sequences migration on nGraph. (#55)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nshchego
nshchego authored and mandrono committed Apr 30, 2021
1 parent 4e226fc commit 01af7f2
Show file tree
Hide file tree
Showing 13 changed files with 579 additions and 233 deletions.
4 changes: 2 additions & 2 deletions inference-engine/src/mkldnn_plugin/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ set(LAYERS
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_fake_quantize_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_reorder_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_reshape_node.cpp
# ${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_rnn.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_rnn.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_roi_align_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_roi_pooling_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_softmax_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_split_node.cpp
# ${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_tensoriterator_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_tensoriterator_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_tile_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_mvn_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/mkldnn_normalize_node.cpp
Expand Down
181 changes: 83 additions & 98 deletions inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,107 +91,92 @@ void MKLDNNGraph::CreateGraph(NET &net, const MKLDNNExtensionManager::Ptr& extMg
status = Ready;
}

//template void MKLDNNGraph::CreateGraph(const TensorIterator::Body&,
// const MKLDNNExtensionManager::Ptr&, MKLDNNWeightsSharing::Ptr&);
template void MKLDNNGraph::CreateGraph(const std::shared_ptr<const ngraph::Function>&,
const MKLDNNExtensionManager::Ptr&, MKLDNNWeightsSharing::Ptr&);
template void MKLDNNGraph::CreateGraph(const CNNNetwork&,
const MKLDNNExtensionManager::Ptr&, MKLDNNWeightsSharing::Ptr&);
//template void MKLDNNGraph::CreateGraph(CNNNetwork&,
// const MKLDNNExtensionManager::Ptr&, MKLDNNWeightsSharing::Ptr&);

//void MKLDNNGraph::Replicate(const TensorIterator::Body &subgraph, const MKLDNNExtensionManager::Ptr& extMgr) {
// this->_name = "subgraph";
// this->reuse_io_tensors = false;
//
// // Map data object onto producer layer(node)
// std::unordered_map<Data*, std::pair<MKLDNNNodePtr, int>> data2node;
//
// // nodes which has no consumers (output or just unused). But doesn't marked as graph output.
// // Will be stored as fake output separately.
// std::unordered_set<DataPtr> unused_data;
//
// // Step 1. Replicate input nodes
// for (const auto &input : subgraph.inputs) {
// if (input->getPrecision() == Precision::UNSPECIFIED) continue; // const node holder
//
// auto creator = getCreatorLayer(input).lock();
// if (creator == nullptr) {
// creator.reset(new CNNLayer({input->getName(), "Input", input->getTensorDesc().getPrecision()}));
// creator->outData.push_back(input);
// }
//
// const MKLDNNNodePtr node(MKLDNNNode::factory().create(creator, getEngine(), extMgr, weightsCache));
// data2node[input.get()] = {node, 0};
//
// graphNodes.push_back(node);
// inputNodesMap[input->getName()] = node;
//
// if (getInputTo(input).empty()) {
// unused_data.insert(input);
// }
// }
//
// // Step 2. Replicate all internal nodes.
// for (const auto layer : NetPass::TIBodySortTopologically(subgraph)) {
// const MKLDNNNodePtr node {MKLDNNNode::factory().create(layer, getEngine(), extMgr, weightsCache)};
// graphNodes.push_back(node);
//
// for (int port = 0; port < layer->insData.size(); port++) {
// auto data = layer->insData[port].lock();
//
// auto port_info = data2node[data.get()];
// auto parent_node = port_info.first;
// auto parent_port_idx = port_info.second;
//
// MKLDNNEdgePtr edge(new MKLDNNEdge(parent_node, node, parent_port_idx, port));
// node->addEdge(edge);
// graphEdges.push_back(edge);
// }
// int out_port_idx = 0;
// for (auto &out_data : layer->outData) {
// data2node[out_data.get()] = {node, out_port_idx++};
// if (getInputTo(out_data).empty()) {
// unused_data.insert(out_data);
// }
// }
// }
//
// // Step 3. Add output nodes and output stubs for unused data objects.
// for (const auto &output : subgraph.outputs) {
// auto port_info = data2node[output.get()];
// auto parent_node = port_info.first;
// auto parent_port_idx = port_info.second;
//
// CNNLayerPtr layer(new CNNLayer({"out_" + output->getName(), "Output", output->getTensorDesc().getPrecision()}));
// layer->insData.push_back(output);
//
// const MKLDNNNodePtr node {MKLDNNNode::factory().create(layer, getEngine(), extMgr, weightsCache)};
//
// MKLDNNEdgePtr edge(new MKLDNNEdge(parent_node, node, parent_port_idx, 0));
// node->addEdge(edge);
// graphEdges.push_back(edge);
// graphNodes.push_back(node);
// outputNodesMap.push_back(node);
//
// unused_data.erase(output);
// }
//
// // Add stub output node for unused data
// for (auto to_stub_data : unused_data) {
// auto port_info = data2node[to_stub_data.get()];
// auto parent_node = port_info.first;
// auto parent_port_idx = port_info.second;
//
// CNNLayerPtr layer(new CNNLayer({"stub_" + to_stub_data->getName(), "Output", to_stub_data->getTensorDesc().getPrecision()}));
// layer->insData.push_back(to_stub_data);
//
// const MKLDNNNodePtr node(MKLDNNNode::factory().create(layer, getEngine(), extMgr, weightsCache));
//
// MKLDNNEdgePtr edge(new MKLDNNEdge(parent_node, node, parent_port_idx, 0));
// node->addEdge(edge);
// graphEdges.push_back(edge);
// graphNodes.push_back(node);
// }
//}
void MKLDNNGraph::Replicate(const std::shared_ptr<const ngraph::Function> &subgraph, const MKLDNNExtensionManager::Ptr& extMgr) {
this->_name = "subgraph";
this->reuse_io_tensors = false;

// Map data object onto producer node
std::map<std::shared_ptr<ngraph::Node>, std::pair<MKLDNNNodePtr, int>> op2node;

// nodes which has no consumers (output or just unused). But doesn't marked as graph output.
// Will be stored as fake output separately.
std::deque<ngraph::Output<ngraph::Node>> unusedOutputs;

auto getParentOutputPort = [](const std::shared_ptr<ngraph::Node> childOp, const std::shared_ptr<ngraph::Node> parentOp,
const size_t childInputPort) -> int {
for (size_t parentPort = 0; parentPort < parentOp->get_output_size(); parentPort++) {
if (childOp->input(childInputPort).get_tensor_ptr() == parentOp->output(parentPort).get_tensor_ptr()) {
return static_cast<int>(parentPort);
}
}

return -1;
};

for (const auto op : subgraph->get_ordered_ops()) {
const MKLDNNNodePtr node {MKLDNNNode::factory().create(op, getEngine(), extMgr, weightsCache)};
graphNodes.push_back(node);

if (op->get_type_info() == ngraph::op::v0::Parameter::type_info) {
inputNodesMap[node->getName()] = node;
}

if (op->get_type_info() == ngraph::op::v0::Result::type_info) {
auto prev = op->get_input_node_shared_ptr(0);
std::string inputID;
inputID = prev->get_friendly_name();
if (prev->get_output_size() > 1) {
inputID += "." + std::to_string(op->get_input_source_output(0).get_index());
}

outputNodesMap[inputID] = node;
}

for (size_t port = 0; port < op->get_input_size(); port++) {
auto parentOp = op->get_input_node_shared_ptr(port);

auto portInfo = op2node[parentOp];
auto parentNode = portInfo.first;

MKLDNNEdgePtr edge(new MKLDNNEdge(parentNode, node, getParentOutputPort(op, parentOp, port), port));
node->addEdge(edge);
graphEdges.push_back(edge);
}

if (!MKLDNNPlugin::one_of(op->get_type_info(),
ngraph::op::v0::Result::type_info,
ngraph::op::v3::Assign::type_info,
ngraph::op::v6::Assign::type_info)) {
int outPortIdx = 0;
for (int oi = 0; oi < op->get_output_size(); oi++) {
op2node[op->output(oi).get_node_shared_ptr()] = {node, outPortIdx++};
if (op->get_output_target_inputs(oi).empty()) {
unusedOutputs.push_back(op->output(oi));
}
}
}
}

// Add stub output node for unused data
for (auto unusedOutput : unusedOutputs) {
auto portInfo = op2node[unusedOutput.get_node_shared_ptr()];
auto parentNode = portInfo.first;
auto port = portInfo.second;
const auto nodeName = std::string("stub_") + std::to_string(unusedOutput.get_index()) + "_" + parentNode->getName();
const MKLDNNNodePtr outNode = std::make_shared<MKLDNNInputNode>(parentNode->outDims[port].ToSizeVector(),
parentNode->getOriginalOutputPrecisionAtPort(port),
nodeName, "Result", getEngine(), weightsCache);
MKLDNNEdgePtr edge(new MKLDNNEdge(parentNode, outNode, port, 0));
outNode->addEdge(edge);
graphEdges.push_back(edge);
graphNodes.push_back(outNode);
}
}

void MKLDNNGraph::Replicate(const CNNNetwork &network, const MKLDNNExtensionManager::Ptr& extMgr) {
OV_ITT_SCOPE_CHAIN(FIRST_INFERENCE, taskChain, itt::domains::MKLDNN_LT, "MKLDNNGraph::Replicate", "CNNNetwork");
Expand Down
4 changes: 1 addition & 3 deletions inference-engine/src/mkldnn_plugin/mkldnn_graph.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,12 @@

#pragma once

#include "ie_parallel.hpp"
#include "cpp/ie_cnn_network.h"
#include "config.h"
#include "mkldnn_memory.h"
#include "mean_image.h"
#include "mkldnn_node.h"
#include "mkldnn_edge.h"
#include "threading/ie_thread_local.hpp"
#include <map>
#include <string>
#include <vector>
Expand Down Expand Up @@ -190,7 +188,7 @@ class MKLDNNGraph {
static mkldnn::engine eng;

void Replicate(const InferenceEngine::CNNNetwork &network, const MKLDNNExtensionManager::Ptr& extMgr);
// void Replicate(const InferenceEngine::TensorIterator::Body &subgraph, const MKLDNNExtensionManager::Ptr& extMgr);
void Replicate(const std::shared_ptr<const ngraph::Function> &subgraph, const MKLDNNExtensionManager::Ptr& extMgr);
void InitGraph();
void InitNodes();
void InitDescriptors();
Expand Down
57 changes: 28 additions & 29 deletions inference-engine/src/mkldnn_plugin/mkldnn_node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,17 +161,17 @@ static const InferenceEngine::details::caseless_unordered_map<std::string, Type>
{ "Pad", Pad },
{ "Transpose", Transpose },
// { "Copy", Copy },
// { "LSTMCell", RNNCell },
// { "GRUCell", RNNCell },
// { "RNNCell", RNNCell },
// { "LSTMSequence", RNNSeq },
// { "GRUSequence", RNNSeq },
// { "RNNSequence", RNNSeq },
{ "LSTMCell", RNNCell },
{ "GRUCell", RNNCell },
{ "RNNCell", RNNCell },
{ "LSTMSequence", RNNSeq },
{ "GRUSequence", RNNSeq },
{ "RNNSequence", RNNSeq },
{ "FakeQuantize", FakeQuantize },
{ "BinaryConvolution", BinaryConvolution },
{ "DeformableConvolution", DeformableConvolution },
// { "TensorIterator", TensorIterator },
// { "Loop", TensorIterator },
{ "TensorIterator", TensorIterator },
{ "Loop", TensorIterator },
{ "ReadValue", MemoryInput}, // for construction from name ctor, arbitrary name is used
{ "Assign", MemoryOutput }, // for construction from layer ctor
{ "Convert", Convert },
Expand Down Expand Up @@ -1293,27 +1293,26 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr<ngraph::Node>
}

// TODO [NM]: enable after all nodes will be migrated on ngraph
// if (newNode == nullptr) {
// try {
// std::unique_ptr<MKLDNNNode> ol(new MKLDNNReferenceNode(op, eng, w_cache, errorMessage));
// if (ol != nullptr && ol->created(extMgr))
// newNode = ol.release();
// } catch (const InferenceEngine::Exception& ex) {
// if (ex.getStatus() != NOT_IMPLEMENTED) {
// throw;
// } else {
// errorMessage += getExceptionDescWithoutStatus(ex);
// }
// }
// }

// TODO [NM]: Not implemented
// // WA-start : TI node requires all attributes to construct internal subgpath
// // including extManager, socket and mkldnn::eng.
// MKLDNNTensorIteratorNode *ti = dynamic_cast<MKLDNNTensorIteratorNode*>(newNode);
// if (ti != nullptr)
// ti->setExtManager(extMgr);
// // WA-end
// if (newNode == nullptr) {
// try {
// std::unique_ptr<MKLDNNNode> ol(new MKLDNNReferenceNode(op, eng, w_cache, errorMessage));
// if (ol != nullptr && ol->created(extMgr))
// newNode = ol.release();
// } catch (const InferenceEngine::Exception& ex) {
// if (ex.getStatus() != NOT_IMPLEMENTED) {
// throw;
// } else {
// errorMessage += getExceptionDescWithoutStatus(ex);
// }
// }
// }

// WA-start : TI node requires all attributes to construct internal subgpath
// including extManager, socket and mkldnn::eng.
MKLDNNTensorIteratorNode *ti = dynamic_cast<MKLDNNTensorIteratorNode*>(newNode);
if (ti != nullptr)
ti->setExtManager(extMgr);
// WA-end

if (!newNode) {
std::string errorDetails;
Expand Down
1 change: 0 additions & 1 deletion inference-engine/src/mkldnn_plugin/mkldnn_node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
#include <cassert>
#include <algorithm>
#include <caseless.hpp>
#include <ie_common.h>
#include "mkldnn_dims.h"
#include "mkldnn_memory.h"
#include "mkldnn_edge.h"
Expand Down
4 changes: 4 additions & 0 deletions inference-engine/src/mkldnn_plugin/ngraph_transformations/convert_to_cpu_specific_opset.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "convert_to_leaky_relu.hpp"
#include "convert_to_swish_cpu.hpp"
#include "reshape_prelu.hpp"
#include "rnn_sequences_optimization.hpp"

namespace MKLDNNPlugin {

Expand All @@ -34,6 +35,9 @@ inline void ConvertToCPUSpecificOpset(std::shared_ptr<ngraph::Function> &nGraphF
manager.register_pass<ConvertToLeakyRelu>();
manager.register_pass<ReshapePRelu>();
manager.register_pass<ConvertToSwishCPU>();
manager.register_pass<OptimizeGRUSequenceTransposes>();
manager.register_pass<OptimizeLSTMSequenceTransposes>();
manager.register_pass<OptimizeRNNSequenceTransposes>();
if (!ngraph::op::util::has_op_with_type<ngraph::op::FakeQuantize>(nGraphFunc)) {
manager.register_pass<ReshapeFullyConnectedFusion>();
}
Expand Down
Loading

0 comments on commit 01af7f2

Please sign in to comment.