From 822dd0780a46373d1a011199bd3d5cbe8d4f287d Mon Sep 17 00:00:00 2001 From: Nikita Demashov Date: Wed, 23 Mar 2022 03:04:33 +0300 Subject: [PATCH] added RNNCell node support --- .../src/lstm.cpp | 13 ++- src/plugins/intel_cpu/src/graph_dumper.cpp | 39 ++------ .../lstm_transformation.cpp | 96 ++++++++++++++++--- .../lpt_ngraph_functions/lstm_function.hpp | 3 +- .../src/lstm_function.cpp | 77 ++++++++------- 5 files changed, 149 insertions(+), 79 deletions(-) diff --git a/src/common/low_precision_transformations/src/lstm.cpp b/src/common/low_precision_transformations/src/lstm.cpp index 903d6e24760c58..fb905ddfc288f7 100644 --- a/src/common/low_precision_transformations/src/lstm.cpp +++ b/src/common/low_precision_transformations/src/lstm.cpp @@ -93,6 +93,12 @@ LSTMTransformation::LSTMTransformation(const Params& params) : LayerTransformati const auto gru_cell_with_dequantizations_without_subtract = ngraph::pattern::wrap_type( {dequantization_multiply_without_subtract_X, dequantization_multiply_without_subtract_H, fq_W, fq_R, B}); + const auto rnn_cell = ngraph::pattern::wrap_type({fq_X, fq_H, fq_W, fq_R, B}); + const auto rnn_cell_with_dequantizations = ngraph::pattern::wrap_type( + {dequantization_multiply_X, dequantization_multiply_X, fq_W, fq_R, B}); + const auto rnn_cell_with_dequantizations_without_subtract = ngraph::pattern::wrap_type( + {dequantization_multiply_without_subtract_X, dequantization_multiply_without_subtract_H, fq_W, fq_R, B}); + ngraph::graph_rewrite_callback callback = [this](pattern::Matcher& m) { auto op = m.get_match_root(); if (transformation_callback(op)) { @@ -111,7 +117,10 @@ LSTMTransformation::LSTMTransformation(const Params& params) : LayerTransformati lstm_sequence_with_dequantizations_without_subtract, gru_cell, gru_cell_with_dequantizations, - gru_cell_with_dequantizations_without_subtract}), + gru_cell_with_dequantizations_without_subtract, + rnn_cell, + rnn_cell_with_dequantizations, + rnn_cell_with_dequantizations_without_subtract}), "LSTM"); this->register_matcher(m, callback); } @@ -177,7 +186,7 @@ bool LSTMTransformation::isPrecisionPreserved(std::shared_ptr) const noexc return true; } -void LSTMTransformation::propagateSkipCleanupAttribute(std::shared_ptr multiply){ +void LSTMTransformation::propagateSkipCleanupAttribute(std::shared_ptr multiply) { SkipCleanupAttribute::create(multiply, true); auto multiply_parent = multiply->get_input_node_shared_ptr(0); SkipCleanupAttribute::create(multiply_parent, true); diff --git a/src/plugins/intel_cpu/src/graph_dumper.cpp b/src/plugins/intel_cpu/src/graph_dumper.cpp index 6679b288563c07..84de0b35a9e946 100644 --- a/src/plugins/intel_cpu/src/graph_dumper.cpp +++ b/src/plugins/intel_cpu/src/graph_dumper.cpp @@ -4,18 +4,14 @@ #include "graph_dumper.h" +#include "utils/debug_capabilities.h" +#include #include "exec_graph_info.hpp" #include "ie_common.h" -#include "ie_ngraph_utils.hpp" #include "mkldnn_debug.h" -#include -#include "cpu_types.h" -#include "utils/debug_capabilities.h" - #include #include "ngraph/ngraph.hpp" #include -#include "ngraph/op/tensor_iterator.hpp" #include #include @@ -43,19 +39,7 @@ std::map extract_node_metadata(const NodePtr &node) { // Path to print actual name for extension layers serialization_info[ExecGraphInfoSerialization::LAYER_TYPE] = node->getTypeStr(); } else { - std::string layerTypeStr; - - auto layerType = node->getType(); - - /* replace CPU proprietary input/output types with the ones which serializer can process */ - if (layerType == Type::Input) - layerTypeStr = "Parameter"; - else if (layerType == Type::Output) - layerTypeStr = "Result"; - else - layerTypeStr = NameFromType(node->getType()); - - serialization_info[ExecGraphInfoSerialization::LAYER_TYPE] = layerTypeStr; + serialization_info[ExecGraphInfoSerialization::LAYER_TYPE] = NameFromType(node->getType()); } // Original layers @@ -189,16 +173,12 @@ std::shared_ptr dump_graph_as_ie_ngraph_net(const Graph &graph results.emplace_back(std::make_shared(get_inputs(node).back())); return_node = results.back(); } else { - if (node->getAlgorithm() == Algorithm::TensorIteratorCommon) { - return_node = create_ngraph_ti_node(node); - } else { - return_node = std::make_shared( - get_inputs(node), node->getSelectedPrimitiveDescriptor()->getConfig().outConfs.size()); - - for (size_t port = 0; port < return_node->get_output_size(); ++port) { - auto& desc = node->getChildEdgeAt(port)->getMemory().getDesc(); - return_node->set_output_type(port, details::convertPrecision(desc.getPrecision()), desc.getShape().toPartialShape()); - } + return_node = std::make_shared( + get_inputs(node), node->getSelectedPrimitiveDescriptor()->getConfig().outConfs.size()); + + for (size_t port = 0; port < return_node->get_output_size(); ++port) { + auto& desc = node->getChildEdgeAt(port)->getMemory().getDesc(); + return_node->set_output_type(port, details::convertPrecision(desc.getPrecision()), desc.getShape().toPartialShape()); } } @@ -213,6 +193,7 @@ std::shared_ptr dump_graph_as_ie_ngraph_net(const Graph &graph return return_node; }; + ngraph::NodeVector nodes; nodes.reserve(graph.graphNodes.size()); for (auto &node : graph.graphNodes) { // important: graph.graphNodes are in topological order nodes.emplace_back(create_ngraph_node(node)); diff --git a/src/tests/functional/inference_engine/lp_transformations/lstm_transformation.cpp b/src/tests/functional/inference_engine/lp_transformations/lstm_transformation.cpp index 91696c5f377f80..9124f16b4b033b 100644 --- a/src/tests/functional/inference_engine/lp_transformations/lstm_transformation.cpp +++ b/src/tests/functional/inference_engine/lp_transformations/lstm_transformation.cpp @@ -62,14 +62,12 @@ class LSTMTransformationTestValues { public: LSTMTransformationTestValues() = default; LSTMTransformationTestValues(const TestTransformationParams& params, - const bool bias, const LSTMFunction::RNNType type, const LSTMTransformationValues& actual, const LSTMTransformationValues& result, const bool addNotPrecisionPreservedOperation = false, const bool checkIntervalsAlignmentAttributes = true) : params(params), - bias(bias), type(type), actual(actual), result(result), @@ -77,7 +75,6 @@ class LSTMTransformationTestValues { checkIntervalsAlignmentAttributes(checkIntervalsAlignmentAttributes) {} TestTransformationParams params; - bool bias; LSTMFunction::RNNType type; LSTMTransformationValues actual; LSTMTransformationValues result; @@ -88,7 +85,7 @@ class LSTMTransformationTestValues { }; inline std::ostream& operator<<(std::ostream& out, const LSTMTransformationTestValues& values) { - return out << "_" << values.bias << "_" << values.actual << "_" << values.result; + return out << "_" << values.actual << "_" << values.result; } typedef std::tuple, std::vector, LSTMTransformationTestValues> @@ -106,7 +103,6 @@ class LSTMTransformation : public LayerTransformation, public testing::WithParam activations_shapes, weights_shapes, testValues.type, - testValues.bias, { testValues.actual.fakeQuantize_X, testValues.actual.fakeQuantize_H, @@ -162,7 +158,6 @@ class LSTMTransformation : public LayerTransformation, public testing::WithParam activations_shapes, weights_shapes, testValues.type, - testValues.bias, { testValues.result.fakeQuantize_X, testValues.result.fakeQuantize_H, @@ -195,8 +190,6 @@ class LSTMTransformation : public LayerTransformation, public testing::WithParam std::ostringstream result; result << LayerTransformation::getTestCaseNameByParams(precision, activations_shapes[0], testValues.params) - << "_" - << (testValues.bias ? "with_bias_" : "without_bias_") << "_" << testValues.actual << "_" << testValues.result << "_"; return result.str(); } @@ -235,7 +228,6 @@ const std::vector> weights_shapes = {{{512, 16}, {512 const std::vector testValues = { // LSTM Cell {LayerTransformation::createParamsU8I8(), - false, LSTMFunction::RNNType::LSTMCell, { // X @@ -318,7 +310,6 @@ const std::vector> weights_shapes = {{{1, 512, 16}, { const std::vector testValues = { // LSTM Sequence {LayerTransformation::createParamsU8I8(), - false, LSTMFunction::RNNType::LSTMSequence, { // X @@ -401,7 +392,6 @@ const std::vector> weights_shapes = {{{9, 3}, {9, 3}, const std::vector testValues = { // GRU {LayerTransformation::createParamsU8I8(), - false, LSTMFunction::RNNType::GRU, { // X @@ -454,7 +444,7 @@ const std::vector testValues = { {}, {0.01f} }, - // R + // R {}, {}, { @@ -475,4 +465,86 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(testValues)), LSTMTransformation::getTestCaseName); } // namespace testValues3 + +namespace testValues4 { +const std::vector> activations_shapes = {{{2, 3}, {2, 3}, {}}}; + +const std::vector> weights_shapes = {{{3, 3}, {3, 3}, {9}}}; + +const std::vector testValues = { + // RNNCell + {LayerTransformation::createParamsU8I8(), + LSTMFunction::RNNType::RNNCell, + { + // X + {256ul, {}, {0.f}, {2.55f}, {0.f}, {255.f}}, + {ngraph::element::u8}, + { + {element::f32}, + {}, + {0.01f}, + }, + // H + {256ul, {}, {0.f}, {2.55f}, {0.f}, {255.f}}, + {ngraph::element::u8}, + { + {element::f32}, + {}, + {0.01f}, + }, + // W + {255ul, {}, {0.f}, {2.55f}, {0.f}, {2.55f}}, + {}, + {{}, {}, {}}, + // R + {255ul, {}, {0.f}, {2.55f}, {0.f}, {2.55f}}, + {}, + {{}, {}, {}}, + }, + { + // X + {256ul, {}, {0.f}, {2.55f}, {0.f}, {255.f}}, + {ngraph::element::u8}, + { + {element::f32}, + {}, + {0.01f}, + }, + // H + {256ul, {}, {0.f}, {2.55f}, {0.f}, {255.f}}, + {ngraph::element::u8}, + { + {element::f32}, + {}, + {0.01f}, + }, + // W + {}, + {}, + { + {element::f32}, + {}, + {0.01f} + }, + // R + {}, + {}, + { + {element::f32}, + {}, + {0.01f} + }, + } + } +}; +INSTANTIATE_TEST_SUITE_P( + smoke_LPT, + LSTMTransformation, + ::testing::Combine( + ::testing::ValuesIn(precisions), + ::testing::ValuesIn(activations_shapes), + ::testing::ValuesIn(weights_shapes), + ::testing::ValuesIn(testValues)), + LSTMTransformation::getTestCaseName); +} // namespace testValues4 } // namespace diff --git a/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/lstm_function.hpp b/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/lstm_function.hpp index 1354f13482a618..8119a0bfcf9102 100644 --- a/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/lstm_function.hpp +++ b/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/lstm_function.hpp @@ -17,14 +17,13 @@ namespace subgraph { class LSTMFunction { public: - enum class RNNType { LSTMCell, LSTMSequence, GRU }; + enum class RNNType { LSTMCell, LSTMSequence, GRU, RNNCell }; static std::shared_ptr get( const ngraph::element::Type inputPrecision, const std::vector& inputActivationsShapes, const std::vector& inputWeightsShapes, const RNNType type, - const bool bias, const std::vector& fqOnDatas, const std::vector& converts, const std::vector& dequantizations, diff --git a/src/tests/ngraph_helpers/lpt_ngraph_functions/src/lstm_function.cpp b/src/tests/ngraph_helpers/lpt_ngraph_functions/src/lstm_function.cpp index 962e724a32c5d2..9fd318fa105a26 100644 --- a/src/tests/ngraph_helpers/lpt_ngraph_functions/src/lstm_function.cpp +++ b/src/tests/ngraph_helpers/lpt_ngraph_functions/src/lstm_function.cpp @@ -28,7 +28,6 @@ std::shared_ptr LSTMFunction::get( const std::vector& inputActivationsShapes, const std::vector& inputWeightsShapes, const RNNType type, - const bool bias, const std::vector& fqOnDatas, const std::vector& converts, const std::vector& dequantizations, @@ -75,55 +74,65 @@ std::shared_ptr LSTMFunction::get( auto B = ngraph::opset5::Constant::create(inputPrecision, inputWeightsShapes[2], {1}); auto seq_lengths = ngraph::opset5::Constant::create(element::i32, Shape{1}, {3}); - std::shared_ptr rnn_cell; + std::shared_ptr rnn_layer; switch (type) { case RNNType::LSTMCell: - rnn_cell = std::make_shared(parent_X, + rnn_layer = std::make_shared(parent_X, + parent_H, + C, + parent_W, + parent_R, + 128); + rnn_layer->set_friendly_name("lstm_cell"); + break; + case RNNType::LSTMSequence: + rnn_layer = std::make_shared(parent_X, + parent_H, + C, + seq_lengths, + parent_W, + parent_R, + B, + 128, + op::RecurrentSequenceDirection::FORWARD); + rnn_layer->set_friendly_name("lstm_sequense"); + break; + case RNNType::GRU: + rnn_layer = std::make_shared(parent_X, parent_H, - C, parent_W, parent_R, - 128); - rnn_cell->set_friendly_name("lstm_cell"); + 3); + rnn_layer->set_friendly_name("gru_cell"); break; - case RNNType::LSTMSequence: - rnn_cell = std::make_shared(parent_X, - parent_H, - C, - seq_lengths, - parent_W, - parent_R, - B, - 128, - op::RecurrentSequenceDirection::FORWARD); - rnn_cell->set_friendly_name("lstm_sequense"); + case RNNType::RNNCell: + rnn_layer = std::make_shared(parent_X, + parent_H, + parent_W, + parent_R, + 3); + rnn_layer->set_friendly_name("rnn_layer"); break; - case RNNType::GRU: - rnn_cell = std::make_shared(parent_X, - parent_H, - parent_W, - parent_R, - 3); - rnn_cell->set_friendly_name("gru_cell"); default: break; } - auto& rtInfo = rnn_cell->get_rt_info(); - rtInfo["Variant::std::string"] = "rnn_cell"; + auto& rtInfo = rnn_layer->get_rt_info(); + bool is_lstm = type == RNNType::LSTMCell || type == RNNType::LSTMSequence; + rtInfo["Variant::std::string"] = "rnn_layer"; - auto rnn_cell_res_1 = std::make_shared(rnn_cell->output(0)); - rnn_cell_res_1->set_friendly_name("output_1"); - std::shared_ptr rnn_cell_res_2 = {}; - if (type != RNNType::GRU) { - rnn_cell_res_2 = std::make_shared(rnn_cell->output(1)); - rnn_cell_res_2->set_friendly_name("output_2"); + auto rnn_layer_res_1 = std::make_shared(rnn_layer->output(0)); + rnn_layer_res_1->set_friendly_name("output_1"); + std::shared_ptr rnn_layer_res_2 = {}; + if (is_lstm) { + rnn_layer_res_2 = std::make_shared(rnn_layer->output(1)); + rnn_layer_res_2->set_friendly_name("output_2"); } - ngraph::ResultVector results{rnn_cell_res_2 ? rnn_cell_res_1, rnn_cell_res_2 : rnn_cell_res_1}; + ngraph::ResultVector results{rnn_layer_res_2 ? rnn_layer_res_1, rnn_layer_res_2 : rnn_layer_res_1}; std::shared_ptr function = std::make_shared( results, - type != RNNType::GRU ? ngraph::ParameterVector{X, H, C} : ngraph::ParameterVector{X, H}, + is_lstm ? ngraph::ParameterVector{X, H, C} : ngraph::ParameterVector{X, H}, "LSTMTransformation"); return function;