Improve reshapeability of models with eltwise nodes influencing shapes (

openvinotoolkit#2767) * Fix ElementwiseInputReshape transformation Reshape node always needs to be inserted in order to preserve ShapeOf nodes (reshapability of a model) that can potentially be above elementwise node. Refactor EltwiseInputReshape_test and EltwiseInputNormalization_test since the logic of maintaining reshape for eltwise has been changed. Signed-off-by: Roman Kazantsev <[email protected]> * Merge EltwiseInputNormalization and EltwiseInputReshape transformations Signed-off-by: Roman Kazantsev <[email protected]> * Remove Unsqueeze from Fused_op Signed-off-by: Roman Kazantsev <[email protected]> * Fix code after code review #1 Signed-off-by: Roman Kazantsev <[email protected]> * Fix code after review #2 Signed-off-by: Roman Kazantsev <[email protected]> * Fix code review #4 Signed-off-by: Roman Kazantsev <[email protected]> * Perform full normalization based on shapes of all inputs to eltwise Signed-off-by: Roman Kazantsev <[email protected]> * Refactor much to avoid old API and edges with unsqueeze_dims attribute Signed-off-by: Roman Kazantsev <[email protected]> * Fix code after review Signed-off-by: Roman Kazantsev <[email protected]>
eshoguli · Nov 16, 2020 · 10b18a0 · 10b18a0
1 parent 2a7f2f5
commit 10b18a0
Show file tree

Hide file tree

Showing 21 changed files with 527 additions and 490 deletions.
diff --git a/inference-engine/src/legacy_api/src/ie_cnn_layer_builder_ngraph.cpp b/inference-engine/src/legacy_api/src/ie_cnn_layer_builder_ngraph.cpp
@@ -651,11 +651,11 @@ CNNLayer::Ptr NodeConverter<ngraph::op::Squeeze>::createLayer(const std::shared_
 }
 
 template <>
-CNNLayer::Ptr NodeConverter<ngraph::op::Unsqueeze>::createLayer(const std::shared_ptr<ngraph::Node>& layer) const {
+CNNLayer::Ptr NodeConverter<ngraph::op::v0::Unsqueeze>::createLayer(const std::shared_ptr<ngraph::Node>& layer) const {
     LayerParams params = {layer->get_friendly_name(), "Unsqueeze",
                           details::convertPrecision(layer->get_output_element_type(0))};
     auto res = std::make_shared<InferenceEngine::CNNLayer>(params);
-    auto castedLayer = ngraph::as_type_ptr<ngraph::op::Unsqueeze>(layer);
+    auto castedLayer = ngraph::as_type_ptr<ngraph::op::v0::Unsqueeze>(layer);
     if (castedLayer == nullptr) THROW_IE_EXCEPTION << "Cannot get " << params.type << " layer " << params.name;
 
     return res;

diff --git a/...ine/src/legacy_api/src/transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.cpp b/...ine/src/legacy_api/src/transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.cpp
@@ -52,22 +52,22 @@ ngraph::pass::ConvertNMSToNMSIEMatcher::ConvertNMSToNMSIEMatcher() {
             if (auto new_max_per_class_const = std::dynamic_pointer_cast<opset1::Constant>(new_max_per_class.get_node_shared_ptr())) {
                 new_max_per_class = opset1::Constant::create(element::i64, Shape{1}, new_max_per_class_const->cast_vector<int64_t>());
             } else {
-                new_max_per_class = std::make_shared<ngraph::op::Unsqueeze>(
+                new_max_per_class = std::make_shared<ngraph::op::v0::Unsqueeze>(
                         nms->input_value(2),
                         opset1::Constant::create(element::i64, Shape{1}, {0}));
                 new_ops.push_back(new_max_per_class.get_node_shared_ptr());
             }
         }
         auto new_iou_threshold = nms->input_value(3);
         if (iou_threshold_rank.get_length() == 0) {
-            new_iou_threshold = std::make_shared<ngraph::op::Unsqueeze>(
+            new_iou_threshold = std::make_shared<ngraph::op::v0::Unsqueeze>(
                     nms->input_value(3),
                     opset1::Constant::create(element::i64, Shape{1}, {0}));
             new_ops.push_back(new_iou_threshold.get_node_shared_ptr());
         }
         auto new_score_threshold = nms->input_value(4);
         if (score_threshold_rank.get_length() == 0) {
-            new_score_threshold = std::make_shared<ngraph::op::Unsqueeze>(
+            new_score_threshold = std::make_shared<ngraph::op::v0::Unsqueeze>(
                     nms->input_value(4),
                     opset1::Constant::create(element::i64, Shape{1}, {0}));
             new_ops.push_back(new_score_threshold.get_node_shared_ptr());

diff --git a/inference-engine/tests/functional/plugin/shared/src/subgraph_tests/matmul_squeeze_add.cpp b/inference-engine/tests/functional/plugin/shared/src/subgraph_tests/matmul_squeeze_add.cpp
@@ -58,7 +58,7 @@ void MatmulSqueezeAddTest::SetUp() {
     auto matmul_0 = std::make_shared<ngraph::op::MatMul>(params[0], constant_0, false, true);
 
     auto constant_1 = std::make_shared<ngraph::op::Constant>(ngraph::element::Type_t::i64, ngraph::Shape{ 1 }, std::vector<size_t>{0});
-    auto unsqueeze_0 = std::make_shared<ngraph::op::Unsqueeze>(matmul_0, constant_1);
+    auto unsqueeze_0 = std::make_shared<ngraph::op::v0::Unsqueeze>(matmul_0, constant_1);
 
     auto constant_2 = ngraph::builder::makeConstant<float>(ngPrc, { 1, inputShape[0], outputSize },
         CommonTestUtils::generate_float_numbers(inputShape[0] * outputSize, 0, 1, seed), false);

diff --git a/inference-engine/tests/functional/plugin/shared/src/subgraph_tests/memory_LSTMCell.cpp b/inference-engine/tests/functional/plugin/shared/src/subgraph_tests/memory_LSTMCell.cpp
@@ -75,7 +75,7 @@ namespace SubgraphTestsDefinitions {
         auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);
 
         auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-        auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
+        auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);
 
         auto permute_in_params = std::make_shared<ngraph::opset1::Constant>(ngraph::element::i64, ngraph::Shape{3}, ngraph::Shape{{1, 0, 2}});
         auto permute_in = std::make_shared<ngraph::opset1::Transpose>(unsqueeze_input, permute_in_params);
@@ -100,7 +100,7 @@ namespace SubgraphTestsDefinitions {
         auto lstm = std::make_shared<ngraph::opset4::LSTMCell>(squeeze, H_t, C_t, weightsNode, reccurrenceWeightsNode, biasNode, hiddenSize);
 
         auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-        auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
+        auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
         // body - outputs
         auto H_o = lstm->output(0);
         auto C_o = lstm->output(1);
@@ -158,7 +158,7 @@ namespace SubgraphTestsDefinitions {
         auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);
 
         auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-        auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
+        auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);
 
         auto cell_memory_constant = ngraph::builder::makeConstant<float>(ngPrc, cell_memory_dims, cell_memory_init);
 
@@ -175,7 +175,7 @@ namespace SubgraphTestsDefinitions {
                                                                reccurrenceWeightsNode, biasNode, hiddenSize);
 
         auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-        auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
+        auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
 
         auto final_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
                                                                             ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));

diff --git a/inference-engine/tests/functional/plugin/shared/src/subgraph_tests/multiple_LSTMCell.cpp b/inference-engine/tests/functional/plugin/shared/src/subgraph_tests/multiple_LSTMCell.cpp
@@ -73,7 +73,7 @@ void MultipleLSTMCellTest::SetUp() {
     auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);
 
     auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-    auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
+    auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);
 
     auto permute_in_params = std::make_shared<ngraph::opset1::Constant>(ngraph::element::i64, ngraph::Shape{3}, ngraph::Shape{{1, 0, 2}});
     auto permute_in = std::make_shared<ngraph::opset1::Transpose>(unsqueeze_input, permute_in_params);
@@ -100,7 +100,7 @@ void MultipleLSTMCellTest::SetUp() {
     auto lstm = std::make_shared<ngraph::opset4::LSTMCell>(squeeze, H_t, C_t, weightsNode, reccurrenceWeightsNode, biasNode, hiddenSize);
 
     auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-    auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
+    auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
     // body - outputs
     auto H_o = lstm->output(0);
     auto C_o = lstm->output(1);
@@ -155,7 +155,7 @@ void MultipleLSTMCellTest::SetUp() {
     auto lstm_2 = std::make_shared<ngraph::opset4::LSTMCell>(squeeze_2, H_t_2, C_t_2, weightsNode_2, reccurrenceWeightsNode_2, biasNode_2, hiddenSize);
 
     auto unsqueeze_2_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-    auto unsqueeze_2 = std::make_shared<ngraph::op::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
+    auto unsqueeze_2 = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
     // body - outputs
     auto H_o_2 = lstm_2->output(0);
     auto C_o_2 = lstm_2->output(1);
@@ -219,7 +219,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
     auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);
 
     auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-    auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
+    auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);
 
     // Body 1 - layers
     auto cell_memory_constant = ngraph::builder::makeConstant<float>(ngPrc, cell_memory_dims, cell_memory_init);
@@ -236,7 +236,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
                                                            reccurrenceWeightsNode, biasNode, hiddenSize);
 
     auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-    auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
+    auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
 
     auto first_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
                                                                         ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
@@ -261,7 +261,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
         reccurrenceWeightsNode_2, biasNode_2, hiddenSize);
 
     auto unsqueeze_2_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
-    auto unsqueeze_2 = std::make_shared<ngraph::op::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
+    auto unsqueeze_2 = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
 
     auto final_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
         ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));

diff --git a/...ce-engine/tests/ngraph_functions/src/low_precision_transformations/unsqueeze_function.cpp b/...ce-engine/tests/ngraph_functions/src/low_precision_transformations/unsqueeze_function.cpp
@@ -67,7 +67,7 @@ std::shared_ptr<ngraph::Function> UnsqueezeFunction::getReference(
 
     const std::shared_ptr<Node> dequantizationOpBefore = makeDequantization(input, dequantizationBefore);
     const auto unsqueeze = std::make_shared<ngraph::op::TypeRelaxed<ngraph::opset1::Unsqueeze>>(
-        op::Unsqueeze(dequantizationOpBefore, std::make_shared<ngraph::opset1::Constant>(element::i64, Shape{ axes.size() }, axes)),
+        op::v0::Unsqueeze(dequantizationOpBefore, std::make_shared<ngraph::opset1::Constant>(element::i64, Shape{ axes.size() }, axes)),
         precisionAfterOperation);
     const std::shared_ptr<Node> dequantizationOpAfter = makeDequantization(unsqueeze, dequantizationAfter);
     dequantizationOpAfter->set_friendly_name("output");

diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt
@@ -526,7 +526,6 @@ extensions/middle/DeleteControlFlowEdges.py
 extensions/middle/DeleteNotExecutable.py
 extensions/middle/DilatedConvolution.py
 extensions/middle/EltwiseChecker.py
-extensions/middle/EltwiseInputNormalization.py
 extensions/middle/EltwiseInputReshape.py
 extensions/middle/FakeSplitOutputs.py
 extensions/middle/FusedBatchNormNonConstant.py

diff --git a/model-optimizer/extensions/middle/EltwiseInputNormalization.py b/model-optimizer/extensions/middle/EltwiseInputNormalization.py