Skip to content

Commit

Permalink
Improve reshapeability of models with eltwise nodes influencing shapes (
Browse files Browse the repository at this point in the history 
#2767)

* Fix ElementwiseInputReshape transformation

Reshape node always needs to be inserted
in order to preserve ShapeOf nodes (reshapability of a model) that can potentially be above
elementwise node.

Refactor EltwiseInputReshape_test and EltwiseInputNormalization_test since the logic of maintaining reshape for eltwise has been changed.

Signed-off-by: Roman Kazantsev <[email protected]>

* Merge EltwiseInputNormalization and EltwiseInputReshape transformations

Signed-off-by: Roman Kazantsev <[email protected]>

* Remove Unsqueeze from Fused_op

Signed-off-by: Roman Kazantsev <[email protected]>

* Fix code after code review #1

Signed-off-by: Roman Kazantsev <[email protected]>

* Fix code after review #2

Signed-off-by: Roman Kazantsev <[email protected]>

* Fix code review #4

Signed-off-by: Roman Kazantsev <[email protected]>

* Perform full normalization based on shapes of all inputs to eltwise

Signed-off-by: Roman Kazantsev <[email protected]>

* Refactor much to avoid old API and edges with unsqueeze_dims attribute

Signed-off-by: Roman Kazantsev <[email protected]>

* Fix code after review

Signed-off-by: Roman Kazantsev <[email protected]>
  • Loading branch information
@rkazants
rkazants authored Nov 16, 2020
1 parent 2a7f2f5 commit 10b18a0
Show file tree
Hide file tree
Showing 21 changed files with 527 additions and 490 deletions.
4 changes: 2 additions & 2 deletions inference-engine/src/legacy_api/src/ie_cnn_layer_builder_ngraph.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,11 +651,11 @@ CNNLayer::Ptr NodeConverter<ngraph::op::Squeeze>::createLayer(const std::shared_
}

template <>
CNNLayer::Ptr NodeConverter<ngraph::op::Unsqueeze>::createLayer(const std::shared_ptr<ngraph::Node>& layer) const {
CNNLayer::Ptr NodeConverter<ngraph::op::v0::Unsqueeze>::createLayer(const std::shared_ptr<ngraph::Node>& layer) const {
LayerParams params = {layer->get_friendly_name(), "Unsqueeze",
details::convertPrecision(layer->get_output_element_type(0))};
auto res = std::make_shared<InferenceEngine::CNNLayer>(params);
auto castedLayer = ngraph::as_type_ptr<ngraph::op::Unsqueeze>(layer);
auto castedLayer = ngraph::as_type_ptr<ngraph::op::v0::Unsqueeze>(layer);
if (castedLayer == nullptr) THROW_IE_EXCEPTION << "Cannot get " << params.type << " layer " << params.name;

return res;
Expand Down
6 changes: 3 additions & 3 deletions ...ine/src/legacy_api/src/transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,22 @@ ngraph::pass::ConvertNMSToNMSIEMatcher::ConvertNMSToNMSIEMatcher() {
if (auto new_max_per_class_const = std::dynamic_pointer_cast<opset1::Constant>(new_max_per_class.get_node_shared_ptr())) {
new_max_per_class = opset1::Constant::create(element::i64, Shape{1}, new_max_per_class_const->cast_vector<int64_t>());
} else {
new_max_per_class = std::make_shared<ngraph::op::Unsqueeze>(
new_max_per_class = std::make_shared<ngraph::op::v0::Unsqueeze>(
nms->input_value(2),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_max_per_class.get_node_shared_ptr());
}
}
auto new_iou_threshold = nms->input_value(3);
if (iou_threshold_rank.get_length() == 0) {
new_iou_threshold = std::make_shared<ngraph::op::Unsqueeze>(
new_iou_threshold = std::make_shared<ngraph::op::v0::Unsqueeze>(
nms->input_value(3),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_iou_threshold.get_node_shared_ptr());
}
auto new_score_threshold = nms->input_value(4);
if (score_threshold_rank.get_length() == 0) {
new_score_threshold = std::make_shared<ngraph::op::Unsqueeze>(
new_score_threshold = std::make_shared<ngraph::op::v0::Unsqueeze>(
nms->input_value(4),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_score_threshold.get_node_shared_ptr());
Expand Down
2 changes: 1 addition & 1 deletion inference-engine/tests/functional/plugin/shared/src/subgraph_tests/matmul_squeeze_add.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ void MatmulSqueezeAddTest::SetUp() {
auto matmul_0 = std::make_shared<ngraph::op::MatMul>(params[0], constant_0, false, true);

auto constant_1 = std::make_shared<ngraph::op::Constant>(ngraph::element::Type_t::i64, ngraph::Shape{ 1 }, std::vector<size_t>{0});
auto unsqueeze_0 = std::make_shared<ngraph::op::Unsqueeze>(matmul_0, constant_1);
auto unsqueeze_0 = std::make_shared<ngraph::op::v0::Unsqueeze>(matmul_0, constant_1);

auto constant_2 = ngraph::builder::makeConstant<float>(ngPrc, { 1, inputShape[0], outputSize },
CommonTestUtils::generate_float_numbers(inputShape[0] * outputSize, 0, 1, seed), false);
Expand Down
8 changes: 4 additions & 4 deletions inference-engine/tests/functional/plugin/shared/src/subgraph_tests/memory_LSTMCell.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ namespace SubgraphTestsDefinitions {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

auto permute_in_params = std::make_shared<ngraph::opset1::Constant>(ngraph::element::i64, ngraph::Shape{3}, ngraph::Shape{{1, 0, 2}});
auto permute_in = std::make_shared<ngraph::opset1::Transpose>(unsqueeze_input, permute_in_params);
Expand All @@ -100,7 +100,7 @@ namespace SubgraphTestsDefinitions {
auto lstm = std::make_shared<ngraph::opset4::LSTMCell>(squeeze, H_t, C_t, weightsNode, reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
// body - outputs
auto H_o = lstm->output(0);
auto C_o = lstm->output(1);
Expand Down Expand Up @@ -158,7 +158,7 @@ namespace SubgraphTestsDefinitions {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

auto cell_memory_constant = ngraph::builder::makeConstant<float>(ngPrc, cell_memory_dims, cell_memory_init);

Expand All @@ -175,7 +175,7 @@ namespace SubgraphTestsDefinitions {
reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);

auto final_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
Expand Down
12 changes: 6 additions & 6 deletions inference-engine/tests/functional/plugin/shared/src/subgraph_tests/multiple_LSTMCell.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ void MultipleLSTMCellTest::SetUp() {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

auto permute_in_params = std::make_shared<ngraph::opset1::Constant>(ngraph::element::i64, ngraph::Shape{3}, ngraph::Shape{{1, 0, 2}});
auto permute_in = std::make_shared<ngraph::opset1::Transpose>(unsqueeze_input, permute_in_params);
Expand All @@ -100,7 +100,7 @@ void MultipleLSTMCellTest::SetUp() {
auto lstm = std::make_shared<ngraph::opset4::LSTMCell>(squeeze, H_t, C_t, weightsNode, reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);
// body - outputs
auto H_o = lstm->output(0);
auto C_o = lstm->output(1);
Expand Down Expand Up @@ -155,7 +155,7 @@ void MultipleLSTMCellTest::SetUp() {
auto lstm_2 = std::make_shared<ngraph::opset4::LSTMCell>(squeeze_2, H_t_2, C_t_2, weightsNode_2, reccurrenceWeightsNode_2, biasNode_2, hiddenSize);

auto unsqueeze_2_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_2 = std::make_shared<ngraph::op::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
auto unsqueeze_2 = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
// body - outputs
auto H_o_2 = lstm_2->output(0);
auto C_o_2 = lstm_2->output(1);
Expand Down Expand Up @@ -219,7 +219,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
auto mul = ngraph::builder::makeEltwise(add, input_mul_const, ngraph::helpers::EltwiseTypes::MULTIPLY);

auto unsqueeze_input_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_input = std::make_shared<ngraph::op::Unsqueeze>(mul, unsqueeze_input_const);
auto unsqueeze_input = std::make_shared<ngraph::op::v0::Unsqueeze>(mul, unsqueeze_input_const);

// Body 1 - layers
auto cell_memory_constant = ngraph::builder::makeConstant<float>(ngPrc, cell_memory_dims, cell_memory_init);
Expand All @@ -236,7 +236,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
reccurrenceWeightsNode, biasNode, hiddenSize);

auto unsqueeze_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze = std::make_shared<ngraph::op::Unsqueeze>(lstm->output(0), unsqueeze_const);
auto unsqueeze = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm->output(0), unsqueeze_const);

auto first_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
Expand All @@ -261,7 +261,7 @@ void MultipleLSTMCellTest::switchToNgraphFriendlyModel() {
reccurrenceWeightsNode_2, biasNode_2, hiddenSize);

auto unsqueeze_2_const = std::make_shared<ngraph::op::Constant>(ngraph::element::i64, ngraph::Shape{1}, squeeze_axes);
auto unsqueeze_2 = std::make_shared<ngraph::op::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);
auto unsqueeze_2 = std::make_shared<ngraph::op::v0::Unsqueeze>(lstm_2->output(0), unsqueeze_2_const);

auto final_reshape_pattern = std::make_shared<ngraph::op::Constant>(ngraph::element::i64,
ngraph::Shape{4}, std::vector<size_t>({1, 1, 1, hiddenSize}));
Expand Down
2 changes: 1 addition & 1 deletion ...ce-engine/tests/ngraph_functions/src/low_precision_transformations/unsqueeze_function.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ std::shared_ptr<ngraph::Function> UnsqueezeFunction::getReference(

const std::shared_ptr<Node> dequantizationOpBefore = makeDequantization(input, dequantizationBefore);
const auto unsqueeze = std::make_shared<ngraph::op::TypeRelaxed<ngraph::opset1::Unsqueeze>>(
op::Unsqueeze(dequantizationOpBefore, std::make_shared<ngraph::opset1::Constant>(element::i64, Shape{ axes.size() }, axes)),
op::v0::Unsqueeze(dequantizationOpBefore, std::make_shared<ngraph::opset1::Constant>(element::i64, Shape{ axes.size() }, axes)),
precisionAfterOperation);
const std::shared_ptr<Node> dequantizationOpAfter = makeDequantization(unsqueeze, dequantizationAfter);
dequantizationOpAfter->set_friendly_name("output");
Expand Down
1 change: 0 additions & 1 deletion model-optimizer/automation/package_BOM.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,6 @@ extensions/middle/DeleteControlFlowEdges.py
extensions/middle/DeleteNotExecutable.py
extensions/middle/DilatedConvolution.py
extensions/middle/EltwiseChecker.py
extensions/middle/EltwiseInputNormalization.py
extensions/middle/EltwiseInputReshape.py
extensions/middle/FakeSplitOutputs.py
extensions/middle/FusedBatchNormNonConstant.py
Expand Down
48 changes: 0 additions & 48 deletions model-optimizer/extensions/middle/EltwiseInputNormalization.py
View file

This file was deleted.

Loading

0 comments on commit 10b18a0

Please sign in to comment.