From 500af6493468fd83f7738c2d51fed5f5d43856fb Mon Sep 17 00:00:00 2001 From: Dmitrii Khurtin Date: Mon, 24 May 2021 14:18:45 +0300 Subject: [PATCH 1/4] fix bug with broadcasting constant layer with fq layer --- .../src/gna_plugin/gna_plugin.cpp | 1 + .../gna_plugin/optimizer/gna_pass_manager.cpp | 53 +++++++---- .../gna_plugin/optimizer/gna_pass_manager.hpp | 5 + .../pass_tests/broadcast_const_with_fq.cpp | 94 +++++++++++++++++++ 4 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp diff --git a/inference-engine/src/gna_plugin/gna_plugin.cpp b/inference-engine/src/gna_plugin/gna_plugin.cpp index f23cbb2f4e188a..fe0714a93d41df 100644 --- a/inference-engine/src/gna_plugin/gna_plugin.cpp +++ b/inference-engine/src/gna_plugin/gna_plugin.cpp @@ -764,6 +764,7 @@ void GNAPlugin::LoadNetwork(CNNNetwork & _network) { passes->registerPass(); passes->registerPass(); passes->registerPass(); + passes->registerPass(); passes->registerPass(); passes->registerPass(); #if GNA_LIB_VER == 2 diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp index 06e72f8ab87428..e7e9fc83543f17 100644 --- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp +++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp @@ -1608,51 +1608,66 @@ void SubstituteScaleShiftBroadCastPass::run() { } } -void BroadcastConstPass::run() { - for (auto& constLayer : *pLayers) { +static void BroadcastConstLayer(const std::vector& layers, bool withFakeQuantize) { + for (auto constLayer : layers) { if (!LayerInfo(constLayer).isConst()) { continue; } - auto isNonFunctional = [](CNNLayerPtr l) { - return LayerInfo(l).isNonFunctional(); + + auto isNonFunctional = [](CNNLayerPtr layer) { + return LayerInfo(layer).isNonFunctional(); }; - if (!CNNNetHasNextLayerSkipCertain(constLayer, 0, 0, isNonFunctional)) { + + auto nextLayer = CNNNetCheckNextLayerSkipCertain(constLayer, 0, 0, true, isNonFunctional).first; + if (!nextLayer || !withFakeQuantize && !LayerInfo(nextLayer).isEltwise()) { continue; } - auto nextLayer = CNNNetGetNextLayerSkipCertain(constLayer, 0, 0, isNonFunctional).first; + auto prevLayer = nextLayer; + if (withFakeQuantize) { + if (!LayerInfo(nextLayer).isFakeQuantize() || CNNNetPrevLayer(nextLayer, 0) != constLayer) { + continue; + } - if (!LayerInfo(nextLayer).isEltwise()) { - continue; + nextLayer = CNNNetCheckNextLayerSkipCertain(nextLayer, 0, 0, true, isNonFunctional).first; + if (!nextLayer || !LayerInfo(nextLayer).isEltwise()) { + continue; + } } auto constDims = constLayer->outData.front()->getTensorDesc().getDims(); auto constDimsSize = product(constDims.begin(), constDims.end()); auto eltwiseDims = nextLayer->outData.front()->getTensorDesc().getDims(); auto eltwiseDimsSize = product(eltwiseDims.begin(), eltwiseDims.end()); - - if (constDimsSize == eltwiseDimsSize) { + if (constDimsSize == eltwiseDimsSize || eltwiseDimsSize % constDimsSize) { continue; } - if (eltwiseDimsSize % constDimsSize) { - continue; - } - - if (constLayer->blobs.find("custom") == constLayer->blobs.end()) { + auto blobsIter = constLayer->blobs.find("custom"); + if (blobsIter == constLayer->blobs.end()) { THROW_GNA_LAYER_EXCEPTION(constLayer) << "Const layer " << constLayer->name << " is missing 'custom' parameter"; } - auto currentConstBlob = constLayer->blobs.find("custom")->second; - - constLayer->blobs.find("custom")->second = tileBlob(currentConstBlob, eltwiseDimsSize); - + auto currentConstBlob = blobsIter->second; + blobsIter->second = tileBlob(currentConstBlob, eltwiseDimsSize); constLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); constLayer->outData.front()->setLayout(nextLayer->outData.front()->getLayout()); + if (prevLayer != nextLayer) { + prevLayer->insData.front().lock()->setDims(nextLayer->outData.front()->getDims()); + prevLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); + } gnalog() << "Const layer '" << constLayer->name << "' was changed to match output of '" << nextLayer->name << "'\n"; } } +void BroadcastConstPass::run() { + BroadcastConstLayer(*pLayers, false); +} + +void BroadcastConstWithFakeQuantizePass::run() { + BroadcastConstLayer(*pLayers, true); +} + void InsertIdentityToLSTMCellPass::run() { for (auto layer : *pLayers) { if (layer->type == "LSTMCell") { diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp index af98288396e8c4..61714afa6fe923 100644 --- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp +++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp @@ -201,6 +201,11 @@ DECL_PASS(FuseMultipleIdentities); */ DECL_PASS(BroadcastConst); +/** +* @brief Brodcast data in Const layer with FakeQuantize layer +*/ +DECL_PASS(BroadcastConstWithFakeQuantize); + /** * @brief runs static quantisation on given floating weights and replaces fakeQuantize with constblobs */ diff --git a/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp b/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp new file mode 100644 index 00000000000000..f80507442d18ae --- /dev/null +++ b/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp @@ -0,0 +1,94 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// +#include +#include +#include + +#include + +#include "common_test_utils/common_utils.hpp" +#include "functional_test_utils/plugin_cache.hpp" +#include "shared_test_classes/base/layer_test_utils.hpp" +#include "functional_test_utils/blob_utils.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" +#include "ngraph_functions/builders.hpp" +#include "ngraph_functions/pass/convert_prc.hpp" + +using BroadcastConstWithFqParamsTuple = typename std::tuple< + InferenceEngine::Precision, // Network Precision + std::vector, // Input shapes for Params Layer + std::vector, // Input shapes for Constant Layer + size_t, // Quantization level + std::map, // Configuration + std::string>; // Device name + +namespace LayerTestsDefinitions { + +class BroadcastConstWithFq : public testing::WithParamInterface, + public LayerTestsUtils::LayerTestsCommon { +public: + static std::string getTestCaseName(testing::TestParamInfo obj) { + InferenceEngine::Precision netPrecision; + std::vector inputShape1; + std::vector inputShape2; + size_t level{0}; + std::map configuration; + std::string targetDevice; + std::tie(netPrecision, inputShape1, inputShape2, level, configuration, targetDevice) = obj.param; + std::ostringstream result; + result << "netPRC=" << netPrecision.name() << "_"; + result << "targetDevice=" << targetDevice << "_"; + for (auto const& configItem : configuration) { + result << "configItem=" << configItem.first << "_" << configItem.second << "_"; + } + result << "inputShape1=" << CommonTestUtils::vec2str(inputShape1) << "_"; + result << "inputShape2=" << CommonTestUtils::vec2str(inputShape2) << "_"; + result << "level=" << level; + return result.str(); + } + +protected: + void SetUp() override { + size_t level{0}; + InferenceEngine::Precision netPrecision; + std::vector inputShape1; + std::vector inputShape2; + std::tie(netPrecision, inputShape1, inputShape2, level, configuration, targetDevice) = this->GetParam(); + auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); + auto params = ngraph::builder::makeParams(ngPrc, {inputShape1}); + auto fakeQuantize1 = ngraph::builder::makeFakeQuantize(params[0], ngPrc, level, {}); + auto constant = ngraph::builder::makeConstant(ngPrc, inputShape2, {}, true); + auto fakeQuantize2 = ngraph::builder::makeFakeQuantize(constant, ngPrc, level, {}); + auto add = std::make_shared(fakeQuantize1, fakeQuantize2); + ngraph::ResultVector results{ std::make_shared(add)}; + function = std::make_shared(results, params, "BroadcastConstWithFq"); + } +}; + +TEST_P(BroadcastConstWithFq, CompareWithRefImpl) { + Run(); +}; + +std::vector> inputShapes1 = { {1, 1, 21, 160} }; +std::vector> inputShapes2 = { {1, 1, 1, 160} }; +const std::vector level = { 65535 }; +const std::vector netPrecisions = {InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16}; +const std::vector> configs = { + { + {"GNA_DEVICE_MODE", "GNA_SW_FP32"}, + } +}; + + +INSTANTIATE_TEST_CASE_P(smoke_broadcast_const_with_fq, BroadcastConstWithFq, + ::testing::Combine( + ::testing::ValuesIn(netPrecisions), + ::testing::ValuesIn(inputShapes1), + ::testing::ValuesIn(inputShapes2), + ::testing::ValuesIn(level), + ::testing::ValuesIn(configs), + ::testing::Values(CommonTestUtils::DEVICE_GNA)), + BroadcastConstWithFq::getTestCaseName); +} // namespace LayerTestsDefinitions From 672ce15b5792db9f7da971b88751397c31bf7f51 Mon Sep 17 00:00:00 2001 From: Dmitrii Khurtin Date: Mon, 24 May 2021 17:47:59 +0300 Subject: [PATCH 2/4] BroadcastConstWithFakeQuantizePass is removed; BroadcastConstPass is moved up in pass list --- inference-engine/src/gna_plugin/gna_plugin.cpp | 3 +-- .../gna_plugin/optimizer/gna_pass_manager.cpp | 18 +++++------------- .../gna_plugin/optimizer/gna_pass_manager.hpp | 5 ----- .../gna/pass_tests/broadcast_const_with_fq.cpp | 10 ++++------ 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/inference-engine/src/gna_plugin/gna_plugin.cpp b/inference-engine/src/gna_plugin/gna_plugin.cpp index fe0714a93d41df..64f7512bb742ff 100644 --- a/inference-engine/src/gna_plugin/gna_plugin.cpp +++ b/inference-engine/src/gna_plugin/gna_plugin.cpp @@ -764,7 +764,7 @@ void GNAPlugin::LoadNetwork(CNNNetwork & _network) { passes->registerPass(); passes->registerPass(); passes->registerPass(); - passes->registerPass(); + passes->registerPass(); passes->registerPass(); passes->registerPass(); #if GNA_LIB_VER == 2 @@ -772,7 +772,6 @@ void GNAPlugin::LoadNetwork(CNNNetwork & _network) { #endif passes->registerPass(); passes->registerPass(); - passes->registerPass(); passIdx = passes->run(passIdx); }; diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp index e7e9fc83543f17..addc2d639e5907 100644 --- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp +++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp @@ -1608,8 +1608,8 @@ void SubstituteScaleShiftBroadCastPass::run() { } } -static void BroadcastConstLayer(const std::vector& layers, bool withFakeQuantize) { - for (auto constLayer : layers) { +void BroadcastConstPass::run() { + for (auto constLayer : *pLayers) { if (!LayerInfo(constLayer).isConst()) { continue; } @@ -1619,13 +1619,13 @@ static void BroadcastConstLayer(const std::vector& }; auto nextLayer = CNNNetCheckNextLayerSkipCertain(constLayer, 0, 0, true, isNonFunctional).first; - if (!nextLayer || !withFakeQuantize && !LayerInfo(nextLayer).isEltwise()) { + if (!nextLayer || !LayerInfo(nextLayer).isEltwise() && !LayerInfo(nextLayer).isFakeQuantize()) { continue; } auto prevLayer = nextLayer; - if (withFakeQuantize) { - if (!LayerInfo(nextLayer).isFakeQuantize() || CNNNetPrevLayer(nextLayer, 0) != constLayer) { + if (LayerInfo(nextLayer).isFakeQuantize()) { + if (CNNNetPrevLayer(nextLayer, 0) != constLayer) { continue; } @@ -1660,14 +1660,6 @@ static void BroadcastConstLayer(const std::vector& } } -void BroadcastConstPass::run() { - BroadcastConstLayer(*pLayers, false); -} - -void BroadcastConstWithFakeQuantizePass::run() { - BroadcastConstLayer(*pLayers, true); -} - void InsertIdentityToLSTMCellPass::run() { for (auto layer : *pLayers) { if (layer->type == "LSTMCell") { diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp index 61714afa6fe923..af98288396e8c4 100644 --- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp +++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp @@ -201,11 +201,6 @@ DECL_PASS(FuseMultipleIdentities); */ DECL_PASS(BroadcastConst); -/** -* @brief Brodcast data in Const layer with FakeQuantize layer -*/ -DECL_PASS(BroadcastConstWithFakeQuantize); - /** * @brief runs static quantisation on given floating weights and replaces fakeQuantize with constblobs */ diff --git a/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp b/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp index f80507442d18ae..7c5f24cf3e84e9 100644 --- a/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp +++ b/inference-engine/tests/functional/plugin/gna/pass_tests/broadcast_const_with_fq.cpp @@ -58,9 +58,9 @@ class BroadcastConstWithFq : public testing::WithParamInterfaceGetParam(); auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); auto params = ngraph::builder::makeParams(ngPrc, {inputShape1}); - auto fakeQuantize1 = ngraph::builder::makeFakeQuantize(params[0], ngPrc, level, {}); + auto fakeQuantize1 = ngraph::builder::makeFakeQuantize(params[0], ngPrc, level, {}, {-0.5}, {0.5}, {-0.5}, {0.5}); auto constant = ngraph::builder::makeConstant(ngPrc, inputShape2, {}, true); - auto fakeQuantize2 = ngraph::builder::makeFakeQuantize(constant, ngPrc, level, {}); + auto fakeQuantize2 = ngraph::builder::makeFakeQuantize(constant, ngPrc, level, {}, {-0.5}, {0.5}, {-0.5}, {0.5}); auto add = std::make_shared(fakeQuantize1, fakeQuantize2); ngraph::ResultVector results{ std::make_shared(add)}; function = std::make_shared(results, params, "BroadcastConstWithFq"); @@ -76,12 +76,10 @@ std::vector> inputShapes2 = { {1, 1, 1, 160} }; const std::vector level = { 65535 }; const std::vector netPrecisions = {InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16}; const std::vector> configs = { - { - {"GNA_DEVICE_MODE", "GNA_SW_FP32"}, - } + { {"GNA_DEVICE_MODE", "GNA_SW_FP32"} }, + { {"GNA_DEVICE_MODE", "GNA_SW_EXACT"} } }; - INSTANTIATE_TEST_CASE_P(smoke_broadcast_const_with_fq, BroadcastConstWithFq, ::testing::Combine( ::testing::ValuesIn(netPrecisions), From 24b446339d80a7b54253fd0b903339e2d2a2405f Mon Sep 17 00:00:00 2001 From: Dmitrii Khurtin Date: Tue, 25 May 2021 11:10:27 +0300 Subject: [PATCH 3/4] constLayer->outData.front()->setDims is moved to conditions --- inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp index addc2d639e5907..64908efd8a2953 100644 --- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp +++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp @@ -1650,11 +1650,12 @@ void BroadcastConstPass::run() { auto currentConstBlob = blobsIter->second; blobsIter->second = tileBlob(currentConstBlob, eltwiseDimsSize); - constLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); constLayer->outData.front()->setLayout(nextLayer->outData.front()->getLayout()); if (prevLayer != nextLayer) { prevLayer->insData.front().lock()->setDims(nextLayer->outData.front()->getDims()); prevLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); + } else { + constLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); } gnalog() << "Const layer '" << constLayer->name << "' was changed to match output of '" << nextLayer->name << "'\n"; } From 190ebf5b180734800d2cf401f0d168e4480f6f50 Mon Sep 17 00:00:00 2001 From: Dmitrii Khurtin Date: Tue, 25 May 2021 11:30:35 +0300 Subject: [PATCH 4/4] prevLayer->outData.front()->setLayout(nextLayer->outData.front()->getLayout()); is added --- .../src/gna_plugin/optimizer/gna_pass_manager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp index 64908efd8a2953..daed41e5343c18 100644 --- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp +++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp @@ -1650,12 +1650,11 @@ void BroadcastConstPass::run() { auto currentConstBlob = blobsIter->second; blobsIter->second = tileBlob(currentConstBlob, eltwiseDimsSize); + constLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); constLayer->outData.front()->setLayout(nextLayer->outData.front()->getLayout()); if (prevLayer != nextLayer) { - prevLayer->insData.front().lock()->setDims(nextLayer->outData.front()->getDims()); prevLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); - } else { - constLayer->outData.front()->setDims(nextLayer->outData.front()->getDims()); + prevLayer->outData.front()->setLayout(nextLayer->outData.front()->getLayout()); } gnalog() << "Const layer '" << constLayer->name << "' was changed to match output of '" << nextLayer->name << "'\n"; }