diff --git a/inference-engine/src/inference_engine/cpp_interfaces/interface/ie_iinfer_request_internal.cpp b/inference-engine/src/inference_engine/cpp_interfaces/interface/ie_iinfer_request_internal.cpp index fc77b834b699d9..800b5f3cc01956 100644 --- a/inference-engine/src/inference_engine/cpp_interfaces/interface/ie_iinfer_request_internal.cpp +++ b/inference-engine/src/inference_engine/cpp_interfaces/interface/ie_iinfer_request_internal.cpp @@ -215,16 +215,18 @@ bool IInferRequestInternal::findInputAndOutputBlobByName(const std::string& name [&](const std::pair& pair) { return pair.first == name; }); - if (foundOutputPair == std::end(_networkOutputs) && (foundInputPair == std::end(_networkInputs))) { - IE_THROW(NotFound) << "Failed to find input or output with name: \'" << name << "\'"; - } + bool retVal; + if (foundInputPair != std::end(_networkInputs)) { foundInput = foundInputPair->second; - return true; - } else { + retVal = true; + } else if (foundOutputPair != std::end(_networkOutputs)) { foundOutput = foundOutputPair->second; - return false; + retVal = false; + } else { + IE_THROW(NotFound) << "Failed to find input or output with name: \'" << name << "\'"; } + return retVal; } void IInferRequestInternal::checkBlob(const Blob::Ptr& blob, const std::string& name, bool isInput, const SizeVector& refDims) const { diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp index 197373005c5055..86820c88ce405a 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp @@ -738,7 +738,7 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine:: } } -void MKLDNNGraph::PullOutputData(BlobMap &out) { +void MKLDNNGraph::PullOutputData(const BlobMap &out) { if (!IsReady()) IE_THROW() << "Wrong state. Topology not ready."; @@ -746,23 +746,13 @@ void MKLDNNGraph::PullOutputData(BlobMap &out) { auto name = outputMap.first; auto node = outputMap.second; const MKLDNNMemory& intr_blob = node->getParentEdgeAt(0)->getMemory(); - if (out.find(name) == out.end()) { - // TODO [NM]: Do we really need this path? - // TODO: Create blob from MemoryDesc - Blob::Ptr outBlob = make_shared_blob({Precision::FP32, node->getParentEdgeAt(0)->getDims().ToSizeVector(), - TensorDesc::getLayoutByDims(node->getParentEdgeAt(0)->getDims().ToSizeVector())}, - reinterpret_cast(intr_blob.GetData())); - out[name] = outBlob; - } - - Blob::Ptr &ext_blob = out[name]; - // TODO: Why we allow allocation of output memory inside Infer call?? - // Suggestion is to disable this behaviour - if (ext_blob->buffer() == nullptr) { - ext_blob->allocate(); + if (!out.count(name)) { + IE_THROW(Unexpected) << "The network outputs do not contain mkldnn graph output node name: \"" << name << "\""; } + const Blob::Ptr &ext_blob = out.at(name); + auto srcPrec = MKLDNNExtensionUtils::DataTypeToIEPrecision(intr_blob.GetDataType()); auto dstPrec = ext_blob->getTensorDesc().getPrecision(); if (srcPrec == dstPrec && ext_blob->byteSize() != intr_blob.GetSize()) diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.h b/inference-engine/src/mkldnn_plugin/mkldnn_graph.h index b572558cda4c46..822cdeb387b6a6 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.h +++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.h @@ -55,7 +55,7 @@ class MKLDNNGraph { } void PushInputData(const std::string& name, const InferenceEngine::Blob::Ptr &in); - void PullOutputData(InferenceEngine::BlobMap &out); + void PullOutputData(const InferenceEngine::BlobMap &out); void Infer(MKLDNNInferRequest* request = nullptr, int batch = -1); @@ -79,6 +79,14 @@ class MKLDNNGraph { return outputNodesMap; } + bool hasInputWithName(const std::string& name) const { + return inputNodesMap.count(name); + } + + bool hasOutputWithName(const std::string& name) const { + return outputNodesMap.count(name); + } + mkldnn::engine getEngine() const { return eng; } diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp index ca1dd9ed5891bd..2496ea27fb6913 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp @@ -34,6 +34,8 @@ MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsData if (execNetwork->_graphs.size() == 0) IE_THROW() << "No graph was found"; graph = &(execNetwork->GetGraph()._graph); + + // Allocate all input blobs for (const auto& it : _networkInputs) { MKLDNNInferRequest::GetBlob(it.first); } @@ -210,10 +212,9 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std:: InferenceEngine::Blob::Ptr data; - InferenceEngine::BlobMap blobs; - graph->getInputBlobs(blobs); - - if (blobs.find(name) != blobs.end()) { + if (graph->hasInputWithName(name)) { + InferenceEngine::BlobMap blobs; + graph->getInputBlobs(blobs); // ROI blob is returned only if it was set previously. auto it = _preProcData.find(name); if (it != _preProcData.end()) { @@ -221,60 +222,77 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std:: return data; } - if (_inputs.find(name) != _inputs.end()) { - data = _inputs[name]; - checkBlob(data, name, true); - return data; - } + if (_inputs.find(name) == _inputs.end()) { + InferenceEngine::TensorDesc desc = blobs[name]->getTensorDesc(); - InferenceEngine::TensorDesc desc = blobs[name]->getTensorDesc(); - if (_networkInputs.find(name) != _networkInputs.end()) { - InferenceEngine::Layout l = _networkInputs[name]->getLayout(); - InferenceEngine::Precision p = _networkInputs[name]->getPrecision(); - InferenceEngine::SizeVector dims = _networkInputs[name]->getTensorDesc().getDims(); + if (_networkInputs.find(name) != _networkInputs.end()) { + InferenceEngine::Layout l = _networkInputs[name]->getLayout(); + InferenceEngine::Precision p = _networkInputs[name]->getPrecision(); + InferenceEngine::SizeVector dims = _networkInputs[name]->getTensorDesc().getDims(); - desc = InferenceEngine::TensorDesc(p, dims, l); - } + desc = InferenceEngine::TensorDesc(p, dims, l); + } - _inputs[name] = make_blob_with_precision(desc); - _inputs[name]->allocate(); - if (blobs[name]->getTensorDesc() == desc && + _inputs[name] = make_blob_with_precision(desc); + _inputs[name]->allocate(); + if (blobs[name]->getTensorDesc() == desc && graph->_meanImages.find(name) == graph->_meanImages.end() && !graph->getProperty().batchLimit) { - externalPtr[name] = _inputs[name]->buffer(); + externalPtr[name] = _inputs[name]->buffer(); + } } data = _inputs[name]; checkBlob(data, name, true); - return data; } - blobs.clear(); - graph->getOutputBlobs(blobs); - if (blobs.find(name) != blobs.end()) { - if (_outputs.find(name) != _outputs.end()) { - data = _outputs[name]; - checkBlob(data, name, false); - return data; - } - InferenceEngine::TensorDesc desc = _networkOutputs[name]->getTensorDesc(); - desc.setPrecision(normalizeToSupportedPrecision(desc.getPrecision())); + if (graph->hasOutputWithName(name)) { + InferenceEngine::BlobMap blobs; + graph->getOutputBlobs(blobs); + if (_outputs.find(name) == _outputs.end()) { + if (!data) { + InferenceEngine::TensorDesc desc = _networkOutputs[name]->getTensorDesc(); + desc.setPrecision(normalizeToSupportedPrecision(desc.getPrecision())); + + // WA: need to avoid exception thrown when we compare blocking desc in SetBlob + // in situation if we push output blobs as inputs for next network (in Hetero plugin) + // it may be that output tensor desc will be different from real input tensor desc for next network + // because the optimal descriptor was chosen (e.g. inPlace case for Split node) + auto currBlockDesc = InferenceEngine::BlockingDesc(desc.getBlockingDesc().getBlockDims(), desc.getBlockingDesc().getOrder()); + desc = InferenceEngine::TensorDesc(desc.getPrecision(), desc.getDims(), currBlockDesc); + + data = make_blob_with_precision(desc); + data->allocate(); + } else { + const auto& expectedTensorDesc = blobs[name]->getTensorDesc(); + + if (expectedTensorDesc.getPrecision() != data->getTensorDesc().getPrecision()) { + IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different precision: " + << data->getTensorDesc().getPrecision() << " for input and " << expectedTensorDesc.getPrecision() + << " for output."; + } + + if (expectedTensorDesc.getDims() != data->getTensorDesc().getDims()) { + IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different shapes."; + } - // WA: need to avoid exception thrown when we compare blocking desc in SetBlob - // in situation if we push output blobs as inputs for next network (in Hetero plugin) - // it may be that output tensor desc will be different from real input tensor desc for next network - // because the optimal descriptor was chosen (e.g. inPlace case for Split node) - auto currBlockDesc = InferenceEngine::BlockingDesc(desc.getBlockingDesc().getBlockDims(), desc.getBlockingDesc().getOrder()); - desc = InferenceEngine::TensorDesc(desc.getPrecision(), desc.getDims(), currBlockDesc); + if (data->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY && expectedTensorDesc.getLayout() != InferenceEngine::Layout::ANY && + expectedTensorDesc.getBlockingDesc() != data->getTensorDesc().getBlockingDesc()) { + IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name + << " but expect blobs with different blocking descriptors."; + } + } - _outputs[name] = make_blob_with_precision(desc); - _outputs[name]->allocate(); - if (blobs[name]->getTensorDesc() == desc && !graph->getProperty().batchLimit) { - externalPtr[name] = _outputs[name]->buffer(); + _outputs[name] = data; + if (!externalPtr.count(name) && data->getTensorDesc() == blobs[name]->getTensorDesc() && !graph->getProperty().batchLimit) { + externalPtr[name] = data->buffer(); + } } data = _outputs[name]; checkBlob(data, name, false); - return data; } - IE_THROW() << "Cannot find blob with name: " << name; + if (!data) { + IE_THROW() << "Cannot find blob with name: " << name; + } + return data; } void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) { @@ -295,7 +313,9 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In InferenceEngine::InputInfo::Ptr foundInput; InferenceEngine::DataPtr foundOutput; size_t dataSize = data->size(); - if (findInputAndOutputBlobByName(name, foundInput, foundOutput)) { + findInputAndOutputBlobByName(name, foundInput, foundOutput); + + if (foundInput) { if (foundInput->getPrecision() != data->getTensorDesc().getPrecision()) { IE_THROW(ParameterMismatch) << "Failed to set input blob with precision: " << data->getTensorDesc().getPrecision() << ", if CNNNetwork input blob precision is: " << foundInput->getPrecision(); @@ -346,7 +366,8 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In } _inputs[name] = data; } - } else { + } + if (foundOutput) { if (compoundBlobPassed) { IE_THROW(NotImplemented) << "cannot set compound blob: supported only for input pre-processing"; diff --git a/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp index 7c8d65b7686c21..0c409cc2639381 100644 --- a/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp +++ b/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp @@ -208,7 +208,6 @@ class INFERENCE_ENGINE_API_CLASS(IInferRequestInternal) : public std::enable_sha * @param foundOutput A pointer to output DataPtr if found. * @return `True` - if loaded network has input with provided name, * `false` - if loaded network has output with provided name - * @throws [parameter_mismatch] exception if input and output has the same name * @throws [not_found] exception if there is no input and output layers with given name */ bool findInputAndOutputBlobByName(const std::string& name, InputInfo::Ptr& foundInput, DataPtr& foundOutput) const; diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/parameter_result.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/parameter_result.cpp new file mode 100644 index 00000000000000..7eb8265150c599 --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/subgraph_tests/parameter_result.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "subgraph_tests/parameter_result.hpp" +#include "common_test_utils/test_constants.hpp" + +using namespace SubgraphTestsDefinitions; + +namespace { + INSTANTIATE_TEST_CASE_P(smoke_Check, ParameterResultSubgraphTest, + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ParameterResultSubgraphTest::getTestCaseName); +} // namespace diff --git a/inference-engine/tests/functional/plugin/cpu/subgraph_tests/src/param_result_custom_blob.cpp b/inference-engine/tests/functional/plugin/cpu/subgraph_tests/src/param_result_custom_blob.cpp new file mode 100644 index 00000000000000..65ac992ec76ac1 --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/subgraph_tests/src/param_result_custom_blob.cpp @@ -0,0 +1,88 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shared_test_classes/subgraph/parameter_result.hpp" +#include "common_test_utils/test_constants.hpp" + +using namespace SubgraphTestsDefinitions; +using namespace InferenceEngine; + +namespace CPULayerTestsDefinitions { + +class ParameterResultCustomBlobTest : public ParameterResultSubgraphTest { + protected: + void Infer() override { + constexpr size_t inferIterations = 10lu; + + inferRequest = executableNetwork.CreateInferRequest(); + + auto inputBlob = inputs.front(); + const size_t elementsCount = inputBlob->size(); + for (size_t i = 0; i < inferIterations; ++i) { + CommonTestUtils::fill_data_random(inputBlob, 10, 0, 1, i); + const auto& inputsInfo = cnnNetwork.getInputsInfo().begin()->second; + std::string inputName = cnnNetwork.getInputsInfo().begin()->first; + + float* customInpData = new float[elementsCount]; + auto inpBlobData = inputBlob->buffer().as(); + std::copy(inpBlobData, inpBlobData + elementsCount, customInpData); + + auto& tensorDesc = inputsInfo->getTensorDesc(); + auto customBlob = make_shared_blob(tensorDesc, customInpData, elementsCount * sizeof(float)); + inferRequest.SetBlob(inputName, customBlob); + + inferRequest.Infer(); + + ParameterResultSubgraphTest::Validate(); + + delete[] customInpData; + } + } + void Validate() override { + //Do nothing. We call Validate() in the Infer() method + } +}; + +TEST_P(ParameterResultCustomBlobTest, CompareWithRefs) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + // Just to show that it is not possible to set different precisions for inputs and outputs with the same name. + // If it was possible, the input would have I8 precision and couldn't store data from the custom blob. + inPrc = Precision::I8; + outPrc = Precision::FP32; + + Run(); +} +namespace { + INSTANTIATE_TEST_CASE_P(smoke_Check_Custom_Blob, ParameterResultCustomBlobTest, + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ParameterResultSubgraphTest::getTestCaseName); +} // namespace + +class ParameterResultSameBlobTest : public ParameterResultSubgraphTest { +protected: + void Infer() override { + constexpr size_t inferIterations = 10lu; + + for (size_t i = 0; i < inferIterations; ++i) { + ParameterResultSubgraphTest::Infer(); + ParameterResultSubgraphTest::Validate(); + } + } + void Validate() override { + //Do nothing. We call Validate() in the Infer() method + } +}; + +TEST_P(ParameterResultSameBlobTest, CompareWithRefs) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + Run(); +} +namespace { + INSTANTIATE_TEST_CASE_P(smoke_Check_Same_Blob, ParameterResultSameBlobTest, + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ParameterResultSubgraphTest::getTestCaseName); +} // namespace +} // namespace CPULayerTestsDefinitions \ No newline at end of file