diff --git a/inference-engine/include/cpp/ie_executable_network.hpp b/inference-engine/include/cpp/ie_executable_network.hpp index 0430ce17d64192..15c1ecbedd5e91 100644 --- a/inference-engine/include/cpp/ie_executable_network.hpp +++ b/inference-engine/include/cpp/ie_executable_network.hpp @@ -29,7 +29,10 @@ class IExecutableNetwork; */ class INFERENCE_ENGINE_API_CLASS(ExecutableNetwork) : protected details::SOPointer { using details::SOPointer::SOPointer; - friend class InferencePlugin; + // TODO: remove? + ExecutableNetwork(const details::SOPointer & obj) : + details::SOPointer::SOPointer(obj) { } + friend class Core; public: /** diff --git a/inference-engine/src/auto_plugin/auto_exec_network.cpp b/inference-engine/src/auto_plugin/auto_exec_network.cpp index 0d691214835701..04b5253d7098d2 100644 --- a/inference-engine/src/auto_plugin/auto_exec_network.cpp +++ b/inference-engine/src/auto_plugin/auto_exec_network.cpp @@ -13,11 +13,12 @@ #include "auto_infer_request.hpp" namespace AutoPlugin { - using namespace InferenceEngine; -AutoExecutableNetwork::AutoExecutableNetwork(const ExecutableNetwork& network, - const DeviceInformation& deviceInfo, - const bool needPerfCounters) : +using namespace InferenceEngine; + +AutoExecutableNetwork::AutoExecutableNetwork(const SoExecutableNetworkInternal& network, + const DeviceInformation& deviceInfo, + const bool needPerfCounters) : _deviceInfo(deviceInfo), _network(network), _config(deviceInfo.config.begin(), deviceInfo.config.end()), @@ -28,32 +29,32 @@ AutoExecutableNetwork::~AutoExecutableNetwork() = default; IInferRequestInternal::Ptr AutoExecutableNetwork::CreateInferRequestImpl(InputsDataMap networkInputs, OutputsDataMap networkOutputs) { - auto inferRequest = _network.CreateInferRequest(); + auto inferRequest = _network->CreateInferRequest(); return std::make_shared(networkInputs, networkOutputs, inferRequest); } void AutoExecutableNetwork::Export(std::ostream& networkModel) { - _network.Export(networkModel); + _network->Export(networkModel); } RemoteContext::Ptr AutoExecutableNetwork::GetContext() const { - return _network.GetContext(); + return _network->GetContext(); } InferenceEngine::CNNNetwork AutoExecutableNetwork::GetExecGraphInfo() { - return _network.GetExecGraphInfo(); + return _network->GetExecGraphInfo(); } Parameter AutoExecutableNetwork::GetMetric(const std::string &name) const { - return _network.GetMetric(name); + return _network->GetMetric(name); } void AutoExecutableNetwork::SetConfig(const std::map& config) { - _network.SetConfig(config); + _network->SetConfig(config); } Parameter AutoExecutableNetwork::GetConfig(const std::string& name) const { - return _network.GetConfig(name); + return _network->GetConfig(name); } } // namespace AutoPlugin diff --git a/inference-engine/src/auto_plugin/auto_exec_network.hpp b/inference-engine/src/auto_plugin/auto_exec_network.hpp index 635d9573b9da66..0a99ba061336e9 100644 --- a/inference-engine/src/auto_plugin/auto_exec_network.hpp +++ b/inference-engine/src/auto_plugin/auto_exec_network.hpp @@ -28,9 +28,9 @@ class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSaf public: using Ptr = std::shared_ptr; - AutoExecutableNetwork(const InferenceEngine::ExecutableNetwork& network, - const DeviceInformation& deviceInfo, - const bool needPerfCounters = false); + AutoExecutableNetwork(const InferenceEngine::SoExecutableNetworkInternal& network, + const DeviceInformation& deviceInfo, + const bool needPerfCounters = false); void Export(std::ostream& networkModel) override; InferenceEngine::RemoteContext::Ptr GetContext() const override; @@ -43,7 +43,7 @@ class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSaf ~AutoExecutableNetwork() override; DeviceInformation _deviceInfo; - InferenceEngine::ExecutableNetwork _network; + InferenceEngine::SoExecutableNetworkInternal _network; std::unordered_map _config; bool _needPerfCounters = false; }; diff --git a/inference-engine/src/auto_plugin/auto_infer_request.cpp b/inference-engine/src/auto_plugin/auto_infer_request.cpp index 0adc4ba09388b3..33deef1892f9f5 100644 --- a/inference-engine/src/auto_plugin/auto_infer_request.cpp +++ b/inference-engine/src/auto_plugin/auto_infer_request.cpp @@ -9,31 +9,31 @@ namespace AutoPlugin { using namespace InferenceEngine; -AutoInferRequest::AutoInferRequest(const InputsDataMap& networkInputs, - const OutputsDataMap& networkOutputs, - const InferRequest& inferRequest) +AutoInferRequest::AutoInferRequest(const InputsDataMap& networkInputs, + const OutputsDataMap& networkOutputs, + const IInferRequestInternal::Ptr& inferRequest) : IInferRequestInternal(networkInputs, networkOutputs) , _inferRequest(inferRequest) { } std::map AutoInferRequest::GetPerformanceCounts() const { - return _inferRequest.GetPerformanceCounts(); + return _inferRequest->GetPerformanceCounts(); } void AutoInferRequest::InferImpl() { - _inferRequest.Infer(); + _inferRequest->Infer(); } void AutoInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& data) { - _inferRequest.SetBlob(name, data); + _inferRequest->SetBlob(name, data); } Blob::Ptr AutoInferRequest::GetBlob(const std::string& name) { - return _inferRequest.GetBlob(name); + return _inferRequest->GetBlob(name); } void AutoInferRequest::Cancel() { - _inferRequest.Cancel(); + _inferRequest->Cancel(); } } // namespace AutoPlugin diff --git a/inference-engine/src/auto_plugin/auto_infer_request.hpp b/inference-engine/src/auto_plugin/auto_infer_request.hpp index 49da8a117e1779..473f93fb45a1c3 100644 --- a/inference-engine/src/auto_plugin/auto_infer_request.hpp +++ b/inference-engine/src/auto_plugin/auto_infer_request.hpp @@ -24,9 +24,9 @@ namespace AutoPlugin { class AutoInferRequest : public InferenceEngine::IInferRequestInternal { public: using Ptr = std::shared_ptr; - explicit AutoInferRequest(const InferenceEngine::InputsDataMap& networkInputs, - const InferenceEngine::OutputsDataMap& networkOutputs, - const InferenceEngine::InferRequest& inferRequest); + explicit AutoInferRequest(const InferenceEngine::InputsDataMap& networkInputs, + const InferenceEngine::OutputsDataMap& networkOutputs, + const InferenceEngine::IInferRequestInternal::Ptr& inferRequest); std::map GetPerformanceCounts() const override; void InferImpl() override; void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& data) override; @@ -34,7 +34,7 @@ class AutoInferRequest : public InferenceEngine::IInferRequestInternal { void Cancel() override; private: - InferenceEngine::InferRequest _inferRequest; + InferenceEngine::IInferRequestInternal::Ptr _inferRequest; }; } // namespace AutoPlugin diff --git a/inference-engine/src/auto_plugin/auto_plugin.cpp b/inference-engine/src/auto_plugin/auto_plugin.cpp index 280a74e52acf13..79f89ecdf4b76c 100644 --- a/inference-engine/src/auto_plugin/auto_plugin.cpp +++ b/inference-engine/src/auto_plugin/auto_plugin.cpp @@ -49,7 +49,7 @@ IE::IExecutableNetworkInternal::Ptr AutoInferencePlugin::LoadNetwork(const std:: // FIXME: always select CPU device now DeviceInformation selectedDevice = SelectDevice(metaDevices); - IE::ExecutableNetwork executableNetwork; + IE::SoExecutableNetworkInternal executableNetwork; try { executableNetwork = GetCore()->LoadNetwork(fileName, selectedDevice.deviceName, selectedDevice.config); } catch(const IE::Exception &iie) { @@ -60,7 +60,7 @@ IE::IExecutableNetworkInternal::Ptr AutoInferencePlugin::LoadNetwork(const std:: bool enablePerfCounters = false; try { enablePerfCounters = - executableNetwork.GetConfig(IE::PluginConfigParams::KEY_PERF_COUNT).as() == + executableNetwork->GetConfig(IE::PluginConfigParams::KEY_PERF_COUNT).as() == IE::PluginConfigParams::YES; } catch (...) { } @@ -85,7 +85,7 @@ IE::ExecutableNetworkInternal::Ptr AutoInferencePlugin::LoadExeNetworkImpl(const // FIXME: always select CPU device now DeviceInformation selectedDevice = SelectDevice(metaDevices); - IE::ExecutableNetwork executableNetwork; + IE::SoExecutableNetworkInternal executableNetwork; try { executableNetwork = GetCore()->LoadNetwork(network, selectedDevice.deviceName, selectedDevice.config); } catch(const IE::Exception &iie) { @@ -96,7 +96,7 @@ IE::ExecutableNetworkInternal::Ptr AutoInferencePlugin::LoadExeNetworkImpl(const bool enablePerfCounters = false; try { enablePerfCounters = - executableNetwork.GetConfig(IE::PluginConfigParams::KEY_PERF_COUNT).as() == + executableNetwork->GetConfig(IE::PluginConfigParams::KEY_PERF_COUNT).as() == IE::PluginConfigParams::YES; } catch (...) { } diff --git a/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp b/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp index cdd3c54ae17fd4..7113e34abe301c 100644 --- a/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp +++ b/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp @@ -13,35 +13,31 @@ HeteroAsyncInferRequest::HeteroAsyncInferRequest(const IInferRequestInternal::Pt const ITaskExecutor::Ptr& taskExecutor, const ITaskExecutor::Ptr& callbackExecutor) : AsyncInferRequestThreadSafeDefault(request, taskExecutor, callbackExecutor), - _heteroInferRequest(std::static_pointer_cast(request)), - _statusCodes{_heteroInferRequest->_inferRequests.size(), StatusCode::OK} { + _heteroInferRequest(std::static_pointer_cast(request)) { _pipeline.clear(); for (std::size_t requestId = 0; requestId < _heteroInferRequest->_inferRequests.size(); ++requestId) { struct RequestExecutor : ITaskExecutor { - explicit RequestExecutor(InferRequest & inferRequest) : _inferRequest(inferRequest) { - _inferRequest.SetCompletionCallback>( - [this] (InferRequest, StatusCode sts) mutable { - _status = sts; + explicit RequestExecutor(IInferRequestInternal::Ptr & inferRequest) : _inferRequest(inferRequest) { + _inferRequest->SetCallback( + [this] (std::exception_ptr exceptionPtr) mutable { + _exceptionPtr = exceptionPtr; auto capturedTask = std::move(_task); capturedTask(); }); } void run(Task task) override { _task = std::move(task); - _inferRequest.StartAsync(); + _inferRequest->StartAsync(); }; - InferRequest & _inferRequest; - StatusCode _status = StatusCode::OK; - Task _task; + IInferRequestInternal::Ptr & _inferRequest; + std::exception_ptr _exceptionPtr; + Task _task; }; auto requestExecutor = std::make_shared(_heteroInferRequest->_inferRequests[requestId]._request); _pipeline.emplace_back(requestExecutor, [requestExecutor] { - if (StatusCode::OK != requestExecutor->_status) { - IE_EXCEPTION_SWITCH(requestExecutor->_status, ExceptionType, - InferenceEngine::details::ThrowNow{} - <<= std::stringstream{} << IE_LOCATION - << InferenceEngine::details::ExceptionTraits::string()); + if (nullptr != requestExecutor->_exceptionPtr) { + std::rethrow_exception(requestExecutor->_exceptionPtr); } }); } @@ -58,7 +54,7 @@ StatusCode HeteroAsyncInferRequest::Wait(int64_t millis_timeout) { waitStatus = AsyncInferRequestThreadSafeDefault::Wait(millis_timeout); } catch(...) { for (auto&& requestDesc : _heteroInferRequest->_inferRequests) { - requestDesc._request.Wait(InferRequest::RESULT_READY); + requestDesc._request->Wait(InferRequest::RESULT_READY); } throw; } diff --git a/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp b/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp index 1a36f8724be416..22f8d065432287 100644 --- a/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp +++ b/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp @@ -23,7 +23,6 @@ class HeteroAsyncInferRequest : public InferenceEngine::AsyncInferRequestThreadS private: HeteroInferRequest::Ptr _heteroInferRequest; - std::vector _statusCodes; }; } // namespace HeteroPlugin diff --git a/inference-engine/src/hetero_plugin/hetero_executable_network.cpp b/inference-engine/src/hetero_plugin/hetero_executable_network.cpp index 83f8a57156cded..bc5ace2160c514 100644 --- a/inference-engine/src/hetero_plugin/hetero_executable_network.cpp +++ b/inference-engine/src/hetero_plugin/hetero_executable_network.cpp @@ -368,17 +368,17 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::CNNNetwo InputsDataMap externalInputsData = network.getInputsInfo(); OutputsDataMap externalOutputsData = network.getOutputsInfo(); - networks.resize(orderedSubgraphs.size()); + _networks.resize(orderedSubgraphs.size()); std::vector> subFunctions(orderedSubgraphs.size()); int id = 0; for (auto&& subgraph : orderedSubgraphs) { - networks[id]._device = subgraph._affinity; + _networks[id]._device = subgraph._affinity; subFunctions[id] = std::make_shared(subgraph._results, subgraph._parameters, _name + '_' + std::to_string(id)); - networks[id]._clonedNetwork = CNNNetwork{subFunctions[id]}; + _networks[id]._clonedNetwork = CNNNetwork{subFunctions[id]}; // update of pre-processing info - auto clonedInputs = networks[id]._clonedNetwork.getInputsInfo(); + auto clonedInputs = _networks[id]._clonedNetwork.getInputsInfo(); for (auto&& externalInput : externalInputsData) { auto itClonedInput = clonedInputs.find(externalInput.first); if (itClonedInput != clonedInputs.end() && nullptr != itClonedInput->second) { @@ -388,7 +388,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::CNNNetwo } } // update output info - auto clonedOutputs = networks[id]._clonedNetwork.getOutputsInfo(); + auto clonedOutputs = _networks[id]._clonedNetwork.getOutputsInfo(); for (auto&& externalOutput : externalOutputsData) { auto itClonedOutput = clonedOutputs.find(externalOutput.first); if (itClonedOutput != clonedOutputs.end() && nullptr != itClonedOutput->second) { @@ -418,7 +418,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::CNNNetwo } }}.run_on_function(ngraph::clone_function(*function)); } - for (auto&& network : networks) { + for (auto&& network : _networks) { auto metaDevices = _heteroPlugin->GetDevicePlugins(network._device, _config); metaDevices[network._device].emplace(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE), ""); network._network = _heteroPlugin->GetCore()->LoadNetwork(network._clonedNetwork, @@ -481,7 +481,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream& assert(metaDevices.size() == 1); auto& loadConfig = metaDevices[deviceName]; - InferenceEngine::ExecutableNetwork executableNetwork; + InferenceEngine::SoExecutableNetworkInternal executableNetwork; CNNNetwork cnnnetwork; bool loaded = false; if (ImportExportSupported(deviceName)) { @@ -526,13 +526,13 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream& } // restore network inputs and outputs - for (auto&& input : executableNetwork.GetInputsInfo()) { + for (auto&& input : executableNetwork->GetInputsInfo()) { if (networkInputs.end() != networkInputs.find(input.first)) { _networkInputs.emplace(input.first, std::make_shared(*input.second)); } } - for (auto&& output : executableNetwork.GetOutputsInfo()) { + for (auto&& output : executableNetwork->GetOutputsInfo()) { if (networkOutputs.end() != networkOutputs.find(output.first)) { _networkOutputs.emplace(output.first, std::make_shared(*output.second)); } @@ -547,7 +547,7 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(std::istream& // save state this->_config = importedConfigs; - this->networks = std::move(descs); + this->_networks = std::move(descs); this->SetPointerToPlugin(_heteroPlugin->shared_from_this()); } @@ -567,7 +567,7 @@ void HeteroExecutableNetwork::ExportImpl(std::ostream& heteroModel) { } auto subnetworksNode = heteroNode.append_child("subnetworks"); - for (auto&& subnetwork : networks) { + for (auto&& subnetwork : _networks) { auto subnet = subnetwork._clonedNetwork; IE_ASSERT(subnet.getFunction() != nullptr); @@ -611,9 +611,9 @@ void HeteroExecutableNetwork::ExportImpl(std::ostream& heteroModel) { doc.reset(); heteroModel << std::endl; - for (auto&& subnetwork : networks) { + for (auto&& subnetwork : _networks) { if (ImportExportSupported(subnetwork._device)) { - subnetwork._network.Export(heteroModel); + subnetwork._network->Export(heteroModel); } else { auto subnet = subnetwork._clonedNetwork; if (!subnet.getFunction()) { @@ -645,7 +645,7 @@ IInferRequestInternal::Ptr HeteroExecutableNetwork::CreateInferRequestImpl( OutputsDataMap networkOutputs) { HeteroInferRequest::SubRequestsList inferRequests; int index = 0; - for (auto&& subnetwork : networks) { + for (auto&& subnetwork : _networks) { HeteroInferRequest::SubRequestDesc desc; desc._network = subnetwork._network; desc._profilingTask = openvino::itt::handle("Infer" + std::to_string(index++)); @@ -677,12 +677,12 @@ InferenceEngine::Parameter HeteroExecutableNetwork::GetConfig(const std::string result = it->second == YES ? true : false; } else { // find config key among plugin config keys - for (auto&& desc : networks) { + for (auto&& desc : _networks) { auto execNetwork = desc._network; - auto param = execNetwork.GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS)); + auto param = execNetwork->GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS)); for (auto && configKey : param.as>()) { if (configKey == name) { - return execNetwork.GetConfig(configKey); + return execNetwork->GetConfig(configKey); } } } @@ -741,12 +741,12 @@ InferenceEngine::Parameter HeteroExecutableNetwork::GetMetric(const std::string { std::vector<::Metrics> pluginMetrics; - for (auto&& desc : networks) { + for (auto&& desc : _networks) { auto execNetwork = desc._network; - auto param = execNetwork.GetMetric(METRIC_KEY(SUPPORTED_METRICS)); + auto param = execNetwork->GetMetric(METRIC_KEY(SUPPORTED_METRICS)); ::Metrics metrics; for (auto && metricName : param.as>()) { - metrics[metricName] = execNetwork.GetMetric(metricName); + metrics[metricName] = execNetwork->GetMetric(metricName); } pluginMetrics.push_back(std::move(metrics)); } @@ -764,12 +764,12 @@ InferenceEngine::Parameter HeteroExecutableNetwork::GetMetric(const std::string { std::vector<::Metrics> pluginConfigKeys; - for (auto&& desc : networks) { + for (auto&& desc : _networks) { auto execNetwork = desc._network; - auto param = execNetwork.GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS)); + auto param = execNetwork->GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS)); ::Metrics configKeys; for (auto && metricName : param.as>()) { - configKeys[metricName] = execNetwork.GetConfig(metricName); + configKeys[metricName] = execNetwork->GetConfig(metricName); } pluginConfigKeys.push_back(std::move(configKeys)); } @@ -782,18 +782,18 @@ InferenceEngine::Parameter HeteroExecutableNetwork::GetMetric(const std::string IE_SET_METRIC_RETURN(NETWORK_NAME, _name); } else if (EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS) == name) { unsigned int value = 0u; - for (auto&& desc : networks) { - value = std::max(value, desc._network.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as()); + for (auto&& desc : _networks) { + value = std::max(value, desc._network->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as()); } IE_SET_METRIC_RETURN(OPTIMAL_NUMBER_OF_INFER_REQUESTS, value); } else { // find metric key among plugin metrics - for (auto&& desc : networks) { + for (auto&& desc : _networks) { auto execNetwork = desc._network; - auto param = execNetwork.GetMetric(METRIC_KEY(SUPPORTED_METRICS)); + auto param = execNetwork->GetMetric(METRIC_KEY(SUPPORTED_METRICS)); for (auto && metricKey : param.as>()) { if (metricKey == name) { - return execNetwork.GetMetric(metricKey); + return execNetwork->GetMetric(metricKey); } } } diff --git a/inference-engine/src/hetero_plugin/hetero_executable_network.hpp b/inference-engine/src/hetero_plugin/hetero_executable_network.hpp index 323cafbeb1eb39..9189d965a813af 100644 --- a/inference-engine/src/hetero_plugin/hetero_executable_network.hpp +++ b/inference-engine/src/hetero_plugin/hetero_executable_network.hpp @@ -50,7 +50,7 @@ class HeteroExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadS ~HeteroExecutableNetwork() override = default; InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, - InferenceEngine::OutputsDataMap networkOutputs) override; + InferenceEngine::OutputsDataMap networkOutputs) override; InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override; @@ -66,15 +66,15 @@ class HeteroExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadS bool ImportExportSupported(const std::string& deviceName) const; struct NetworkDesc { - std::string _device; - InferenceEngine::CNNNetwork _clonedNetwork; - InferenceEngine::ExecutableNetwork _network; + std::string _device; + InferenceEngine::CNNNetwork _clonedNetwork; + InferenceEngine::SoExecutableNetworkInternal _network; }; - std::vector networks; - Engine* _heteroPlugin; - std::string _name; - std::map _config; + std::vector _networks; + Engine* _heteroPlugin; + std::string _name; + std::map _config; std::unordered_map _blobNameMap; }; diff --git a/inference-engine/src/hetero_plugin/hetero_infer_request.cpp b/inference-engine/src/hetero_plugin/hetero_infer_request.cpp index e6fb139daa1060..1c8a72e0a22ae8 100644 --- a/inference-engine/src/hetero_plugin/hetero_infer_request.cpp +++ b/inference-engine/src/hetero_plugin/hetero_infer_request.cpp @@ -26,7 +26,7 @@ HeteroInferRequest::HeteroInferRequest(InferenceEngine::InputsDataMap networkInp IE_THROW() << "Internal error: no information about network's output/input"; } - auto requestBlob([&](const std::string& blobName, InferenceEngine::InferRequest & r) { + auto requestBlob([&](const std::string& blobName, InferenceEngine::IInferRequestInternal::Ptr & r) { std::string intermediateBlobName = blobName; auto itName = subgraphInputToOutputBlobNames.find(blobName); if (itName != subgraphInputToOutputBlobNames.end()) { @@ -36,29 +36,29 @@ HeteroInferRequest::HeteroInferRequest(InferenceEngine::InputsDataMap networkInp bool emplaced = false; std::tie(itBlob, emplaced) = _blobs.emplace(intermediateBlobName, Blob::Ptr{}); if (emplaced) { - itBlob->second = r.GetBlob(blobName); + itBlob->second = r->GetBlob(blobName); if (InferenceEngine::details::contains(networkInputs, blobName)) { _inputs[blobName] = itBlob->second; } else if (InferenceEngine::details::contains(networkOutputs, blobName)) { _outputs[blobName] = itBlob->second; } } else { - r.SetBlob(blobName, itBlob->second); + r->SetBlob(blobName, itBlob->second); } }); // go over all subnet and create requests for (auto&& desc : _inferRequests) { - desc._request = desc._network.CreateInferRequest(); + desc._request = desc._network->CreateInferRequest(); // go over all inputs and get blobs from subnet infer requests - for (auto&& outputInfo : desc._network.GetOutputsInfo()) { + for (auto&& outputInfo : desc._network->GetOutputsInfo()) { requestBlob(outputInfo.first, desc._request); } } // go over all outputs and get blobs from subnet infer requests for (auto&& desc : _inferRequests) { - for (auto&& inputInfo : desc._network.GetInputsInfo()) { + for (auto&& inputInfo : desc._network->GetInputsInfo()) { requestBlob(inputInfo.first, desc._request); } } @@ -75,7 +75,7 @@ void HeteroInferRequest::SetBlob(const std::string& name, const InferenceEngine: try { // if `name` is input blob if (findInputAndOutputBlobByName(name, foundInput, foundOutput)) { - r.SetBlob(name, data, foundInput->getPreProcess()); + r->SetBlob(name, data, foundInput->getPreProcess()); } } catch (const InferenceEngine::NotFound& ex) {} } @@ -87,14 +87,14 @@ void HeteroInferRequest::InferImpl() { OV_ITT_SCOPED_TASK(itt::domains::HeteroPlugin, desc._profilingTask); auto &r = desc._request; assert(r); - r.Infer(); + r->Infer(); } } std::map HeteroInferRequest::GetPerformanceCounts() const { std::map perfMap; for (size_t i = 0; i < _inferRequests.size(); i++) { - auto perfMapRequest = _inferRequests[i]._request.GetPerformanceCounts(); + auto perfMapRequest = _inferRequests[i]._request->GetPerformanceCounts(); for (auto &&r : perfMapRequest) { perfMap[std::string("subgraph") + std::to_string(i) + ": " + r.first] = r.second; } @@ -108,30 +108,30 @@ void HeteroInferRequest::updateInOutIfNeeded() { for (auto &&desc : _inferRequests) { auto &r = desc._request; assert(r); - for (auto&& inputInfo : desc._network.GetInputsInfo()) { + for (auto&& inputInfo : desc._network->GetInputsInfo()) { auto& ioname = inputInfo.first; auto iti = _inputs.find(ioname); if (iti != _inputs.end()) { auto it = _preProcData.find(ioname); if (it != _preProcData.end()) { if (it->second->getRoiBlob() != _blobs[ioname]) { - r.SetBlob(ioname.c_str(), it->second->getRoiBlob()); + r->SetBlob(ioname.c_str(), it->second->getRoiBlob()); _blobs[ioname] = iti->second; } } else { if (iti->second != _blobs[ioname]) { - r.SetBlob(ioname.c_str(), iti->second); + r->SetBlob(ioname.c_str(), iti->second); _blobs[ioname] = iti->second; } } } } - for (auto&& outputInfo : desc._network.GetOutputsInfo()) { + for (auto&& outputInfo : desc._network->GetOutputsInfo()) { auto& ioname = outputInfo.first; auto ito = _outputs.find(ioname); if (ito != _outputs.end()) { if (ito->second != _blobs[ioname]) { - r.SetBlob(ioname.c_str(), ito->second); + r->SetBlob(ioname.c_str(), ito->second); _blobs[ioname] = ito->second; } } diff --git a/inference-engine/src/hetero_plugin/hetero_infer_request.hpp b/inference-engine/src/hetero_plugin/hetero_infer_request.hpp index 9f13e57a817eb4..b6cb02c6dd04e7 100644 --- a/inference-engine/src/hetero_plugin/hetero_infer_request.hpp +++ b/inference-engine/src/hetero_plugin/hetero_infer_request.hpp @@ -23,9 +23,9 @@ class HeteroInferRequest : public InferenceEngine::IInferRequestInternal { typedef std::shared_ptr Ptr; struct SubRequestDesc { - InferenceEngine::ExecutableNetwork _network; - InferenceEngine::InferRequest _request; - openvino::itt::handle_t _profilingTask; + InferenceEngine::SoExecutableNetworkInternal _network; + InferenceEngine::IInferRequestInternal::Ptr _request; + openvino::itt::handle_t _profilingTask; }; using SubRequestsList = std::vector; diff --git a/inference-engine/src/inference_engine/cpp/ie_plugin.hpp b/inference-engine/src/inference_engine/cpp/ie_plugin.hpp index 0df92057607136..d9802e6c59f5aa 100644 --- a/inference-engine/src/inference_engine/cpp/ie_plugin.hpp +++ b/inference-engine/src/inference_engine/cpp/ie_plugin.hpp @@ -65,15 +65,15 @@ class InferencePlugin : protected details::SOPointer { PLUGIN_CALL_STATEMENT(_ptr->SetConfig(config)); } - ExecutableNetwork LoadNetwork(const CNNNetwork& network, const std::map& config) { + details::SOPointer LoadNetwork(const CNNNetwork& network, const std::map& config) { PLUGIN_CALL_STATEMENT(return {_so, _ptr->LoadNetwork(network, config)}); } - ExecutableNetwork LoadNetwork(const CNNNetwork& network, RemoteContext::Ptr context, const std::map& config) { + details::SOPointer LoadNetwork(const CNNNetwork& network, RemoteContext::Ptr context, const std::map& config) { PLUGIN_CALL_STATEMENT(return {_so, _ptr->LoadNetwork(network, config, context)}); } - ExecutableNetwork LoadNetwork(const std::string& modelPath, const std::map& config) { + details::SOPointer LoadNetwork(const std::string& modelPath, const std::map& config) { PLUGIN_CALL_STATEMENT(return {_so, _ptr->LoadNetwork(modelPath, config)}); } @@ -85,17 +85,17 @@ class InferencePlugin : protected details::SOPointer { return res; } - ExecutableNetwork ImportNetwork(const std::string& modelFileName, + details::SOPointer ImportNetwork(const std::string& modelFileName, const std::map& config) { PLUGIN_CALL_STATEMENT(return {_so, _ptr->ImportNetwork(modelFileName, config)}); } - ExecutableNetwork ImportNetwork(std::istream& networkModel, + details::SOPointer ImportNetwork(std::istream& networkModel, const std::map& config) { PLUGIN_CALL_STATEMENT(return {_so, _ptr->ImportNetwork(networkModel, config)}); } - ExecutableNetwork ImportNetwork(std::istream& networkModel, + details::SOPointer ImportNetwork(std::istream& networkModel, const RemoteContext::Ptr& context, const std::map& config) { PLUGIN_CALL_STATEMENT(return {_so, _ptr->ImportNetwork(networkModel, context, config)}); diff --git a/inference-engine/src/inference_engine/ie_core.cpp b/inference-engine/src/inference_engine/ie_core.cpp index 9f026ecf693364..fa2fa8a6422dfd 100644 --- a/inference-engine/src/inference_engine/ie_core.cpp +++ b/inference-engine/src/inference_engine/ie_core.cpp @@ -26,6 +26,7 @@ #include "ie_network_reader.hpp" #include "xml_parse_utils.h" #include "cpp_interfaces/interface/ie_internal_plugin_config.hpp" +#include "cpp_interfaces/interface/ie_iexecutable_network_internal.hpp" using namespace InferenceEngine::PluginConfigParams; using namespace std::placeholders; @@ -223,15 +224,15 @@ class Core::Impl : public ICore { return supported; } - ExecutableNetwork LoadNetworkImpl(const CNNNetwork& network, - InferencePlugin& plugin, - const std::map& parsedConfig, - const RemoteContext::Ptr& context, - const std::string& blobID, - const std::string& modelPath = std::string(), - bool forceDisableCache = false) { + SoExecutableNetworkInternal LoadNetworkImpl(const CNNNetwork& network, + InferencePlugin& plugin, + const std::map& parsedConfig, + const RemoteContext::Ptr& context, + const std::string& blobID, + const std::string& modelPath = std::string(), + bool forceDisableCache = false) { OV_ITT_SCOPED_TASK(itt::domains::IE, "Core::Impl::LoadNetworkImpl"); - ExecutableNetwork execNetwork; + SoExecutableNetworkInternal execNetwork; execNetwork = context ? plugin.LoadNetwork(network, context, parsedConfig) : plugin.LoadNetwork(network, parsedConfig); auto cacheManager = coreConfig.getCacheConfig()._cacheManager; @@ -242,7 +243,7 @@ class Core::Impl : public ICore { cacheManager->writeCacheEntry(blobID, [&](std::ostream& networkStream) { networkStream << CompiledBlobHeader(GetInferenceEngineVersion()->buildNumber, NetworkCompilationContext::calculateFileInfo(modelPath)); - execNetwork.Export(networkStream); + execNetwork->Export(networkStream); }); } catch (...) { cacheManager->removeCacheEntry(blobID); @@ -252,14 +253,14 @@ class Core::Impl : public ICore { return execNetwork; } - ExecutableNetwork LoadNetworkFromCache(const std::shared_ptr& cacheManager, - const std::string& blobId, - InferencePlugin& plugin, - const std::map& config, - const RemoteContext::Ptr& context, - bool& networkIsImported, - const std::string& modelPath = std::string()) { - ExecutableNetwork execNetwork; + SoExecutableNetworkInternal LoadNetworkFromCache(const std::shared_ptr& cacheManager, + const std::string& blobId, + InferencePlugin& plugin, + const std::map& config, + const RemoteContext::Ptr& context, + bool& networkIsImported, + const std::string& modelPath = std::string()) { + SoExecutableNetworkInternal execNetwork; struct HeaderException {}; IE_ASSERT(cacheManager != nullptr); @@ -446,15 +447,15 @@ class Core::Impl : public ICore { } // TODO: In future this method can be added to ICore interface - ExecutableNetwork LoadNetwork(const CNNNetwork& network, const RemoteContext::Ptr& context, - const std::map& config) { + SoExecutableNetworkInternal LoadNetwork(const CNNNetwork& network, const RemoteContext::Ptr& context, + const std::map& config) { OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::IE_LT, "Core::LoadNetwork::RemoteContext"); if (context == nullptr) { IE_THROW() << "Remote context is null"; } auto parsed = parseDeviceNameIntoConfig(context->getDeviceName(), config); auto plugin = GetCPPPluginByName(parsed._deviceName); - ExecutableNetwork res; + SoExecutableNetworkInternal res; auto cacheManager = coreConfig.getCacheConfig()._cacheManager; if (cacheManager && DeviceSupportsImportExport(plugin)) { auto hash = CalculateNetworkHash(network, parsed._deviceName, plugin, parsed._config); @@ -470,8 +471,8 @@ class Core::Impl : public ICore { return res; } - ExecutableNetwork LoadNetwork(const CNNNetwork& network, const std::string& deviceName, - const std::map& config) override { + SoExecutableNetworkInternal LoadNetwork(const CNNNetwork& network, const std::string& deviceName, + const std::map& config) override { OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::IE_LT, "Core::LoadNetwork::CNN"); bool forceDisableCache = config.count(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)) > 0; auto parsed = parseDeviceNameIntoConfig(deviceName, config); @@ -480,7 +481,7 @@ class Core::Impl : public ICore { parsed._config.erase(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)); } auto plugin = GetCPPPluginByName(parsed._deviceName); - ExecutableNetwork res; + SoExecutableNetworkInternal res; auto cacheManager = coreConfig.getCacheConfig()._cacheManager; if (!forceDisableCache && cacheManager && DeviceSupportsImportExport(plugin)) { auto hash = CalculateNetworkHash(network, parsed._deviceName, plugin, parsed._config); @@ -496,12 +497,12 @@ class Core::Impl : public ICore { return res; } - ExecutableNetwork LoadNetwork(const std::string& modelPath, const std::string& deviceName, - const std::map& config) override { + SoExecutableNetworkInternal LoadNetwork(const std::string& modelPath, const std::string& deviceName, + const std::map& config) override { OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::IE_LT, "Core::LoadNetwork::Path"); auto parsed = parseDeviceNameIntoConfig(deviceName, config); auto plugin = GetCPPPluginByName(parsed._deviceName); - ExecutableNetwork res; + SoExecutableNetworkInternal res; auto cacheManager = coreConfig.getCacheConfig()._cacheManager; if (cacheManager && DeviceSupportsImportExport(plugin)) { bool loadedFromCache = false; @@ -522,8 +523,8 @@ class Core::Impl : public ICore { return res; } - ExecutableNetwork ImportNetwork(std::istream& networkModel, const std::string& deviceName, - const std::map& config) override { + SoExecutableNetworkInternal ImportNetwork(std::istream& networkModel, const std::string& deviceName, + const std::map& config) override { auto parsed = parseDeviceNameIntoConfig(deviceName, config); if (parsed._deviceName.empty()) { @@ -925,7 +926,6 @@ RemoteContext::Ptr Core::CreateContext(const std::string& deviceName, const Para IE_THROW() << "AUTO device does not support remote context"; } - auto parsed = parseDeviceNameIntoConfig(deviceName, params); return _impl->GetCPPPluginByName(parsed._deviceName).CreateContext(parsed._config); } diff --git a/inference-engine/src/inference_engine/ie_system_conf.cpp b/inference-engine/src/inference_engine/ie_system_conf.cpp index d9219597a93799..2626ad7e424407 100644 --- a/inference-engine/src/inference_engine/ie_system_conf.cpp +++ b/inference-engine/src/inference_engine/ie_system_conf.cpp @@ -111,9 +111,4 @@ std::vector getAvailableCoresTypes() { } #endif -std::exception_ptr& CurrentException() { - static thread_local std::exception_ptr currentException = nullptr; - return currentException; -} - } // namespace InferenceEngine diff --git a/inference-engine/src/multi_device/multi_device_async_infer_request.cpp b/inference-engine/src/multi_device/multi_device_async_infer_request.cpp index ca1c98976a2aea..9f845586a55330 100644 --- a/inference-engine/src/multi_device/multi_device_async_infer_request.cpp +++ b/inference-engine/src/multi_device/multi_device_async_infer_request.cpp @@ -28,7 +28,7 @@ MultiDeviceAsyncInferRequest::MultiDeviceAsyncInferRequest( void run(Task task) override { auto workerInferRequest = _this->_workerInferRequest; workerInferRequest->_task = std::move(task); - workerInferRequest->_inferRequest.StartAsync(); + workerInferRequest->_inferRequest->StartAsync(); }; MultiDeviceAsyncInferRequest* _this = nullptr; }; @@ -69,18 +69,11 @@ MultiDeviceAsyncInferRequest::MultiDeviceAsyncInferRequest( }}, // final task in the pipeline: { /*TaskExecutor*/std::make_shared(this), /*task*/ [this] { - auto status = _workerInferRequest->_status; - if (InferenceEngine::StatusCode::OK != status) { - if (nullptr != InferenceEngine::CurrentException()) - std::rethrow_exception(InferenceEngine::CurrentException()); - else - IE_EXCEPTION_SWITCH(status, ExceptionType, - InferenceEngine::details::ThrowNow{} - <<= std::stringstream{} << IE_LOCATION - << InferenceEngine::details::ExceptionTraits::string()); + if (nullptr != _workerInferRequest->_exceptionPtr) { + std::rethrow_exception(_workerInferRequest->_exceptionPtr); } if (_needPerfCounters) - _perfMap = _workerInferRequest->_inferRequest.GetPerformanceCounts(); + _perfMap = _workerInferRequest->_inferRequest->GetPerformanceCounts(); }} }; } diff --git a/inference-engine/src/multi_device/multi_device_exec_network.cpp b/inference-engine/src/multi_device/multi_device_exec_network.cpp index 9e42cbe080919d..5265e233cfd0a0 100644 --- a/inference-engine/src/multi_device/multi_device_exec_network.cpp +++ b/inference-engine/src/multi_device/multi_device_exec_network.cpp @@ -47,7 +47,7 @@ struct IdleGuard { MultiDeviceExecutableNetwork::NotBusyWorkerRequests* _notBusyWorkerRequests = nullptr; }; -MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const DeviceMap& networksPerDevice, +MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const DeviceMap& networksPerDevice, const std::vector& networkDevices, const std::unordered_map& config, const bool needPerfCounters) : @@ -66,7 +66,7 @@ MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const DeviceMap(); + optimalNum = network->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as(); } catch (const InferenceEngine::Exception &iie) { IE_THROW() << "Every device used with the Multi-Device should " @@ -82,13 +82,13 @@ MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const DeviceMapCreateInferRequest(); auto* workerRequestPtr = &workerRequest; IE_ASSERT(idleWorkerRequests.try_push(workerRequestPtr) == true); - workerRequest._inferRequest.SetCompletionCallback>( - [workerRequestPtr, this, device, idleWorkerRequestsPtr] (InferRequest , StatusCode status) mutable { + workerRequest._inferRequest->SetCallback( + [workerRequestPtr, this, device, idleWorkerRequestsPtr] (std::exception_ptr exceptionPtr) mutable { IdleGuard idleGuard{workerRequestPtr, *idleWorkerRequestsPtr}; - workerRequestPtr->_status = status; + workerRequestPtr->_exceptionPtr = exceptionPtr; { auto capturedTask = std::move(workerRequestPtr->_task); capturedTask(); @@ -166,7 +166,7 @@ RemoteContext::Ptr MultiDeviceExecutableNetwork::GetContext() const { devices_names += device.deviceName + " "; const auto& n = _networksPerDevice.at(device.deviceName); try { - return n.GetContext(); + return n->GetContext(); } catch (const NotImplemented&) {} } IE_THROW(NotImplemented) << "None of the devices in the MULTI has an associated remote context." @@ -177,7 +177,7 @@ InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::Create InferenceEngine::OutputsDataMap networkOutputs) { auto num = _numRequestsCreated++; size_t sum = 0; - InferenceEngine::InferRequest request_to_share_blobs_with; + InferenceEngine::IInferRequestInternal::Ptr request_to_share_blobs_with; // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) for (const auto& device : _devicePrioritiesInitial) { @@ -248,7 +248,7 @@ InferenceEngine::Parameter MultiDeviceExecutableNetwork::GetMetric(const std::st unsigned int res = 0u; for (auto n : _networksPerDevice) { try { - res += n.second.GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as(); + res += n.second->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as(); } catch (const InferenceEngine::Exception &iie) { IE_THROW() << "Every device used with the Multi-Device should " @@ -260,7 +260,7 @@ InferenceEngine::Parameter MultiDeviceExecutableNetwork::GetMetric(const std::st } else if (name == METRIC_KEY(NETWORK_NAME)) { auto it = _networksPerDevice.begin(); IE_ASSERT(it != _networksPerDevice.end()); - IE_SET_METRIC_RETURN(NETWORK_NAME, it->second.GetMetric( + IE_SET_METRIC_RETURN(NETWORK_NAME, it->second->GetMetric( METRIC_KEY(NETWORK_NAME)).as()); } else if (name == METRIC_KEY(SUPPORTED_METRICS)) { IE_SET_METRIC_RETURN(SUPPORTED_METRICS, { diff --git a/inference-engine/src/multi_device/multi_device_exec_network.hpp b/inference-engine/src/multi_device/multi_device_exec_network.hpp index dce26006b0502b..39dd6e999c1f47 100644 --- a/inference-engine/src/multi_device/multi_device_exec_network.hpp +++ b/inference-engine/src/multi_device/multi_device_exec_network.hpp @@ -99,13 +99,13 @@ class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkTh public: using Ptr = std::shared_ptr; struct WorkerInferRequest { - InferenceEngine::InferRequest _inferRequest; - InferenceEngine::Task _task; - InferenceEngine::StatusCode _status = InferenceEngine::StatusCode::OK; + InferenceEngine::IInferRequestInternal::Ptr _inferRequest; + InferenceEngine::Task _task; + std::exception_ptr _exceptionPtr = nullptr; }; using NotBusyWorkerRequests = ThreadSafeBoundedQueue; - explicit MultiDeviceExecutableNetwork(const DeviceMap& networksPerDevice, + explicit MultiDeviceExecutableNetwork(const DeviceMap& networksPerDevice, const std::vector& networkDevices, const std::unordered_map& config, const bool needPerfCounters = false); @@ -130,7 +130,7 @@ class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkTh mutable std::mutex _mutex; std::vector _devicePriorities; const std::vector _devicePrioritiesInitial; - DeviceMap _networksPerDevice; + DeviceMap _networksPerDevice; ThreadSafeQueue _inferPipelineTasks; DeviceMap>> _inferPipelineTasksDeviceSpecific; DeviceMap _idleWorkerRequests; diff --git a/inference-engine/src/multi_device/multi_device_infer_request.cpp b/inference-engine/src/multi_device/multi_device_infer_request.cpp index bebf27ec662191..5913a0b39d728f 100644 --- a/inference-engine/src/multi_device/multi_device_infer_request.cpp +++ b/inference-engine/src/multi_device/multi_device_infer_request.cpp @@ -10,18 +10,20 @@ #include namespace MultiDevicePlugin { - using namespace InferenceEngine; + +using namespace InferenceEngine; + // ------------------------------MultiDeviceInferRequest---------------------------- MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkInputs, const OutputsDataMap& networkOutputs, - InferRequest request_to_share_blobs_with) + const IInferRequestInternal::Ptr & request_to_share_blobs_with) : IInferRequestInternal(networkInputs, networkOutputs) { if (request_to_share_blobs_with) { // borrow device-friendly blobs from the request for (const auto &it : _networkInputs) - _inputs[it.first] = request_to_share_blobs_with.GetBlob(it.first); + _inputs[it.first] = request_to_share_blobs_with->GetBlob(it.first); for (const auto &it : _networkOutputs) - _outputs[it.first] = request_to_share_blobs_with.GetBlob(it.first); + _outputs[it.first] = request_to_share_blobs_with->GetBlob(it.first); return; } // Allocate all input blobs @@ -46,20 +48,20 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI } } -void MultiDeviceInferRequest::SetBlobsToAnotherRequest(InferRequest& req) { +void MultiDeviceInferRequest::SetBlobsToAnotherRequest(const IInferRequestInternal::Ptr& req) { for (const auto &it : _networkInputs) { auto &name = it.first; // this request is already in BUSY state, so using the internal functions safely auto blob = GetBlob(name); - if (req.GetBlob(name) != blob) - req.SetBlob(name, blob); + if (req->GetBlob(name) != blob) + req->SetBlob(name, blob); } for (const auto &it : _networkOutputs) { auto &name = it.first; // this request is already in BUSY state, so using the internal functions safely auto blob = GetBlob(name); - if (req.GetBlob(name) != blob) - req.SetBlob(name, blob); + if (req->GetBlob(name) != blob) + req->SetBlob(name, blob); } } diff --git a/inference-engine/src/multi_device/multi_device_infer_request.hpp b/inference-engine/src/multi_device/multi_device_infer_request.hpp index 3284b985054fe5..80f81c37e9c210 100644 --- a/inference-engine/src/multi_device/multi_device_infer_request.hpp +++ b/inference-engine/src/multi_device/multi_device_infer_request.hpp @@ -25,11 +25,11 @@ class MultiDeviceInferRequest : public InferenceEngine::IInferRequestInternal { using Ptr = std::shared_ptr; explicit MultiDeviceInferRequest(const InferenceEngine::InputsDataMap& networkInputs, const InferenceEngine::OutputsDataMap& networkOutputs, - InferenceEngine::InferRequest request_to_share_blobs_with); + const InferenceEngine::IInferRequestInternal::Ptr & request_to_share_blobs_with); std::map GetPerformanceCounts() const override; void InferImpl() override; // Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request) - void SetBlobsToAnotherRequest(InferenceEngine::InferRequest& req); + void SetBlobsToAnotherRequest(const InferenceEngine::IInferRequestInternal::Ptr& req); }; } // namespace MultiDevicePlugin diff --git a/inference-engine/src/multi_device/multi_device_plugin.cpp b/inference-engine/src/multi_device/multi_device_plugin.cpp index 15b25b59ad08d1..f15db83651ec05 100644 --- a/inference-engine/src/multi_device/multi_device_plugin.cpp +++ b/inference-engine/src/multi_device/multi_device_plugin.cpp @@ -164,7 +164,7 @@ ExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadExeNetworkImpl(co std::unordered_map multiNetworkConfig; multiNetworkConfig.insert(*priorities); - DeviceMap executableNetworkPerDevice; + DeviceMap executableNetworkPerDevice; std::mutex load_mutex; std::vector loads; for (auto& p : metaDevices) { @@ -192,7 +192,7 @@ ExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadExeNetworkImpl(co for (auto n : executableNetworkPerDevice) { try { num_plugins_supporting_perf_counters += - n.second.GetConfig(PluginConfigParams::KEY_PERF_COUNT).as() == + n.second->GetConfig(PluginConfigParams::KEY_PERF_COUNT).as() == PluginConfigParams::YES; } catch (...) { } diff --git a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_2_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_2_internal.hpp deleted file mode 100644 index 08e76565cccecc..00000000000000 --- a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_2_internal.hpp +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "cpp_interfaces/impl/ie_executable_network_internal.hpp" -#include "cpp_interfaces/impl/ie_infer_request_2_internal.hpp" -#include "cpp/ie_executable_network.hpp" - -namespace InferenceEngine { - -class ExecutableNetwork2Internal : public IExecutableNetworkInternal { - ExecutableNetwork actual; -public: - explicit ExecutableNetwork2Internal(const ExecutableNetwork & exec) : actual(exec) { - if (!actual) { - IE_THROW(NotAllocated); - } - } - - virtual void setNetworkInputs(const InferenceEngine::InputsDataMap networkInputs) { - // should call internal methods - IE_THROW(NotImplemented); - } - - virtual void setNetworkOutputs(const InferenceEngine::OutputsDataMap networkOutputs) { - // should call internal methods - IE_THROW(NotImplemented); - } - - ConstOutputsDataMap GetOutputsInfo() const override { - return actual.GetOutputsInfo(); - } - - ConstInputsDataMap GetInputsInfo() const override { - return actual.GetInputsInfo(); - } - - void Export(const std::string& modelFileName) override { - actual.Export(modelFileName); - } - - void Export(std::ostream& networkModel) override { - actual.Export(networkModel); - } - - CNNNetwork GetExecGraphInfo() override { - return actual.GetExecGraphInfo(); - } - - std::vector QueryState() override { - IE_SUPPRESS_DEPRECATED_START - std::vector states; - for (auto & state : actual.QueryState()) { - states.push_back(std::make_shared(state)); - } - return states; - IE_SUPPRESS_DEPRECATED_END - } - - void SetConfig(const std::map& config) override { - actual.SetConfig(config); - } - - Parameter GetConfig(const std::string& name) const override { - return actual.GetConfig(name); - } - - Parameter GetMetric(const std::string& name) const override { - return actual.GetConfig(name); - } - - RemoteContext::Ptr GetContext() const override { - return actual.GetContext(); - } - - IInferRequestInternal::Ptr CreateInferRequest() override { - return std::make_shared(actual.CreateInferRequest()); - } -}; - -} // namespace InferenceEngine diff --git a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_request_2_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_request_2_internal.hpp deleted file mode 100644 index 7189edc6b55107..00000000000000 --- a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_request_2_internal.hpp +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp" -#include "cpp_interfaces/impl/ie_variable_state_2_internal.hpp" - -namespace InferenceEngine { - -class InferRequest2Internal : public IInferRequestInternal { - InferRequest actual; - -public: - explicit InferRequest2Internal(const InferRequest & request) : actual(request) { - if (!actual) { - IE_THROW(NotAllocated); - } - } - - void Infer() override { - actual.Infer(); - } - - virtual void InferImpl() { - // should call internal methods - IE_THROW(NotImplemented); - } - - void Cancel() override { - actual.Cancel(); - } - - std::map GetPerformanceCounts() const override { - return actual.GetPerformanceCounts(); - } - - void SetBlob(const std::string& name, const Blob::Ptr& data) override { - actual.SetBlob(name, data); - } - - Blob::Ptr GetBlob(const std::string& name) override { - return actual.GetBlob(name); - } - - void SetBlob(const std::string& name, const Blob::Ptr& data, const PreProcessInfo& info) override { - actual.SetBlob(name, data, info); - } - - const PreProcessInfo& GetPreProcess(const std::string& name) const override { - return actual.GetPreProcess(name); - } - - void SetBatch(int batch) override { - actual.SetBatch(batch); - } - - std::vector> QueryState() override { - std::vector states; - for (auto & state : actual.QueryState()) { - states.push_back(std::make_shared(state)); - } - return states; - } - - void StartAsync() override { - actual.StartAsync(); - } - - void StartAsyncImpl() override { - // should call internal methods - IE_THROW(NotImplemented); - } - - StatusCode Wait(int64_t millis_timeout) override { - return actual.Wait(millis_timeout); - } - - void SetCallback(Callback callback) override { - // actual.SetCallback([&] (std::exception_ptr exceptionPtr) { - // StatusCode statusCode = StatusCode::OK; - // if (exceptionPtr != nullptr) { - // statusCode = [&] { - // try { - // std::rethrow_exception(exceptionPtr); - // } CATCH_IE_EXCEPTIONS_RETURN catch (const std::exception& ex) { - // return GENERAL_ERROR; - // } catch (...) { - // return UNEXPECTED; - // } - // } (); - // } - // callback(weakThis, statusCode); - // }); - } - - void checkBlobs() override { - // should call internal methods - IE_THROW(NotImplemented); - } -}; - -} // namespace InferenceEngine diff --git a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_plugin_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_plugin_internal.hpp index 270daa372d423c..2afb5f066f6509 100644 --- a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_plugin_internal.hpp +++ b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_plugin_internal.hpp @@ -16,7 +16,7 @@ #include #include -#include "cpp_interfaces/impl/ie_executable_network_2_internal.hpp" +#include "cpp_interfaces/impl/ie_executable_network_internal.hpp" #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" #include "cpp_interfaces/plugin_itt.hpp" @@ -75,8 +75,7 @@ class InferencePluginInternal : public IInferencePlugin { IExecutableNetworkInternal::Ptr LoadNetwork(const std::string& modelPath, const std::map& config) override { auto cnnNet = GetCore()->ReadNetwork(modelPath, std::string()); - auto exec = GetCore()->LoadNetwork(cnnNet, GetName(), config); - return std::make_shared(exec); + return GetCore()->LoadNetwork(cnnNet, GetName(), config); } IExecutableNetworkInternal::Ptr ImportNetwork(const std::string& modelFileName, diff --git a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_variable_state_2_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_variable_state_2_internal.hpp deleted file mode 100644 index de9b580c7516b9..00000000000000 --- a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_variable_state_2_internal.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2018-2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include "cpp/ie_memory_state.hpp" - -namespace InferenceEngine { - -class VariableState2Internal : public IVariableStateInternal { - VariableState actual; - -public: - explicit VariableState2Internal(const VariableState & variableState) : - actual(variableState) { - // TODO: added a check for emptyness - // if (!actual) { - // IE_THROW(NotAllocated); - // } - } - - virtual std::string GetName() const override { - return actual.GetName(); - } - - void Reset() override { - actual.Reset(); - } - - void SetState(Blob::Ptr newState) override { - actual.SetState(newState); - } - - Blob::CPtr GetState() const override { - return actual.GetState(); - } -}; - -} // namespace InferenceEngine diff --git a/inference-engine/src/plugin_api/ie_icore.hpp b/inference-engine/src/plugin_api/ie_icore.hpp index 6c35277726f386..7f3ed86ee384b5 100644 --- a/inference-engine/src/plugin_api/ie_icore.hpp +++ b/inference-engine/src/plugin_api/ie_icore.hpp @@ -16,11 +16,14 @@ #include #include #include +#include
#include "threading/ie_itask_executor.hpp" namespace InferenceEngine { +using SoExecutableNetworkInternal = details::SOPointer; + /** * @interface ICore * @brief Minimal ICore interface to allow plugin to get information from Core Inference Engine class. @@ -63,8 +66,8 @@ class ICore { * operation * @return An executable network reference */ - virtual ExecutableNetwork LoadNetwork(const CNNNetwork& network, const std::string& deviceName, - const std::map& config = {}) = 0; + virtual SoExecutableNetworkInternal LoadNetwork(const CNNNetwork& network, const std::string& deviceName, + const std::map& config = {}) = 0; /** * @brief Creates an executable network from a model file. @@ -78,8 +81,8 @@ class ICore { * operation * @return An executable network reference */ - virtual ExecutableNetwork LoadNetwork(const std::string& modelPath, const std::string& deviceName, - const std::map& config) = 0; + virtual SoExecutableNetworkInternal LoadNetwork(const std::string& modelPath, const std::string& deviceName, + const std::map& config) = 0; /** * @brief Creates an executable network from a previously exported network @@ -89,8 +92,8 @@ class ICore { * operation* * @return An executable network reference */ - virtual ExecutableNetwork ImportNetwork(std::istream& networkModel, const std::string& deviceName = {}, - const std::map& config = {}) = 0; + virtual SoExecutableNetworkInternal ImportNetwork(std::istream& networkModel, const std::string& deviceName = {}, + const std::map& config = {}) = 0; /** * @brief Query device if it supports specified network with specified configuration diff --git a/inference-engine/src/plugin_api/ie_system_conf.h b/inference-engine/src/plugin_api/ie_system_conf.h index 28c1e155e87fa8..2e4481bf09e204 100644 --- a/inference-engine/src/plugin_api/ie_system_conf.h +++ b/inference-engine/src/plugin_api/ie_system_conf.h @@ -15,12 +15,6 @@ namespace InferenceEngine { -/** - * @brief Provides the reference to static thread_local std::exception_ptr - * @return A an exception pointer - */ -INFERENCE_ENGINE_API_CPP(std::exception_ptr&) CurrentException(); - /** * @brief Checks whether OpenMP environment variables are defined * @ingroup ie_dev_api_system_conf diff --git a/inference-engine/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp b/inference-engine/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp index 7e264217bf999f..7832894a91390e 100644 --- a/inference-engine/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp +++ b/inference-engine/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp @@ -14,16 +14,16 @@ class MockICore : public InferenceEngine::ICore { MOCK_QUALIFIED_METHOD2(ReadNetwork, const, InferenceEngine::CNNNetwork(const std::string&, const InferenceEngine::Blob::CPtr&)); MOCK_QUALIFIED_METHOD2(ReadNetwork, const, InferenceEngine::CNNNetwork(const std::string&, const std::string&)); - MOCK_METHOD3(LoadNetwork, InferenceEngine::ExecutableNetwork( + MOCK_METHOD3(LoadNetwork, InferenceEngine::SoExecutableNetworkInternal( const InferenceEngine::CNNNetwork&, const std::string&, const std::map&)); - MOCK_METHOD3(LoadNetwork, InferenceEngine::ExecutableNetwork( + MOCK_METHOD3(LoadNetwork, InferenceEngine::SoExecutableNetworkInternal( const InferenceEngine::CNNNetwork&, const InferenceEngine::RemoteContext::Ptr &, const std::map&)); - MOCK_METHOD3(LoadNetwork, InferenceEngine::ExecutableNetwork( + MOCK_METHOD3(LoadNetwork, InferenceEngine::SoExecutableNetworkInternal( const std::string &, const std::string &, const std::map&)); - MOCK_METHOD3(ImportNetwork, InferenceEngine::ExecutableNetwork( + MOCK_METHOD3(ImportNetwork, InferenceEngine::SoExecutableNetworkInternal( std::istream&, const std::string&, const std::map&)); - MOCK_METHOD3(ImportNetwork, InferenceEngine::ExecutableNetwork( + MOCK_METHOD3(ImportNetwork, InferenceEngine::SoExecutableNetworkInternal( std::istream&, const InferenceEngine::RemoteContext::Ptr&, const std::map&)); MOCK_QUALIFIED_METHOD3(QueryNetwork, const, InferenceEngine::QueryNetworkResult( diff --git a/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_infer_async_request_base_test.cpp b/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_infer_async_request_base_test.cpp index 78a1e66cf54805..d0cde74580cb1b 100644 --- a/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_infer_async_request_base_test.cpp +++ b/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_infer_async_request_base_test.cpp @@ -173,11 +173,11 @@ TEST_F(InferRequestBaseTests, canReportErrorInSetCompletionCallback) { class InferRequestTests : public ::testing::Test { protected: std::shared_ptr mock_request; - InferRequest request; + IInferRequestInternal::Ptr request; ResponseDesc dsc; std::shared_ptr mockIExeNet; - InferenceEngine::ExecutableNetwork exeNetwork; + InferenceEngine::SoExecutableNetworkInternal exeNetwork; MockIInferencePlugin* mockIPlugin; InferencePlugin plugin; shared_ptr mockInferRequestInternal; @@ -202,7 +202,7 @@ class InferRequestTests : public ::testing::Test { ON_CALL(*mockIPluginPtr, LoadNetwork(MatcherCast(_), _)).WillByDefault(Return(mockIExeNet)); plugin = InferenceEngine::InferencePlugin{{}, mockIPluginPtr}; exeNetwork = plugin.LoadNetwork(CNNNetwork{}, {}); - request = exeNetwork.CreateInferRequest(); + request = exeNetwork->CreateInferRequest(); _incorrectName = "incorrect_name"; _inputName = MockNotEmptyICNNNetwork::INPUT_BLOB_NAME; _failedToFindInOutError = @@ -213,7 +213,7 @@ class InferRequestTests : public ::testing::Test { + _inputName + "\'"; } - InferRequest getInferRequestWithMockImplInside() { + IInferRequestInternal::Ptr getInferRequestWithMockImplInside() { IInferRequest::Ptr inferRequest; InputsDataMap inputsInfo; mockNotEmptyNet.getInputsInfo(inputsInfo); @@ -226,7 +226,7 @@ class InferRequestTests : public ::testing::Test { ON_CALL(*mockIPluginPtr, LoadNetwork(MatcherCast(_), _)).WillByDefault(Return(mockIExeNet)); auto plugin = InferenceEngine::InferencePlugin{{}, mockIPluginPtr}; auto exeNetwork = plugin.LoadNetwork(CNNNetwork{}, {}); - return exeNetwork.CreateInferRequest(); + return exeNetwork->CreateInferRequest(); } std::string getExceptionMessage(std::function function) { @@ -238,71 +238,54 @@ class InferRequestTests : public ::testing::Test { } return actualError; } - - BlobMap getBlobMapWithIncorrectName() const { - Blob::Ptr Blob = make_shared_blob({ Precision::FP32, {1, 1, 1, 1}, NCHW }); - Blob->allocate(); - return BlobMap{{_incorrectName, Blob}}; - } - - BlobMap getBlobMapWithNotAllocatedInput() const { - Blob::Ptr Blob = make_shared_blob({ Precision::FP32, {1, 1, 1, 1}, NCHW }); - return BlobMap{{_inputName, Blob}}; - } - - BlobMap getBlobMapWithEmptyDimensions() const { - Blob::Ptr Blob = make_shared_blob({ Precision::FP32, {}, NCHW }); - Blob->allocate(); - return BlobMap{{_inputName, Blob}}; - } }; // StartAsync TEST_F(InferRequestTests, canForwardStartAsync) { EXPECT_CALL(*mock_request.get(), StartAsync()); - ASSERT_NO_THROW(request.StartAsync()); + ASSERT_NO_THROW(request->StartAsync()); } TEST_F(InferRequestTests, throwsIfStartAsyncReturnNotOK) { EXPECT_CALL(*mock_request.get(), StartAsync()).WillOnce(Throw(GeneralError{""})); - ASSERT_THROW(request.StartAsync(), Exception); + ASSERT_THROW(request->StartAsync(), Exception); } // Wait TEST_F(InferRequestTests, canForwardWait) { int64_t ms = 0; EXPECT_CALL(*mock_request.get(), Wait(_)).WillOnce(Return(OK)); - ASSERT_TRUE(OK == request.Wait(ms)); + ASSERT_TRUE(OK == request->Wait(ms)); } TEST_F(InferRequestTests, canForwardStatusFromWait) { EXPECT_CALL(*mock_request.get(), Wait(_)).WillOnce(Return(RESULT_NOT_READY)); - ASSERT_EQ(request.Wait(0), RESULT_NOT_READY); + ASSERT_EQ(request->Wait(0), RESULT_NOT_READY); } // Infer TEST_F(InferRequestTests, canForwardInfer) { EXPECT_CALL(*mock_request.get(), Infer()); - ASSERT_NO_THROW(request.Infer()); + ASSERT_NO_THROW(request->Infer()); } TEST_F(InferRequestTests, throwsIfInferReturnNotOK) { EXPECT_CALL(*mock_request.get(), Infer()).WillOnce(Throw(GeneralError{""})); - ASSERT_THROW(request.Infer(), Exception); + ASSERT_THROW(request->Infer(), Exception); } // GetPerformanceCounts TEST_F(InferRequestTests, canForwardGetPerformanceCounts) { std::map info; EXPECT_CALL(*mock_request.get(), GetPerformanceCounts()).WillOnce(Return(info)); - ASSERT_NO_THROW(info = request.GetPerformanceCounts()); + ASSERT_NO_THROW(info = request->GetPerformanceCounts()); } TEST_F(InferRequestTests, throwsIfGetPerformanceCountsReturnNotOK) { std::map info; EXPECT_CALL(*mock_request.get(), GetPerformanceCounts()).WillOnce(Throw(GeneralError{""})); - ASSERT_THROW(info = request.GetPerformanceCounts(), Exception); + ASSERT_THROW(info = request->GetPerformanceCounts(), Exception); } MATCHER_P(blob_in_map_pointer_is_same, ref_blob, "") { @@ -310,38 +293,6 @@ MATCHER_P(blob_in_map_pointer_is_same, ref_blob, "") { return reinterpret_cast(arg.begin()->second->buffer()) == reinterpret_cast(ref_blob->buffer()); } -// SetInput -TEST_F(InferRequestTests, getInputCallsSetBlob) { - Blob::Ptr inblob; - std::string blobName1 = "blob1"; - std::string blobName2 = "blob2"; - BlobMap blobMap{{blobName1, inblob}, - {blobName2, inblob}}; - - EXPECT_CALL(*mock_request.get(), SetBlob(blobName1, inblob)); - EXPECT_CALL(*mock_request.get(), SetBlob(blobName2, inblob)); - ASSERT_NO_THROW(request.SetInput(blobMap)); -} - -TEST_F(InferRequestTests, throwsIfSetInputReturnNotOK) { - EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(GeneralError{""})); - BlobMap blobMap{{{}, {}}}; - ASSERT_THROW(request.SetInput(blobMap), Exception); -} - -// SetOutput -TEST_F(InferRequestTests, getOutputCallsSetBlob) { - Blob::Ptr inblob; - std::string blobName1 = "blob1"; - std::string blobName2 = "blob2"; - BlobMap blobMap{{blobName1, inblob}, - {blobName2, inblob}}; - - EXPECT_CALL(*mock_request.get(), SetBlob(blobName1, inblob)); - EXPECT_CALL(*mock_request.get(), SetBlob(blobName2, inblob)); - ASSERT_NO_THROW(request.SetOutput(blobMap)); -} - // GetBlob TEST_F(InferRequestTests, canForwardGetBlob) { Blob::Ptr blob = make_shared_blob({ Precision::FP32, {}, NCHW }); @@ -349,7 +300,7 @@ TEST_F(InferRequestTests, canForwardGetBlob) { std::string name = "blob1"; EXPECT_CALL(*mock_request.get(), GetBlob(_)).WillOnce(Return(blob)); - ASSERT_NO_THROW(request.GetBlob(name)); + ASSERT_NO_THROW(request->GetBlob(name)); } TEST_F(InferRequestTests, throwsIfGetBlobReturnNotOK) { @@ -357,7 +308,7 @@ TEST_F(InferRequestTests, throwsIfGetBlobReturnNotOK) { std::string name = "blob1"; EXPECT_CALL(*mock_request.get(), GetBlob(_)).WillOnce(Throw(GeneralError{""})); - ASSERT_THROW(blob = request.GetBlob(name), Exception); + ASSERT_THROW(blob = request->GetBlob(name), Exception); } // SetBlob @@ -366,7 +317,7 @@ TEST_F(InferRequestTests, canForwardSetBlob) { std::string name = "blob1"; EXPECT_CALL(*mock_request.get(), SetBlob(name, blob)); - ASSERT_NO_THROW(request.SetBlob(name, blob)); + ASSERT_NO_THROW(request->SetBlob(name, blob)); } TEST_F(InferRequestTests, throwsIfSetBlobReturnNotOK) { @@ -374,40 +325,10 @@ TEST_F(InferRequestTests, throwsIfSetBlobReturnNotOK) { std::string name = "blob1"; EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(GeneralError{""})); - ASSERT_THROW(request.SetBlob(name, blob), Exception); -} - -TEST_F(InferRequestTests, throwsIfSetOutputReturnNotOK) { - EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(GeneralError{""})); - BlobMap blobMap{{{}, {}}}; - ASSERT_THROW(request.SetOutput(blobMap), Exception); + ASSERT_THROW(request->SetBlob(name, blob), Exception); } TEST_F(InferRequestTests, canForwardAnyCallback) { EXPECT_CALL(*mock_request.get(), SetCallback(_)); - ASSERT_NO_THROW(request.SetCompletionCallback([] {})); -} - -TEST_F(InferRequestTests, failToSetInputWithInCorrectName) { - EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(NotFound{""})); - auto blobMap = getBlobMapWithIncorrectName(); - ASSERT_THROW(request.SetInput(blobMap), NotFound); -} - -TEST_F(InferRequestTests, failToSetOutputWithInCorrectName) { - EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(NotFound{""})); - auto blobMap = getBlobMapWithIncorrectName(); - ASSERT_THROW(request.SetOutput(blobMap), NotFound); -} - -TEST_F(InferRequestTests, failToSetInputWithNotAllocatedInput) { - EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(NotAllocated{""})); - auto blobMap = getBlobMapWithNotAllocatedInput(); - ASSERT_THROW(request.SetInput(blobMap), NotAllocated); -} - -TEST_F(InferRequestTests, failToSetInputWithEmptyDimensions) { - EXPECT_CALL(*mock_request.get(), SetBlob(_, _)).WillOnce(Throw(GeneralError{""})); - auto blobMap = getBlobMapWithEmptyDimensions(); - ASSERT_THROW(request.SetInput(blobMap), GeneralError); + ASSERT_NO_THROW(request->SetCallback([] (std::exception_ptr) {})); } diff --git a/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_memory_state_internal_test.cpp b/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_memory_state_internal_test.cpp index 654fc9b4de2113..6867cb92b26cce 100644 --- a/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_memory_state_internal_test.cpp +++ b/inference-engine/tests/unit/inference_engine/cpp_interfaces/ie_memory_state_internal_test.cpp @@ -27,8 +27,8 @@ class VariableStateTests : public ::testing::Test { shared_ptr mockVariableStateInternal; MockIInferencePlugin* mockIPlugin; InferencePlugin plugin; - ExecutableNetwork net; - InferRequest req; + SoExecutableNetworkInternal net; + IInferRequestInternal::Ptr req; virtual void SetUp() { mockExeNetworkInternal = make_shared(); @@ -39,7 +39,7 @@ class VariableStateTests : public ::testing::Test { ON_CALL(*mockIPluginPtr, LoadNetwork(MatcherCast(_), _)).WillByDefault(Return(mockExeNetworkInternal)); plugin = InferenceEngine::InferencePlugin{{}, mockIPluginPtr}; net = plugin.LoadNetwork(CNNNetwork{}, {}); - req = net.CreateInferRequest(); + req = net->CreateInferRequest(); } }; @@ -50,7 +50,7 @@ TEST_F(VariableStateTests, ExecutableNetworkCanConvertOneVariableStateFromCppToA EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - auto state = net.QueryState(); + auto state = net->QueryState(); ASSERT_EQ(state.size(), 1); IE_SUPPRESS_DEPRECATED_END } @@ -61,7 +61,7 @@ TEST_F(VariableStateTests, ExecutableNetworkCanConvertZeroVariableStateFromCppTo EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).WillOnce(Return(toReturn)); - auto state = net.QueryState(); + auto state = net->QueryState(); ASSERT_EQ(state.size(), 0); IE_SUPPRESS_DEPRECATED_END } @@ -74,7 +74,7 @@ TEST_F(VariableStateTests, ExecutableNetworkCanConvert2VariableStatesFromCPPtoAP EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - auto state = net.QueryState(); + auto state = net->QueryState(); ASSERT_EQ(state.size(), 2); IE_SUPPRESS_DEPRECATED_END } @@ -87,8 +87,8 @@ TEST_F(VariableStateTests, VariableStatePropagatesReset) { EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), Reset()).Times(1); - auto state = net.QueryState(); - state.front().Reset(); + auto state = net->QueryState(); + state.front()->Reset(); IE_SUPPRESS_DEPRECATED_END } @@ -100,8 +100,8 @@ TEST_F(VariableStateTests, VariableStatePropagatesExceptionsFromReset) { EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), Reset()).WillOnce(Throw(std::logic_error("some error"))); - auto state = net.QueryState(); - EXPECT_ANY_THROW(state.front().Reset()); + auto state = net->QueryState(); + EXPECT_ANY_THROW(state.front()->Reset()); IE_SUPPRESS_DEPRECATED_END } @@ -113,8 +113,8 @@ TEST_F(VariableStateTests, VariableStatePropagatesGetName) { EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - auto state = net.QueryState(); - EXPECT_STREQ(state.front().GetName().c_str(), "someName"); + auto state = net->QueryState(); + EXPECT_STREQ(state.front()->GetName().c_str(), "someName"); IE_SUPPRESS_DEPRECATED_END } @@ -126,8 +126,8 @@ TEST_F(VariableStateTests, VariableStatePropagatesGetNameWithZeroLen) { EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - auto pState = net.QueryState().front(); - EXPECT_NO_THROW(pState.GetName()); + auto pState = net->QueryState().front(); + EXPECT_NO_THROW(pState->GetName()); IE_SUPPRESS_DEPRECATED_END } @@ -140,11 +140,10 @@ TEST_F(VariableStateTests, VariableStatePropagatesGetNameWithLenOfOne) { EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - auto pState = net.QueryState().front(); + auto pState = net->QueryState().front(); std::string name; - EXPECT_NO_THROW(name = pState.GetName()); + EXPECT_NO_THROW(name = pState->GetName()); EXPECT_EQ(name, "someName"); - EXPECT_EQ(name, ""); IE_SUPPRESS_DEPRECATED_END } @@ -156,9 +155,9 @@ TEST_F(VariableStateTests, VariableStatePropagatesGetNameWithLenOfTwo) { EXPECT_CALL(*mockExeNetworkInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - auto pState = net.QueryState().front(); + auto pState = net->QueryState().front(); std::string name; - EXPECT_NO_THROW(name = pState.GetName()); + EXPECT_NO_THROW(name = pState->GetName()); EXPECT_EQ(name, "someName"); IE_SUPPRESS_DEPRECATED_END } @@ -175,7 +174,7 @@ TEST_F(VariableStateTests, VariableStateCanPropagateSetState) { float data[] = {123, 124, 125}; auto stateBlob = make_shared_blob({ Precision::FP32, {3}, C }, data, sizeof(data) / sizeof(*data)); - EXPECT_NO_THROW(net.QueryState().front().SetState(stateBlob)); + EXPECT_NO_THROW(net->QueryState().front()->SetState(stateBlob)); ASSERT_FLOAT_EQ(saver->buffer().as()[0], 123); ASSERT_FLOAT_EQ(saver->buffer().as()[1], 124); ASSERT_FLOAT_EQ(saver->buffer().as()[2], 125); @@ -196,7 +195,7 @@ TEST_F(VariableStateTests, VariableStateCanPropagateGetLastState) { EXPECT_CALL(*mockVariableStateInternal.get(), GetState()).WillOnce(Return(stateBlob)); - auto saver = net.QueryState().front().GetState(); + auto saver = net->QueryState().front()->GetState(); ASSERT_FLOAT_EQ(saver->cbuffer().as()[0], 123); ASSERT_FLOAT_EQ(saver->cbuffer().as()[1], 124); ASSERT_FLOAT_EQ(saver->cbuffer().as()[2], 125); @@ -214,7 +213,6 @@ TEST_F(VariableStateTests, VariableStateInternalCanSaveName) { ASSERT_STREQ(pState->GetName().c_str(), "name"); } - TEST_F(VariableStateTests, VariableStateInternalCanSaveState) { IVariableStateInternal::Ptr pState(new VariableStateInternalMockImpl("name")); float data[] = {123, 124, 125}; @@ -253,7 +251,7 @@ TEST_F(VariableStateTests, InferRequestCanConvertOneVariableStateFromCppToAPI) { EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - auto state = req.QueryState(); + auto state = req->QueryState(); ASSERT_EQ(state.size(), 1); } @@ -262,7 +260,7 @@ TEST_F(VariableStateTests, InferRequestCanConvertZeroVariableStateFromCppToAPI) EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).WillOnce(Return(toReturn)); - auto state = req.QueryState(); + auto state = req->QueryState(); ASSERT_EQ(state.size(), 0); } @@ -273,7 +271,7 @@ TEST_F(VariableStateTests, InferRequestCanConvert2VariableStatesFromCPPtoAPI) { EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - auto state = req.QueryState(); + auto state = req->QueryState(); ASSERT_EQ(state.size(), 2); } @@ -284,8 +282,8 @@ TEST_F(VariableStateTests, InfReqVariableStatePropagatesReset) { EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), Reset()).Times(1); - auto state = req.QueryState(); - state.front().Reset(); + auto state = req->QueryState(); + state.front()->Reset(); } TEST_F(VariableStateTests, InfReqVariableStatePropagatesExceptionsFromReset) { @@ -295,8 +293,8 @@ TEST_F(VariableStateTests, InfReqVariableStatePropagatesExceptionsFromReset) { EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), Reset()).WillOnce(Throw(std::logic_error("some error"))); - auto state = req.QueryState(); - EXPECT_ANY_THROW(state.front().Reset()); + auto state = req->QueryState(); + EXPECT_ANY_THROW(state.front()->Reset()); } TEST_F(VariableStateTests, InfReqVariableStatePropagatesGetName) { @@ -306,58 +304,8 @@ TEST_F(VariableStateTests, InfReqVariableStatePropagatesGetName) { EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - auto state = req.QueryState(); - EXPECT_STREQ(state.front().GetName().c_str(), "someName"); -} - -TEST_F(VariableStateTests, InfReqVariableStatePropagatesGetNameWithZeroLen) { - IE_SUPPRESS_DEPRECATED_START - std::vector toReturn; - toReturn.push_back(mockVariableStateInternal); - - EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - - IVariableState::Ptr pState; - - static_cast(req)->QueryState(pState, 0, nullptr); - char *name = reinterpret_cast(1); - EXPECT_NO_THROW(pState->GetName(name, 0, nullptr)); - IE_SUPPRESS_DEPRECATED_END -} - -TEST_F(VariableStateTests, InfReqVariableStatePropagatesGetNameWithLenOfOne) { - IE_SUPPRESS_DEPRECATED_START - std::vector toReturn; - toReturn.push_back(mockVariableStateInternal); - - EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - - IVariableState::Ptr pState; - - static_cast(req)->QueryState(pState, 0, nullptr); - char name[1]; - EXPECT_NO_THROW(pState->GetName(name, 1, nullptr)); - EXPECT_STREQ(name, ""); - IE_SUPPRESS_DEPRECATED_END -} - -TEST_F(VariableStateTests, InfReqVariableStatePropagatesGetNameWithLenOfTwo) { - IE_SUPPRESS_DEPRECATED_START - std::vector toReturn; - toReturn.push_back(mockVariableStateInternal); - - EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).Times(1).WillRepeatedly(Return(toReturn)); - EXPECT_CALL(*mockVariableStateInternal.get(), GetName()).WillOnce(Return("someName")); - - IVariableState::Ptr pState; - - static_cast(req)->QueryState(pState, 0, nullptr); - char name[2]; - EXPECT_NO_THROW(pState->GetName(name, 2, nullptr)); - EXPECT_STREQ(name, "s"); - IE_SUPPRESS_DEPRECATED_END + auto state = req->QueryState(); + EXPECT_STREQ(state.front()->GetName().c_str(), "someName"); } TEST_F(VariableStateTests, InfReqVariableStateCanPropagateSetState) { @@ -371,7 +319,7 @@ TEST_F(VariableStateTests, InfReqVariableStateCanPropagateSetState) { float data[] = {123, 124, 125}; auto stateBlob = make_shared_blob({ Precision::FP32, {3}, C }, data, sizeof(data) / sizeof(*data)); - EXPECT_NO_THROW(req.QueryState().front().SetState(stateBlob)); + EXPECT_NO_THROW(req->QueryState().front()->SetState(stateBlob)); ASSERT_FLOAT_EQ(saver->buffer().as()[0], 123); ASSERT_FLOAT_EQ(saver->buffer().as()[1], 124); ASSERT_FLOAT_EQ(saver->buffer().as()[2], 125); @@ -388,7 +336,7 @@ TEST_F(VariableStateTests, InfReqVariableStateCanPropagateGetLastState) { EXPECT_CALL(*mockInferRequestInternal.get(), QueryState()).WillRepeatedly(Return(toReturn)); EXPECT_CALL(*mockVariableStateInternal.get(), GetState()).WillOnce(Return(stateBlob)); - auto saver = req.QueryState().front().GetState(); + auto saver = req->QueryState().front()->GetState(); ASSERT_FLOAT_EQ(saver->cbuffer().as()[0], 123); ASSERT_FLOAT_EQ(saver->cbuffer().as()[1], 124); ASSERT_FLOAT_EQ(saver->cbuffer().as()[2], 125); diff --git a/inference-engine/tests/unit/inference_engine/ie_executable_network_test.cpp b/inference-engine/tests/unit/inference_engine/ie_executable_network_test.cpp index fa7256e3217caa..2c35e8652dfcc0 100644 --- a/inference-engine/tests/unit/inference_engine/ie_executable_network_test.cpp +++ b/inference-engine/tests/unit/inference_engine/ie_executable_network_test.cpp @@ -39,9 +39,9 @@ using testing::SetArgReferee; class ExecutableNetworkTests : public ::testing::Test { protected: std::shared_ptr mockIExeNet; - InferenceEngine::ExecutableNetwork exeNetwork; - MockIInferencePlugin* mockIPlugin; - InferencePlugin plugin; + InferenceEngine::SoExecutableNetworkInternal exeNetwork; + MockIInferencePlugin* mockIPlugin; + InferencePlugin plugin; virtual void TearDown() { mockIExeNet.reset(); @@ -63,12 +63,13 @@ TEST_F(ExecutableNetworkTests, GetOutputsInfoThrowsIfReturnErr) { .Times(1) .WillOnce(Throw(InferenceEngine::GeneralError{""})); - ASSERT_THROW(exeNetwork.GetOutputsInfo(), InferenceEngine::Exception); + ASSERT_THROW(exeNetwork->GetOutputsInfo(), InferenceEngine::Exception); } TEST_F(ExecutableNetworkTests, GetOutputsInfo) { InferenceEngine::ConstOutputsDataMap data; EXPECT_CALL(*mockIExeNet.get(), GetOutputsInfo()).Times(1).WillRepeatedly(Return(InferenceEngine::ConstOutputsDataMap{})); + ASSERT_NO_THROW(data = exeNetwork->GetOutputsInfo()); ASSERT_EQ(data, InferenceEngine::ConstOutputsDataMap{}); } @@ -77,29 +78,24 @@ TEST_F(ExecutableNetworkTests, GetInputsInfoThrowsIfReturnErr) { .Times(1) .WillOnce(Throw(InferenceEngine::GeneralError{""})); - ASSERT_THROW(exeNetwork.GetInputsInfo(), InferenceEngine::Exception); + ASSERT_THROW(exeNetwork->GetInputsInfo(), InferenceEngine::Exception); } TEST_F(ExecutableNetworkTests, GetInputsInfo) { EXPECT_CALL(*mockIExeNet.get(), GetInputsInfo()).Times(1).WillRepeatedly(Return(InferenceEngine::ConstInputsDataMap{})); InferenceEngine::ConstInputsDataMap info; - ASSERT_NO_THROW(info = exeNetwork.GetInputsInfo()); + ASSERT_NO_THROW(info = exeNetwork->GetInputsInfo()); ASSERT_EQ(info, InferenceEngine::ConstInputsDataMap{}); } IE_SUPPRESS_DEPRECATED_START -TEST_F(ExecutableNetworkTests, resetThrowsIfResetToNullptr) { - InferenceEngine::IExecutableNetwork::Ptr mockIExeNet_2{}; - ASSERT_THROW(exeNetwork.reset(mockIExeNet_2), InferenceEngine::Exception); -} - TEST_F(ExecutableNetworkTests, QueryStateThrowsIfReturnErr) { EXPECT_CALL(*mockIExeNet.get(), QueryState()) .Times(1) .WillOnce(Throw(InferenceEngine::GeneralError{""})); - EXPECT_THROW(exeNetwork.QueryState(), InferenceEngine::Exception); + EXPECT_THROW(exeNetwork->QueryState(), InferenceEngine::Exception); } TEST_F(ExecutableNetworkTests, QueryState) { @@ -107,8 +103,8 @@ TEST_F(ExecutableNetworkTests, QueryState) { EXPECT_CALL(*mockIExeNet.get(), QueryState()) .Times(1) .WillOnce(Return(std::vector>(1, mockIMemState_p))); - std::vector MemState_v; - EXPECT_NO_THROW(MemState_v = exeNetwork.QueryState()); + std::vector MemState_v; + EXPECT_NO_THROW(MemState_v = exeNetwork->QueryState()); EXPECT_EQ(MemState_v.size(), 1); } @@ -131,39 +127,17 @@ class ExecutableNetworkWithIInferReqTests : public ExecutableNetworkTests { TEST_F(ExecutableNetworkWithIInferReqTests, CanCreateInferRequest) { EXPECT_CALL(*mockIExeNet.get(), CreateInferRequest()).WillOnce(Return(mockIInferReq_p)); - InferRequest actualInferReq; - ASSERT_NO_THROW(actualInferReq = exeNetwork.CreateInferRequest()); + IInferRequestInternal::Ptr actualInferReq; + ASSERT_NO_THROW(actualInferReq = exeNetwork->CreateInferRequest()); } TEST_F(ExecutableNetworkWithIInferReqTests, CreateInferRequestThrowsIfReturnNotOK) { EXPECT_CALL(*mockIExeNet.get(), CreateInferRequest()).WillOnce(Throw(InferenceEngine::GeneralError{""})); - ASSERT_THROW(exeNetwork.CreateInferRequest(), InferenceEngine::Exception); -} - -TEST_F(ExecutableNetworkWithIInferReqTests, CreateInferRequestThrowsIfSetRequestToNullptr) { - EXPECT_CALL(*mockIExeNet.get(), CreateInferRequest()) - .WillOnce(Return(std::shared_ptr{})); - ASSERT_THROW(exeNetwork.CreateInferRequest(), InferenceEngine::Exception); + ASSERT_THROW(exeNetwork->CreateInferRequest(), InferenceEngine::Exception); } IE_SUPPRESS_DEPRECATED_START -// CreateInferRequestPtr -TEST_F(ExecutableNetworkWithIInferReqTests, CanCreateInferRequestPtr) { - EXPECT_CALL(*mockIExeNet.get(), CreateInferRequest()).WillOnce(Return(mockIInferReq_p)); - ASSERT_NO_THROW(exeNetwork.CreateInferRequestPtr()); -} - -TEST_F(ExecutableNetworkWithIInferReqTests, CreateInferRequestPtrThrowsIfReturnNotOK) { - EXPECT_CALL(*mockIExeNet.get(), CreateInferRequest()).WillOnce(Throw(InferenceEngine::GeneralError{""})); - ASSERT_THROW(exeNetwork.CreateInferRequestPtr(), InferenceEngine::Exception); -} - -TEST_F(ExecutableNetworkWithIInferReqTests, CreateInferRequestPtrThrowsIfSetRequestToNullptr) { - EXPECT_CALL(*mockIExeNet.get(), CreateInferRequest()).WillOnce(Return(std::shared_ptr{})); - ASSERT_THROW(exeNetwork.CreateInferRequestPtr(), InferenceEngine::Exception); -} - class ExecutableNetworkBaseTests : public ::testing::Test { protected: std::shared_ptr mock_impl; @@ -217,5 +191,3 @@ TEST_F(ExecutableNetworkBaseTests, canCatchUnknownErrorInExport) { EXPECT_CALL(*mock_impl.get(), Export(_)).WillOnce(Throw(5)); ASSERT_EQ(UNEXPECTED, exeNetwork->Export({}, nullptr)); } - -