diff --git a/runtime/bindings/python/src/openvino/__init__.py b/runtime/bindings/python/src/openvino/__init__.py index 9ece8649f652e2..5634960d064d8c 100644 --- a/runtime/bindings/python/src/openvino/__init__.py +++ b/runtime/bindings/python/src/openvino/__init__.py @@ -15,8 +15,7 @@ from openvino.ie_api import BlobWrapper from openvino.ie_api import infer -from openvino.ie_api import async_infer -from openvino.ie_api import get_result +from openvino.ie_api import start_async from openvino.ie_api import blob_from_file from openvino.ie_api import tensor_from_file @@ -37,7 +36,7 @@ from openvino.pyopenvino import TensorDesc from openvino.pyopenvino import get_version from openvino.pyopenvino import StatusCode -from openvino.pyopenvino import InferQueue +#from openvino.pyopenvino import InferQueue from openvino.pyopenvino import InferRequest # TODO: move to ie_api? from openvino.pyopenvino import Blob from openvino.pyopenvino import PreProcessInfo @@ -82,7 +81,6 @@ ExecutableNetwork.infer = infer # Patching InferRequest InferRequest.infer = infer -InferRequest.async_infer = async_infer -InferRequest.get_result = get_result +InferRequest.start_async = start_async # Patching InferQueue -InferQueue.async_infer = async_infer +#InferQueue.async_infer = async_infer diff --git a/runtime/bindings/python/src/openvino/ie_api.py b/runtime/bindings/python/src/openvino/ie_api.py index 29588032458062..7a208a0fc926ff 100644 --- a/runtime/bindings/python/src/openvino/ie_api.py +++ b/runtime/bindings/python/src/openvino/ie_api.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +import copy from openvino.pyopenvino import TBlobFloat32 from openvino.pyopenvino import TBlobFloat64 @@ -36,22 +37,17 @@ def normalize_inputs(py_dict: dict) -> dict: """Normalize a dictionary of inputs to contiguous numpy arrays.""" - return {k: (np.ascontiguousarray(v) if isinstance(v, np.ndarray) else v) + return {k: (Tensor(v) if isinstance(v, np.ndarray) else v) for k, v in py_dict.items()} # flake8: noqa: D102 -def infer(request: InferRequest, inputs: dict = None) -> dict: - results = request._infer(inputs=normalize_inputs(inputs if inputs is not None else {})) - return {name: (blob.buffer.copy()) for name, blob in results.items()} +def infer(request: InferRequest, inputs: dict = None) -> np.ndarray: + res = request._infer(inputs=normalize_inputs(inputs if inputs is not None else {})) + return np.asarray([copy.deepcopy(tensor.data) for tensor in res]) # flake8: noqa: D102 -def get_result(request: InferRequest, name: str) -> np.ndarray: - return request.get_blob(name).buffer.copy() - -# flake8: noqa: D102 -def async_infer(request: InferRequest, inputs: dict = None, userdata=None) -> None: # type: ignore - request._async_infer(inputs=normalize_inputs(inputs if inputs is not None else {}), - userdata=userdata) +def start_async(request: InferRequest, inputs: dict = None) -> None: # type: ignore + request._start_async(inputs=normalize_inputs(inputs if inputs is not None else {})) # flake8: noqa: C901 # Dispatch Blob types on Python side. diff --git a/runtime/bindings/python/src/pyopenvino/core/common.hpp b/runtime/bindings/python/src/pyopenvino/core/common.hpp index 314a8290244581..f9ca68fdabc982 100644 --- a/runtime/bindings/python/src/pyopenvino/core/common.hpp +++ b/runtime/bindings/python/src/pyopenvino/core/common.hpp @@ -53,4 +53,5 @@ namespace Common void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary); uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual); + }; // namespace Common diff --git a/runtime/bindings/python/src/pyopenvino/core/containers.cpp b/runtime/bindings/python/src/pyopenvino/core/containers.cpp index 096b6074325815..e91765e8f73aec 100644 --- a/runtime/bindings/python/src/pyopenvino/core/containers.cpp +++ b/runtime/bindings/python/src/pyopenvino/core/containers.cpp @@ -8,44 +8,18 @@ #include #include -PYBIND11_MAKE_OPAQUE(Containers::PyInputsDataMap); -PYBIND11_MAKE_OPAQUE(Containers::PyConstInputsDataMap); -PYBIND11_MAKE_OPAQUE(Containers::PyOutputsDataMap); -PYBIND11_MAKE_OPAQUE(Containers::PyResults); +PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap); +PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap); namespace py = pybind11; namespace Containers { -void regclass_PyInputsDataMap(py::module m) { - auto py_inputs_data_map = py::bind_map(m, "PyInputsDataMap"); - - py_inputs_data_map.def("keys", [](PyInputsDataMap& self) { - return py::make_key_iterator(self.begin(), self.end()); - }); -} - -void regclass_PyConstInputsDataMap(py::module m) { - auto py_const_inputs_data_map = py::bind_map(m, "PyConstInputsDataMap"); - - py_const_inputs_data_map.def("keys", [](PyConstInputsDataMap& self) { - return py::make_key_iterator(self.begin(), self.end()); - }); +void regclass_TensorIndexMap(py::module m) { + auto tensor_index_map = py::bind_map(m, "TensorIndexMap"); } -void regclass_PyOutputsDataMap(py::module m) { - auto py_outputs_data_map = py::bind_map(m, "PyOutputsDataMap"); - - py_outputs_data_map.def("keys", [](PyOutputsDataMap& self) { - return py::make_key_iterator(self.begin(), self.end()); - }); -} - -void regclass_PyResults(py::module m) { - auto py_results = py::bind_map(m, "PyResults"); - - py_results.def("keys", [](PyResults& self) { - return py::make_key_iterator(self.begin(), self.end()); - }); +void regclass_TensorNameMap(py::module m) { + auto tensor_name_map = py::bind_map(m, "TensorNameMap"); } } // namespace Containers diff --git a/runtime/bindings/python/src/pyopenvino/core/containers.hpp b/runtime/bindings/python/src/pyopenvino/core/containers.hpp index 511d9053ea50fa..30a8fac6440403 100644 --- a/runtime/bindings/python/src/pyopenvino/core/containers.hpp +++ b/runtime/bindings/python/src/pyopenvino/core/containers.hpp @@ -5,27 +5,17 @@ #pragma once #include +#include #include -#include -#include "ie_data.h" -#include "ie_blob.h" +#include namespace py = pybind11; namespace Containers { - using PyInputsDataMap = std::map>; + using TensorIndexMap = std::map; + using TensorNameMap = std::map; + using InferResults = std::vector; - using PyConstInputsDataMap = - std::map>; - - using PyOutputsDataMap = - std::map>; - - using PyResults = - std::map>; - - void regclass_PyInputsDataMap(py::module m); - void regclass_PyConstInputsDataMap(py::module m); - void regclass_PyOutputsDataMap(py::module m); - void regclass_PyResults(py::module m); -} \ No newline at end of file + void regclass_TensorIndexMap(py::module m); + void regclass_TensorNameMap(py::module m); +} diff --git a/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp b/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp index 82817b68484f74..a5b04aab2c18cb 100644 --- a/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp +++ b/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp @@ -7,8 +7,8 @@ #include "common.hpp" #include "pyopenvino/core/containers.hpp" -#include "pyopenvino/core/ie_infer_request.hpp" #include "pyopenvino/core/ie_input_info.hpp" +#include "pyopenvino/core/infer_request.hpp" namespace py = pybind11; @@ -17,7 +17,9 @@ void regclass_ExecutableNetwork(py::module m) { m, "ExecutableNetwork"); - cls.def("create_infer_request", &ov::runtime::ExecutableNetwork::create_infer_request); + cls.def("create_infer_request", [](ov::runtime::ExecutableNetwork& self) { + return InferRequestWrapper(self.create_infer_request(), self.inputs(), self.outputs()); + }); // cls.def("infer_new_request", [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) { // TODO: implment after https://github.com/openvinotoolkit/openvino/pull/7962 diff --git a/runtime/bindings/python/src/pyopenvino/core/ie_infer_queue.cpp b/runtime/bindings/python/src/pyopenvino/core/ie_infer_queue.cpp index d54e7cce69c9ff..e80cd33105f01b 100644 --- a/runtime/bindings/python/src/pyopenvino/core/ie_infer_queue.cpp +++ b/runtime/bindings/python/src/pyopenvino/core/ie_infer_queue.cpp @@ -18,7 +18,7 @@ #include #include "pyopenvino/core/common.hpp" -#include "pyopenvino/core/ie_infer_request.hpp" +#include "pyopenvino/core/infer_request.hpp" #define INVALID_ID -1 @@ -59,16 +59,9 @@ class InferQueue { size_t request_id = _idle_handles.front(); - InferenceEngine::StatusCode status = - _requests[request_id]._request.Wait(InferenceEngine::IInferRequest::WaitMode::STATUS_ONLY); - - if (status == InferenceEngine::StatusCode::RESULT_NOT_READY) { - status = _requests[request_id]._request.Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); - } - py::dict request_info = py::dict(); request_info["id"] = request_id; - request_info["status"] = status; + // request_info["status"] = true; // TODO return request_info; } @@ -87,7 +80,7 @@ class InferQueue { return idle_request_id; } - std::vector waitAll() { + std::vector waitAll() { // Wait for all requests to return with callback thus updating // _idle_handles so it matches the size of requests py::gil_scoped_release release; @@ -96,10 +89,10 @@ class InferQueue { return _idle_handles.size() == _requests.size(); }); - std::vector statuses; + std::vector statuses; for (size_t handle = 0; handle < _requests.size(); handle++) { - statuses.push_back(_requests[handle]._request.Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY)); + statuses.push_back(_requests[handle]._request.wait_for(std::chrono::milliseconds(0))); } return statuses; @@ -107,8 +100,8 @@ class InferQueue { void setDefaultCallbacks() { for (size_t handle = 0; handle < _requests.size(); handle++) { - _requests[handle]._request.SetCompletionCallback([this, handle /* ... */]() { - _requests[handle]._endTime = Time::now(); + _requests[handle]._request.set_callback([this, handle /* ... */](std::exception_ptr exception_ptr) { + _requests[handle]._end_time = Time::now(); // Add idle handle to queue _idle_handles.push(handle); // Notify locks in getIdleRequestId() or waitAll() functions @@ -119,16 +112,18 @@ class InferQueue { void setCustomCallbacks(py::function f_callback) { for (size_t handle = 0; handle < _requests.size(); handle++) { - _requests[handle]._request.SetCompletionCallback([this, f_callback, handle /* ... */]() { - _requests[handle]._endTime = Time::now(); - InferenceEngine::StatusCode statusCode = - _requests[handle]._request.Wait(InferenceEngine::IInferRequest::WaitMode::STATUS_ONLY); - if (statusCode == InferenceEngine::StatusCode::RESULT_NOT_READY) { - statusCode = InferenceEngine::StatusCode::OK; + _requests[handle]._request.set_callback([this, f_callback, handle](std::exception_ptr exception_ptr) { + _requests[handle]._end_time = Time::now(); + try { + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } + } catch (const std::exception& e) { + IE_THROW() << "Caught exception: " << e.what(); } // Acquire GIL, execute Python function py::gil_scoped_acquire acquire; - f_callback(_requests[handle], statusCode, _user_ids[handle]); + f_callback(_requests[handle], _user_ids[handle]); // Add idle handle to queue _idle_handles.push(handle); // Notify locks in getIdleRequestId() or waitAll() functions @@ -145,89 +140,89 @@ class InferQueue { std::condition_variable _cv; }; -void regclass_InferQueue(py::module m) { - py::class_> cls(m, "InferQueue"); - - cls.def(py::init([](InferenceEngine::ExecutableNetwork& net, size_t jobs) { - if (jobs == 0) { - const InferenceEngine::ExecutableNetwork& _net = net; - jobs = (size_t)Common::get_optimal_number_of_requests(_net); - } - - std::vector requests; - std::queue idle_handles; - std::vector user_ids(jobs); - - for (size_t handle = 0; handle < jobs; handle++) { - auto request = InferRequestWrapper(net.CreateInferRequest()); - // Get Inputs and Outputs info from executable network - request._inputsInfo = net.GetInputsInfo(); - request._outputsInfo = net.GetOutputsInfo(); - - requests.push_back(request); - idle_handles.push(handle); - } - - return new InferQueue(requests, idle_handles, user_ids); - }), - py::arg("network"), - py::arg("jobs") = 0); - - cls.def( - "_async_infer", - [](InferQueue& self, const py::dict inputs, py::object userdata) { - // getIdleRequestId function has an intention to block InferQueue - // until there is at least one idle (free to use) InferRequest - auto handle = self.getIdleRequestId(); - // Set new inputs label/id from user - self._user_ids[handle] = userdata; - // Update inputs of picked InferRequest - if (!inputs.empty()) { - Common::set_request_blobs(self._requests[handle]._request, inputs); - } - // Now GIL can be released - we are NOT working with Python objects in this block - { - py::gil_scoped_release release; - self._requests[handle]._startTime = Time::now(); - // Start InferRequest in asynchronus mode - self._requests[handle]._request.StartAsync(); - } - }, - py::arg("inputs"), - py::arg("userdata")); - - cls.def("is_ready", [](InferQueue& self) { - return self._is_ready(); - }); - - cls.def("wait_all", [](InferQueue& self) { - return self.waitAll(); - }); - - cls.def("get_idle_request_info", [](InferQueue& self) { - return self._getIdleRequestInfo(); - }); - - cls.def("set_infer_callback", [](InferQueue& self, py::function f_callback) { - self.setCustomCallbacks(f_callback); - }); - - cls.def("__len__", [](InferQueue& self) { - return self._requests.size(); - }); - - cls.def( - "__iter__", - [](InferQueue& self) { - return py::make_iterator(self._requests.begin(), self._requests.end()); - }, - py::keep_alive<0, 1>()); /* Keep set alive while iterator is used */ - - cls.def("__getitem__", [](InferQueue& self, size_t i) { - return self._requests[i]; - }); - - cls.def_property_readonly("userdata", [](InferQueue& self) { - return self._user_ids; - }); -} +// void regclass_InferQueue(py::module m) { +// py::class_> cls(m, "InferQueue"); + +// cls.def(py::init([](InferenceEngine::ExecutableNetwork& net, size_t jobs) { +// if (jobs == 0) { +// const InferenceEngine::ExecutableNetwork& _net = net; +// jobs = (size_t)Common::get_optimal_number_of_requests(_net); +// } + +// std::vector requests; +// std::queue idle_handles; +// std::vector user_ids(jobs); + +// for (size_t handle = 0; handle < jobs; handle++) { +// auto request = InferRequestWrapper(net.CreateInferRequest()); +// // Get Inputs and Outputs info from executable network +// request._inputsInfo = net.GetInputsInfo(); +// request._outputsInfo = net.GetOutputsInfo(); + +// requests.push_back(request); +// idle_handles.push(handle); +// } + +// return new InferQueue(requests, idle_handles, user_ids); +// }), +// py::arg("network"), +// py::arg("jobs") = 0); + +// cls.def( +// "_async_infer", +// [](InferQueue& self, const py::dict inputs, py::object userdata) { +// // getIdleRequestId function has an intention to block InferQueue +// // until there is at least one idle (free to use) InferRequest +// auto handle = self.getIdleRequestId(); +// // Set new inputs label/id from user +// self._user_ids[handle] = userdata; +// // Update inputs of picked InferRequest +// if (!inputs.empty()) { +// Common::set_request_blobs(self._requests[handle]._request, inputs); +// } +// // Now GIL can be released - we are NOT working with Python objects in this block +// { +// py::gil_scoped_release release; +// self._requests[handle]._start_time = Time::now(); +// // Start InferRequest in asynchronus mode +// self._requests[handle]._request.start_async(); +// } +// }, +// py::arg("inputs"), +// py::arg("userdata")); + +// cls.def("is_ready", [](InferQueue& self) { +// return self._is_ready(); +// }); + +// cls.def("wait_all", [](InferQueue& self) { +// return self.waitAll(); +// }); + +// cls.def("get_idle_request_info", [](InferQueue& self) { +// return self._getIdleRequestInfo(); +// }); + +// cls.def("set_infer_callback", [](InferQueue& self, py::function f_callback) { +// self.setCustomCallbacks(f_callback); +// }); + +// cls.def("__len__", [](InferQueue& self) { +// return self._requests.size(); +// }); + +// cls.def( +// "__iter__", +// [](InferQueue& self) { +// return py::make_iterator(self._requests.begin(), self._requests.end()); +// }, +// py::keep_alive<0, 1>()); /* Keep set alive while iterator is used */ + +// cls.def("__getitem__", [](InferQueue& self, size_t i) { +// return self._requests[i]; +// }); + +// cls.def_property_readonly("userdata", [](InferQueue& self) { +// return self._user_ids; +// }); +// } diff --git a/runtime/bindings/python/src/pyopenvino/core/ie_infer_request.cpp b/runtime/bindings/python/src/pyopenvino/core/ie_infer_request.cpp deleted file mode 100644 index 52d250f92ee7b7..00000000000000 --- a/runtime/bindings/python/src/pyopenvino/core/ie_infer_request.cpp +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright (C) 2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#include "pyopenvino/core/ie_infer_request.hpp" - -#include -#include - -#include - -#include "pyopenvino/core/common.hpp" -#include "pyopenvino/core/containers.hpp" -#include "pyopenvino/core/executable_network.hpp" -#include "pyopenvino/core/ie_preprocess_info.hpp" - -namespace py = pybind11; - -void regclass_InferRequest(py::module m) { - py::class_> cls(m, "InferRequest"); - - cls.def( - "set_batch", - [](InferRequestWrapper& self, const int size) { - self._request.SetBatch(size); - }, - py::arg("size")); - - cls.def( - "get_blob", - [](InferRequestWrapper& self, const std::string& name) { - return self._request.GetBlob(name); - }, - py::arg("name")); - - cls.def( - "set_blob", - [](InferRequestWrapper& self, const std::string& name, py::handle& blob) { - self._request.SetBlob(name, Common::cast_to_blob(blob)); - }, - py::arg("name"), - py::arg("blob")); - - cls.def( - "set_blob", - [](InferRequestWrapper& self, - const std::string& name, - py::handle& blob, - const InferenceEngine::PreProcessInfo& info) { - self._request.SetBlob(name, Common::cast_to_blob(blob)); - }, - py::arg("name"), - py::arg("blob"), - py::arg("info")); - - cls.def( - "set_input", - [](InferRequestWrapper& self, const py::dict& inputs) { - Common::set_request_blobs(self._request, inputs); - }, - py::arg("inputs")); - - cls.def( - "set_output", - [](InferRequestWrapper& self, const py::dict& results) { - Common::set_request_blobs(self._request, results); - }, - py::arg("results")); - - cls.def( - "_infer", - [](InferRequestWrapper& self, const py::dict& inputs) { - // Update inputs if there are any - if (!inputs.empty()) { - Common::set_request_blobs(self._request, inputs); - } - // Call Infer function - self._startTime = Time::now(); - self._request.Infer(); - self._endTime = Time::now(); - // Get output Blobs and return - Containers::PyResults results; - for (auto& out : self._outputsInfo) { - results[out.first] = self._request.GetBlob(out.first); - } - return results; - }, - py::arg("inputs")); - - cls.def( - "_async_infer", - [](InferRequestWrapper& self, const py::dict inputs, py::object userdata) { - py::gil_scoped_release release; - if (!inputs.empty()) { - Common::set_request_blobs(self._request, inputs); - } - // TODO: check for None so next async infer userdata can be updated - // if (!userdata.empty()) - // { - // if (user_callback_defined) - // { - // self._request.SetCompletionCallback([self, userdata]() { - // // py::gil_scoped_acquire acquire; - // auto statusCode = const_cast(self).Wait( - // InferenceEngine::IInferRequest::WaitMode::STATUS_ONLY); - // self._request.user_callback(self, statusCode, userdata); - // // py::gil_scoped_release release; - // }); - // } - // else - // { - // py::print("There is no callback function!"); - // } - // } - self._startTime = Time::now(); - self._request.StartAsync(); - }, - py::arg("inputs"), - py::arg("userdata")); - - cls.def("cancel", [](InferRequestWrapper& self) { - self._request.Cancel(); - }); - - cls.def( - "wait", - [](InferRequestWrapper& self, int64_t millis_timeout) { - py::gil_scoped_release release; - return self._request.Wait(millis_timeout); - }, - py::arg("millis_timeout") = InferenceEngine::IInferRequest::WaitMode::RESULT_READY); - - cls.def( - "set_completion_callback", - [](InferRequestWrapper& self, py::function f_callback, py::object userdata) { - self._request.SetCompletionCallback([&self, f_callback, userdata]() { - self._endTime = Time::now(); - InferenceEngine::StatusCode statusCode = - self._request.Wait(InferenceEngine::IInferRequest::WaitMode::STATUS_ONLY); - if (statusCode == InferenceEngine::StatusCode::RESULT_NOT_READY) { - statusCode = InferenceEngine::StatusCode::OK; - } - // Acquire GIL, execute Python function - py::gil_scoped_acquire acquire; - f_callback(self, statusCode, userdata); - }); - }, - py::arg("f_callback"), - py::arg("userdata")); - - cls.def("get_perf_counts", [](InferRequestWrapper& self) { - std::map perfMap; - perfMap = self._request.GetPerformanceCounts(); - py::dict perf_map; - - for (auto it : perfMap) { - py::dict profile_info; - switch (it.second.status) { - case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: - profile_info["status"] = "EXECUTED"; - break; - case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN: - profile_info["status"] = "NOT_RUN"; - break; - case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT: - profile_info["status"] = "OPTIMIZED_OUT"; - break; - default: - profile_info["status"] = "UNKNOWN"; - } - profile_info["exec_type"] = it.second.exec_type; - profile_info["layer_type"] = it.second.layer_type; - profile_info["cpu_time"] = it.second.cpu_uSec; - profile_info["real_time"] = it.second.realTime_uSec; - profile_info["execution_index"] = it.second.execution_index; - perf_map[it.first.c_str()] = profile_info; - } - return perf_map; - }); - - cls.def( - "preprocess_info", - [](InferRequestWrapper& self, const std::string& name) { - return self._request.GetPreProcess(name); - }, - py::arg("name")); - - // cls.def_property_readonly("preprocess_info", [](InferRequestWrapper& self) { - // - // }); - - cls.def_property_readonly("input_blobs", [](InferRequestWrapper& self) { - Containers::PyResults input_blobs; - for (auto& in : self._inputsInfo) { - input_blobs[in.first] = self._request.GetBlob(in.first); - } - return input_blobs; - }); - - cls.def_property_readonly("output_blobs", [](InferRequestWrapper& self) { - Containers::PyResults output_blobs; - for (auto& out : self._outputsInfo) { - output_blobs[out.first] = self._request.GetBlob(out.first); - } - return output_blobs; - }); - - cls.def_property_readonly("latency", [](InferRequestWrapper& self) { - return self.getLatency(); - }); -} diff --git a/runtime/bindings/python/src/pyopenvino/core/ie_infer_request.hpp b/runtime/bindings/python/src/pyopenvino/core/ie_infer_request.hpp deleted file mode 100644 index 13afbac440360d..00000000000000 --- a/runtime/bindings/python/src/pyopenvino/core/ie_infer_request.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2021 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include - -#include -#include -#include - -namespace py = pybind11; - -typedef std::chrono::high_resolution_clock Time; -typedef std::chrono::nanoseconds ns; - -class InferRequestWrapper { -public: - InferRequestWrapper(InferenceEngine::InferRequest request) - : _request(request) - { - } - // ~InferRequestWrapper() = default; - - // bool user_callback_defined; - // py::function user_callback; - - double getLatency() { - auto execTime = std::chrono::duration_cast(_endTime - _startTime); - return static_cast(execTime.count()) * 0.000001; - } - - InferenceEngine::InferRequest _request; - InferenceEngine::ConstInputsDataMap _inputsInfo; - InferenceEngine::ConstOutputsDataMap _outputsInfo; - Time::time_point _startTime; - Time::time_point _endTime; -}; - -void regclass_InferRequest(py::module m); diff --git a/runtime/bindings/python/src/pyopenvino/core/ie_network.cpp b/runtime/bindings/python/src/pyopenvino/core/ie_network.cpp index b57765f19cf55c..e06f9bf79bb4c3 100644 --- a/runtime/bindings/python/src/pyopenvino/core/ie_network.cpp +++ b/runtime/bindings/python/src/pyopenvino/core/ie_network.cpp @@ -79,15 +79,6 @@ void regclass_IENetwork(py::module m) { &InferenceEngine::CNNNetwork::getBatchSize, &InferenceEngine::CNNNetwork::setBatchSize); - cls.def_property_readonly("input_info", [](InferenceEngine::CNNNetwork& self) { - Containers::PyInputsDataMap inputs; - const InferenceEngine::InputsDataMap& inputsInfo = self.getInputsInfo(); - for (auto& in : inputsInfo) { - inputs[in.first] = in.second; - } - return inputs; - }); - cls.def_property_readonly("outputs", [](InferenceEngine::CNNNetwork& self) { return self.getOutputsInfo(); }); diff --git a/runtime/bindings/python/src/pyopenvino/core/infer_request.cpp b/runtime/bindings/python/src/pyopenvino/core/infer_request.cpp new file mode 100644 index 00000000000000..190fbe0d07b053 --- /dev/null +++ b/runtime/bindings/python/src/pyopenvino/core/infer_request.cpp @@ -0,0 +1,266 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#include "pyopenvino/core/infer_request.hpp" + +#include +#include +#include +#include + +#include + +#include "pyopenvino/core/containers.hpp" + +namespace py = pybind11; + +void regclass_InferRequest(py::module m) { + py::class_> cls(m, "InferRequest"); + cls.def( + "set_tensors", + [](InferRequestWrapper& self, const Containers::TensorNameMap& inputs) { + for (auto&& input : inputs) { + self._request.set_tensor(input.first, input.second); + } + }, + py::arg("inputs")); + + cls.def( + "set_output_tensors", + [](InferRequestWrapper& self, const Containers::TensorIndexMap& outputs) { + for (auto&& output : outputs) { + self._request.set_output_tensor(output.first, output.second); + } + }, + py::arg("outputs")); + + cls.def( + "set_input_tensors", + [](InferRequestWrapper& self, const Containers::TensorIndexMap& inputs) { + for (auto&& input : inputs) { + self._request.set_input_tensor(input.first, input.second); + } + }, + py::arg("inputs")); + + cls.def( + "_infer", + [](InferRequestWrapper& self, const Containers::TensorIndexMap& inputs) { + // Update inputs if there are any + for (auto&& input : inputs) { + self._request.set_input_tensor(input.first, input.second); + } + // Call Infer function + self._start_time = Time::now(); + self._request.infer(); + self._end_time = Time::now(); + Containers::InferResults results; + for (auto& out : self._outputs) { + results.push_back(self._request.get_tensor(out)); + } + return results; + }, + py::arg("inputs")); + + cls.def( + "_infer", + [](InferRequestWrapper& self, const Containers::TensorNameMap& inputs) { + // Update inputs if there are any + for (auto&& input : inputs) { + self._request.set_tensor(input.first, input.second); + } + // Call Infer function + self._start_time = Time::now(); + self._request.infer(); + self._end_time = Time::now(); + Containers::InferResults results; + for (auto& out : self._outputs) { + results.push_back(self._request.get_tensor(out)); + } + return results; + }, + py::arg("inputs")); + + cls.def( + "_start_async", + [](InferRequestWrapper& self, const Containers::TensorIndexMap& inputs) { + py::gil_scoped_release release; + for (auto&& input : inputs) { + self._request.set_input_tensor(input.first, input.second); + } + // TODO: check for None so next async infer userdata can be updated + // if (!userdata.empty()) + // { + // if (user_callback_defined) + // { + // self._request.SetCompletionCallback([self, userdata]() { + // // py::gil_scoped_acquire acquire; + // auto statusCode = const_cast(self).Wait( + // InferenceEngine::IInferRequest::WaitMode::STATUS_ONLY); + // self._request.user_callback(self, statusCode, userdata); + // // py::gil_scoped_release release; + // }); + // } + // else + // { + // py::print("There is no callback function!"); + // } + // } + self._start_time = Time::now(); + self._request.start_async(); + }, + py::arg("inputs")); + + cls.def( + "_start_async", + [](InferRequestWrapper& self, const Containers::TensorNameMap& inputs) { + py::gil_scoped_release release; + for (auto&& input : inputs) { + self._request.set_tensor(input.first, input.second); + } + // TODO: check for None so next async infer userdata can be updated + // if (!userdata.empty()) + // { + // if (user_callback_defined) + // { + // self._request.SetCompletionCallback([self, userdata]() { + // // py::gil_scoped_acquire acquire; + // auto statusCode = const_cast(self).Wait( + // InferenceEngine::IInferRequest::WaitMode::STATUS_ONLY); + // self._request.user_callback(self, statusCode, userdata); + // // py::gil_scoped_release release; + // }); + // } + // else + // { + // py::print("There is no callback function!"); + // } + // } + self._start_time = Time::now(); + self._request.start_async(); + }, + py::arg("inputs")); + + cls.def("cancel", [](InferRequestWrapper& self) { + self._request.cancel(); + }); + + cls.def("wait", [](InferRequestWrapper& self) { + py::gil_scoped_release release; + self._request.wait(); + }); + + cls.def( + "wait_for", + [](InferRequestWrapper& self, const int timeout) { + py::gil_scoped_release release; + return self._request.wait_for(std::chrono::milliseconds(timeout)); + }, + py::arg("timeout")); + + cls.def( + "set_callback", + [](InferRequestWrapper& self, py::function f_callback) { + self._request.set_callback([&self, f_callback](std::exception_ptr exception_ptr) { + self._end_time = Time::now(); + try { + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } + } catch (const std::exception& e) { + IE_THROW() << "Caught exception: " << e.what(); + } + // Acquire GIL, execute Python function + py::gil_scoped_acquire acquire; + f_callback(exception_ptr); + }); + }, + py::arg("f_callback")); + + cls.def( + "get_tensor", + [](InferRequestWrapper& self, const std::string& name) { + return self._request.get_tensor(name); + }, + py::arg("name")); + + cls.def( + "get_tensor", + [](InferRequestWrapper& self, const ov::Output& port) { + return self._request.get_tensor(port); + }, + py::arg("port")); + + cls.def( + "get_tensor", + [](InferRequestWrapper& self, const ov::Output& port) { + return self._request.get_tensor(port); + }, + py::arg("port")); + + cls.def( + "set_tensor", + [](InferRequestWrapper& self, const std::string& name, const ov::runtime::Tensor& tensor) { + self._request.set_tensor(name, tensor); + }, + py::arg("name"), + py::arg("tensor")); + + cls.def( + "set_tensor", + [](InferRequestWrapper& self, const ov::Output& port, const ov::runtime::Tensor& tensor) { + self._request.set_tensor(port, tensor); + }, + py::arg("port"), + py::arg("tensor")); + + cls.def( + "set_tensor", + [](InferRequestWrapper& self, const ov::Output& port, const ov::runtime::Tensor& tensor) { + self._request.set_tensor(port, tensor); + }, + py::arg("port"), + py::arg("tensor")); + + cls.def( + "set_input_tensor", + [](InferRequestWrapper& self, size_t idx, const ov::runtime::Tensor& tensor) { + self._request.set_input_tensor(idx, tensor); + }, + py::arg("idx"), + py::arg("tensor")); + + cls.def( + "set_input_tensor", + [](InferRequestWrapper& self, const ov::runtime::Tensor& tensor) { + self._request.set_input_tensor(tensor); + }, + py::arg("tensor")); + + cls.def( + "set_output_tensor", + [](InferRequestWrapper& self, size_t idx, const ov::runtime::Tensor& tensor) { + self._request.set_output_tensor(idx, tensor); + }, + py::arg("idx"), + py::arg("tensor")); + + cls.def( + "set_output_tensor", + [](InferRequestWrapper& self, const ov::runtime::Tensor& tensor) { + self._request.set_output_tensor(tensor); + }, + py::arg("tensor")); + + cls.def_property_readonly("input_tensors", [](InferRequestWrapper& self) { + return self._inputs; + }); + + cls.def_property_readonly("output_tensors", [](InferRequestWrapper& self) { + return self._outputs; + }); + + cls.def_property_readonly("latency", [](InferRequestWrapper& self) { + return self.get_latency(); + }); +} diff --git a/runtime/bindings/python/src/pyopenvino/core/infer_request.hpp b/runtime/bindings/python/src/pyopenvino/core/infer_request.hpp new file mode 100644 index 00000000000000..e81261a54a608c --- /dev/null +++ b/runtime/bindings/python/src/pyopenvino/core/infer_request.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include + +#include + +namespace py = pybind11; + +typedef std::chrono::high_resolution_clock Time; +typedef std::chrono::nanoseconds ns; + +class InferRequestWrapper { +public: + InferRequestWrapper(ov::runtime::InferRequest request) + : _request(request) + { + } + + InferRequestWrapper(ov::runtime::InferRequest request, const std::vector>& inputs, const std::vector>& outputs) + : _request(request), _inputs(inputs), _outputs(outputs) + { + } + // ~InferRequestWrapper() = default; + + // bool user_callback_defined; + // py::function user_callback; + + double get_latency() { + auto execTime = std::chrono::duration_cast(_end_time - _start_time); + return static_cast(execTime.count()) * 0.000001; + } + + ov::runtime::InferRequest _request; + std::vector> _inputs; + std::vector> _outputs; + + Time::time_point _start_time; + Time::time_point _end_time; +}; + +void regclass_InferRequest(py::module m); diff --git a/runtime/bindings/python/src/pyopenvino/core/version.cpp b/runtime/bindings/python/src/pyopenvino/core/version.cpp index 45b2b0ed6b30c9..bed253697fda06 100644 --- a/runtime/bindings/python/src/pyopenvino/core/version.cpp +++ b/runtime/bindings/python/src/pyopenvino/core/version.cpp @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include - #include "openvino/core/version.hpp" +#include + namespace py = pybind11; void regclass_Version(py::module m) { diff --git a/runtime/bindings/python/src/pyopenvino/pyopenvino.cpp b/runtime/bindings/python/src/pyopenvino/pyopenvino.cpp index 42df5384ec5d45..e716063d4ee4ca 100644 --- a/runtime/bindings/python/src/pyopenvino/pyopenvino.cpp +++ b/runtime/bindings/python/src/pyopenvino/pyopenvino.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include "pyopenvino/graph/axis_set.hpp" @@ -26,15 +27,16 @@ #include "pyopenvino/core/ie_blob.hpp" #include "pyopenvino/core/ie_data.hpp" #include "pyopenvino/core/ie_infer_queue.hpp" -#include "pyopenvino/core/ie_infer_request.hpp" #include "pyopenvino/core/ie_input_info.hpp" #include "pyopenvino/core/ie_network.hpp" #include "pyopenvino/core/ie_parameter.hpp" #include "pyopenvino/core/ie_preprocess_info.hpp" #include "pyopenvino/core/offline_transformations.hpp" #include "pyopenvino/core/version.hpp" +#include "pyopenvino/core/infer_request.hpp" #include "pyopenvino/core/tensor.hpp" #include "pyopenvino/core/tensor_description.hpp" +#include "pyopenvino/core/version.hpp" #include "pyopenvino/graph/dimension.hpp" #include "pyopenvino/graph/layout.hpp" #include "pyopenvino/graph/ops/constant.hpp" @@ -139,17 +141,15 @@ PYBIND11_MODULE(pyopenvino, m) { regclass_Tensor(m); // Registering specific types of containers - Containers::regclass_PyInputsDataMap(m); - Containers::regclass_PyConstInputsDataMap(m); - Containers::regclass_PyOutputsDataMap(m); - Containers::regclass_PyResults(m); + Containers::regclass_TensorIndexMap(m); + Containers::regclass_TensorNameMap(m); regclass_ExecutableNetwork(m); regclass_InferRequest(m); regclass_Version(m); regclass_Parameter(m); regclass_InputInfo(m); - regclass_InferQueue(m); + // regclass_InferQueue(m); regclass_PreProcessInfo(m); regmodule_offline_transformations(m); diff --git a/runtime/bindings/python/tests/test_inference_engine/test_infer_request.py b/runtime/bindings/python/tests/test_inference_engine/test_infer_request.py index 50fbf1db853920..1b2cac4a627881 100644 --- a/runtime/bindings/python/tests/test_inference_engine/test_infer_request.py +++ b/runtime/bindings/python/tests/test_inference_engine/test_infer_request.py @@ -6,110 +6,111 @@ import pytest from tests.test_inference_engine.helpers import model_path, read_image -from openvino import Core, Blob, TensorDesc, StatusCode +from openvino import Core, Tensor is_myriad = os.environ.get("TEST_DEVICE") == "MYRIAD" test_net_xml, test_net_bin = model_path(is_myriad) -def test_get_perf_counts(device): +@pytest.mark.skip(reason="ProfilingInfo has to be bound") +def test_get_profiling_info(device): ie_core = Core() func = ie_core.read_model(test_net_xml, test_net_bin) ie_core.set_config({"PERF_COUNT": "YES"}, device) exec_net = ie_core.compile_model(func, device) img = read_image() request = exec_net.create_infer_request() - td = TensorDesc("FP32", [1, 3, 32, 32], "NCHW") - input_blob = Blob(td, img) - request.set_input({"data": input_blob}) - request.infer() - pc = request.get_perf_counts() + request.infer({0: img}) + pc = request.get_profiling_info() + assert pc["29"]["status"] == "EXECUTED" assert pc["29"]["layer_type"] == "FullyConnected" del exec_net del ie_core - del net -@pytest.mark.skipif(os.environ.get("TEST_DEVICE", "CPU") != "CPU", - reason=f"Can't run test on device {os.environ.get('TEST_DEVICE', 'CPU')}, " - "Dynamic batch fully supported only on CPU") -@pytest.mark.skip(reason="Fix") -def test_set_batch_size(device): +def test_tensor_setter(device): ie_core = Core() - ie_core.set_config({"DYN_BATCH_ENABLED": "YES"}, device) func = ie_core.read_model(test_net_xml, test_net_bin) - func.batch_size = 10 - data = np.ones(shape=net.input_info["data"].input_data.shape) - exec_net = ie_core.compile_model(net, device) - data[0] = read_image()[0] - request = exec_net.create_infer_request() - request.set_batch(1) - td = TensorDesc("FP32", [1, 3, 32, 32], "NCHW") - input_blob = Blob(td, data) - request.set_input({"data": input_blob}) - request.infer() - assert np.allclose(int(round(request.output_blobs["fc_out"].buffer[0][2])), 1), \ - "Incorrect data for 1st batch" - del exec_net - del ie_core - del net + exec_net_1 = ie_core.compile_model(network=func, device_name=device) + exec_net_2 = ie_core.compile_model(network=func, device_name=device) + img = read_image() + tensor = Tensor(img) -@pytest.mark.skip(reason="Fix") -def test_set_zero_batch_size(device): - ie_core = Core() - func = ie_core.read_model(test_net_xml, test_net_bin) - exec_net = ie_core.compile_model(func, device) - request = exec_net.create_infer_request() - with pytest.raises(ValueError) as e: - request.set_batch(0) - assert "Batch size should be positive integer number but 0 specified" in str(e.value) - del exec_net - del ie_core - del func + request1 = exec_net_1.create_infer_request() + request1.set_tensor("data", tensor) + t1 = request1.get_tensor("data") + + assert np.allclose(tensor.data, t1.data, atol=1e-2, rtol=1e-2) + + res = request1.infer({0: tensor}) + res_1 = np.sort(res[0]) + t2 = request1.get_tensor("fc_out") + assert np.allclose(t2.data, res[0].data, atol=1e-2, rtol=1e-2) + + request = exec_net_2.create_infer_request() + res = request.infer({"data": tensor}) + res_2 = np.sort(request.get_tensor("fc_out").data) + assert np.allclose(res_1, res_2, atol=1e-2, rtol=1e-2) + + request.set_tensor("data", tensor) + t3 = request.get_tensor("data") + assert np.allclose(t3.data, t1.data, atol=1e-2, rtol=1e-2) -@pytest.mark.skip(reason="Fix") -def test_set_negative_batch_size(device): +def test_set_tensors(device): ie_core = Core() func = ie_core.read_model(test_net_xml, test_net_bin) exec_net = ie_core.compile_model(func, device) + + data1 = read_image() + tensor1 = Tensor(data1) + data2 = np.ones(shape=(1, 10), dtype=np.float32) + tensor2 = Tensor(data2) + data3 = np.ones(shape=(1, 3, 32, 32), dtype=np.float32) + tensor3 = Tensor(data3) + data4 = np.zeros(shape=(1, 10), dtype=np.float32) + tensor4 = Tensor(data4) + request = exec_net.create_infer_request() - with pytest.raises(ValueError) as e: - request.set_batch(-1) - assert "Batch size should be positive integer number but -1 specified" in str(e.value) - del exec_net - del ie_core - del func + request.set_tensors({"data": tensor1, "fc_out": tensor2}) + t1 = request.get_tensor("data") + t2 = request.get_tensor("fc_out") + assert np.allclose(tensor1.data, t1.data, atol=1e-2, rtol=1e-2) + assert np.allclose(tensor2.data, t2.data, atol=1e-2, rtol=1e-2) + request.set_output_tensors({0: tensor2}) + output_node = exec_net.outputs[0] + t3 = request.get_tensor(output_node) + assert np.allclose(tensor2.data, t3.data, atol=1e-2, rtol=1e-2) -def test_blob_setter(device): - ie_core = Core() - func = ie_core.read_model(test_net_xml, test_net_bin) - exec_net_1 = ie_core.compile_model(network=func, device_name=device) + request.set_input_tensors({0: tensor1}) + output_node = exec_net.inputs[0] + t4 = request.get_tensor(output_node) + assert np.allclose(tensor1.data, t4.data, atol=1e-2, rtol=1e-2) - func.input_info["data"].layout = "NHWC" - exec_net_2 = ie_core.compile_model(network=func, device_name=device) + output_node = exec_net.inputs[0] + request.set_tensor(output_node, tensor3) + t5 = request.get_tensor(output_node) + assert np.allclose(tensor3.data, t5.data, atol=1e-2, rtol=1e-2) - img = read_image() + request.set_input_tensor(tensor3) + t6 = request.get_tensor(request.input_tensors[0]) + assert np.allclose(tensor3.data, t6.data, atol=1e-2, rtol=1e-2) - request1 = exec_net_1.create_infer_request() - tensor_desc = TensorDesc("FP32", [1, 3, img.shape[2], img.shape[3]], "NCHW") - img_blob1 = Blob(tensor_desc, img) - request1.set_input({"data": img_blob1}) - request1.infer() - res_1 = np.sort(request1.get_blob("fc_out").buffer) - - img = np.transpose(img, axes=(0, 2, 3, 1)).astype(np.float32) - tensor_desc = TensorDesc("FP32", [1, 3, 32, 32], "NHWC") - img_blob = Blob(tensor_desc, img) - request = exec_net_2.create_infer_request() - request.set_blob("data", img_blob) - request.infer() - res_2 = np.sort(request.get_blob("fc_out").buffer) - assert np.allclose(res_1, res_2, atol=1e-2, rtol=1e-2) + request.set_input_tensor(0, tensor1) + t7 = request.get_tensor(request.input_tensors[0]) + assert np.allclose(tensor1.data, t7.data, atol=1e-2, rtol=1e-2) + + request.set_output_tensor(tensor2) + t8 = request.get_tensor(request.output_tensors[0]) + assert np.allclose(tensor2.data, t8.data, atol=1e-2, rtol=1e-2) + + request.set_output_tensor(0, tensor4) + t9 = request.get_tensor(request.output_tensors[0]) + assert np.allclose(tensor4.data, t9.data, atol=1e-2, rtol=1e-2) def test_cancel(device): @@ -117,25 +118,38 @@ def test_cancel(device): func = ie_core.read_model(test_net_xml, test_net_bin) exec_net = ie_core.compile_model(func, device) img = read_image() - td = TensorDesc("FP32", [1, 3, 32, 32], "NCHW") - input_blob = Blob(td, img) request = exec_net.create_infer_request() - def callback(req, code, array): - array.append(42) + def callback(e): + raise Exception(e) - data = [] - request.set_completion_callback(callback, data) - request.set_input({"data": input_blob}) - request.async_infer() + request.set_callback(callback) + request.start_async({0: img}) request.cancel() with pytest.raises(RuntimeError) as e: request.wait() assert "[ INFER_CANCELLED ]" in str(e.value) - # check if callback has executed - assert data == [42] - request.async_infer() - status = request.wait() - assert status == StatusCode.OK - assert data == [42, 42] + request.start_async({"data": img}) + request.cancel() + with pytest.raises(RuntimeError) as e: + request.wait_for(1) + assert "[ INFER_CANCELLED ]" in str(e.value) + + +def test_infer_mixed_keys(device): + ie_core = Core() + func = ie_core.read_model(test_net_xml, test_net_bin) + ie_core.set_config({"PERF_COUNT": "YES"}, device) + exec_net = ie_core.compile_model(func, device) + + img = read_image() + tensor = Tensor(img) + + data2 = np.ones(shape=(1, 10), dtype=np.float32) + tensor2 = Tensor(data2) + + request = exec_net.create_infer_request() + with pytest.raises(TypeError) as e: + request.infer({0: tensor, "fc_out": tensor2}) + assert "incompatible function arguments." in str(e.value)