Pszmel/bind infer request (openvinotoolkit#51)

* remove set_batch, get_blob and set_blob

* update InferRequest class

* change InferenceEngine::InferRequest to ov::runtime::InferRequest

* update set_callback body

* update bindings to reflect ov::runtime::InferRequest

* bind set_input_tensor and get_input_tensor

* style fix

* clen ie_infer_queue.cpp

* Bind exec core ov (openvinotoolkit#50)

* bind core, exec_net classes

* rm unused function

* add new line

* rename ie_infer_request -> infer_request

* update imports

* update __init__.py

* update ie_api.py

* Replace old containers with the new one

* create impl for create_infer_request

* comment out infer_queue to avoid errors with old infer_request

* update infer_request bind to reflect new infer_request api

* comment out inpuit_info from ie_network to avoid errors with old containers

* Register new containers and comment out InferQueue

* update infer request tests

* style fix

* remove unused imports

* remove unused imports and 2 methods

* add tests to cover all new methods from infer_request

* style fix

* add test

* remove registration of InferResults

* update name of exception_ptr parameter

* update the loops that iterate through inputs and outputs

* clean setCustomCallbacks

* style fix

* add Tensor import

* style fix

* update infer and normalize_inputs

* style fix

* rename startTime and endTime

* Create test for mixed keys as infer arguments

* update infer function

* update return type of infer

Co-authored-by: Bartek Szmelczynski <[email protected]>

runtime/bindings/python/src/openvino/__init__.py

@@ -15,8 +15,7 @@
 
 from openvino.ie_api import BlobWrapper
 from openvino.ie_api import infer
-from openvino.ie_api import async_infer
-from openvino.ie_api import get_result
+from openvino.ie_api import start_async
 from openvino.ie_api import blob_from_file
 from openvino.ie_api import tensor_from_file
 
@@ -37,7 +36,7 @@
 from openvino.pyopenvino import TensorDesc
 from openvino.pyopenvino import get_version
 from openvino.pyopenvino import StatusCode
-from openvino.pyopenvino import InferQueue
+#from openvino.pyopenvino import InferQueue
 from openvino.pyopenvino import InferRequest  # TODO: move to ie_api?
 from openvino.pyopenvino import Blob
 from openvino.pyopenvino import PreProcessInfo
@@ -82,7 +81,6 @@
 ExecutableNetwork.infer = infer
 # Patching InferRequest
 InferRequest.infer = infer
-InferRequest.async_infer = async_infer
-InferRequest.get_result = get_result
+InferRequest.start_async = start_async
 # Patching InferQueue
-InferQueue.async_infer = async_infer
+#InferQueue.async_infer = async_infer

runtime/bindings/python/src/openvino/ie_api.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import numpy as np
+import copy
 
 from openvino.pyopenvino import TBlobFloat32
 from openvino.pyopenvino import TBlobFloat64
@@ -36,22 +37,17 @@
 
 def normalize_inputs(py_dict: dict) -> dict:
     """Normalize a dictionary of inputs to contiguous numpy arrays."""
-    return {k: (np.ascontiguousarray(v) if isinstance(v, np.ndarray) else v)
+    return {k: (Tensor(v) if isinstance(v, np.ndarray) else v)
             for k, v in py_dict.items()}
 
 # flake8: noqa: D102
-def infer(request: InferRequest, inputs: dict = None) -> dict:
-    results = request._infer(inputs=normalize_inputs(inputs if inputs is not None else {}))
-    return {name: (blob.buffer.copy()) for name, blob in results.items()}
+def infer(request: InferRequest, inputs: dict = None) -> np.ndarray:
+    res = request._infer(inputs=normalize_inputs(inputs if inputs is not None else {}))
+    return np.asarray([copy.deepcopy(tensor.data) for tensor in res])
 
 # flake8: noqa: D102
-def get_result(request: InferRequest, name: str) -> np.ndarray:
-    return request.get_blob(name).buffer.copy()
-
-# flake8: noqa: D102
-def async_infer(request: InferRequest, inputs: dict = None, userdata=None) -> None:  # type: ignore
-    request._async_infer(inputs=normalize_inputs(inputs if inputs is not None else {}),
-                         userdata=userdata)
+def start_async(request: InferRequest, inputs: dict = None) -> None:  # type: ignore
+    request._start_async(inputs=normalize_inputs(inputs if inputs is not None else {}))
 
 # flake8: noqa: C901
 # Dispatch Blob types on Python side.

runtime/bindings/python/src/pyopenvino/core/common.hpp

@@ -53,4 +53,5 @@ namespace Common
     void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary);
 
     uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual);
+
 }; // namespace Common

runtime/bindings/python/src/pyopenvino/core/containers.cpp

@@ -8,44 +8,18 @@
 #include <pybind11/stl.h>
 #include <pybind11/stl_bind.h>
 
-PYBIND11_MAKE_OPAQUE(Containers::PyInputsDataMap);
-PYBIND11_MAKE_OPAQUE(Containers::PyConstInputsDataMap);
-PYBIND11_MAKE_OPAQUE(Containers::PyOutputsDataMap);
-PYBIND11_MAKE_OPAQUE(Containers::PyResults);
+PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap);
+PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap);
 
 namespace py = pybind11;
 
 namespace Containers {
 
-void regclass_PyInputsDataMap(py::module m) {
-    auto py_inputs_data_map = py::bind_map<PyInputsDataMap>(m, "PyInputsDataMap");
-
-    py_inputs_data_map.def("keys", [](PyInputsDataMap& self) {
-        return py::make_key_iterator(self.begin(), self.end());
-    });
-}
-
-void regclass_PyConstInputsDataMap(py::module m) {
-    auto py_const_inputs_data_map = py::bind_map<PyConstInputsDataMap>(m, "PyConstInputsDataMap");
-
-    py_const_inputs_data_map.def("keys", [](PyConstInputsDataMap& self) {
-        return py::make_key_iterator(self.begin(), self.end());
-    });
+void regclass_TensorIndexMap(py::module m) {
+    auto tensor_index_map = py::bind_map<TensorIndexMap>(m, "TensorIndexMap");
 }
 
-void regclass_PyOutputsDataMap(py::module m) {
-    auto py_outputs_data_map = py::bind_map<PyOutputsDataMap>(m, "PyOutputsDataMap");
-
-    py_outputs_data_map.def("keys", [](PyOutputsDataMap& self) {
-        return py::make_key_iterator(self.begin(), self.end());
-    });
-}
-
-void regclass_PyResults(py::module m) {
-    auto py_results = py::bind_map<PyResults>(m, "PyResults");
-
-    py_results.def("keys", [](PyResults& self) {
-        return py::make_key_iterator(self.begin(), self.end());
-    });
+void regclass_TensorNameMap(py::module m) {
+    auto tensor_name_map = py::bind_map<TensorNameMap>(m, "TensorNameMap");
 }
 }  // namespace Containers

runtime/bindings/python/src/pyopenvino/core/containers.hpp

@@ -5,27 +5,17 @@
 #pragma once
 
 #include <pybind11/pybind11.h>
+#include <openvino/runtime/tensor.hpp>
 #include <string>
-#include <ie_input_info.hpp>
-#include "ie_data.h"
-#include "ie_blob.h"
+#include <map>
 
 namespace py = pybind11;
 
 namespace Containers {
-    using PyInputsDataMap = std::map<std::string, std::shared_ptr<InferenceEngine::InputInfo>>;
+    using TensorIndexMap = std::map<size_t, ov::runtime::Tensor>;
+    using TensorNameMap = std::map<std::string, ov::runtime::Tensor>;
+    using InferResults = std::vector<ov::runtime::Tensor>;
 
-    using PyConstInputsDataMap =
-        std::map<std::string, std::shared_ptr<const InferenceEngine::InputInfo>>;
-
-    using PyOutputsDataMap =
-        std::map<std::string, std::shared_ptr<const InferenceEngine::Data>>;
-
-    using PyResults =
-        std::map<std::string, std::shared_ptr<const InferenceEngine::Blob>>;
-
-    void regclass_PyInputsDataMap(py::module m);
-    void regclass_PyConstInputsDataMap(py::module m);
-    void regclass_PyOutputsDataMap(py::module m);
-    void regclass_PyResults(py::module m);
-}
+    void regclass_TensorIndexMap(py::module m);
+    void regclass_TensorNameMap(py::module m);
+}

runtime/bindings/python/src/pyopenvino/core/executable_network.cpp

@@ -7,8 +7,8 @@
 
 #include "common.hpp"
 #include "pyopenvino/core/containers.hpp"
-#include "pyopenvino/core/ie_infer_request.hpp"
 #include "pyopenvino/core/ie_input_info.hpp"
+#include "pyopenvino/core/infer_request.hpp"
 
 namespace py = pybind11;
 
@@ -17,7 +17,9 @@ void regclass_ExecutableNetwork(py::module m) {
         m,
         "ExecutableNetwork");
 
-    cls.def("create_infer_request", &ov::runtime::ExecutableNetwork::create_infer_request);
+    cls.def("create_infer_request", [](ov::runtime::ExecutableNetwork& self) {
+        return InferRequestWrapper(self.create_infer_request(), self.inputs(), self.outputs());
+    });
 
     // cls.def("infer_new_request", [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) {
     // TODO: implment after https://github.com/openvinotoolkit/openvino/pull/7962