diff --git a/runtime/bindings/python/src/openvino/__init__.py b/runtime/bindings/python/src/openvino/__init__.py index ceab3f6b48f6d2..d6d94dac942479 100644 --- a/runtime/bindings/python/src/openvino/__init__.py +++ b/runtime/bindings/python/src/openvino/__init__.py @@ -18,6 +18,7 @@ from openvino.ie_api import start_async from openvino.ie_api import blob_from_file from openvino.ie_api import tensor_from_file +from openvino.ie_api import infer_new_request from openvino.impl import Dimension from openvino.impl import Function @@ -79,7 +80,7 @@ # this class will be removed Blob = BlobWrapper # Patching ExecutableNetwork -ExecutableNetwork.infer = infer +ExecutableNetwork.infer_new_request = infer_new_request # Patching InferRequest InferRequest.infer = infer InferRequest.start_async = start_async diff --git a/runtime/bindings/python/src/openvino/ie_api.py b/runtime/bindings/python/src/openvino/ie_api.py index 3d15dc52573f3b..af4d0e97f0625b 100644 --- a/runtime/bindings/python/src/openvino/ie_api.py +++ b/runtime/bindings/python/src/openvino/ie_api.py @@ -3,6 +3,7 @@ import numpy as np import copy +from typing import List from openvino.pyopenvino import TBlobFloat32 from openvino.pyopenvino import TBlobFloat64 @@ -16,6 +17,7 @@ from openvino.pyopenvino import TBlobUint8 from openvino.pyopenvino import TensorDesc from openvino.pyopenvino import InferRequest +from openvino.pyopenvino import ExecutableNetwork from openvino.pyopenvino import Tensor @@ -47,6 +49,13 @@ def infer(request: InferRequest, inputs: dict = None) -> np.ndarray: # dimensions. This results in errors when running ops like variadic split. return [copy.deepcopy(tensor.data) for tensor in res] + +def infer_new_request(exec_net: ExecutableNetwork, inputs: dict = None) -> List[np.ndarray]: + res = exec_net._infer_new_request(inputs=normalize_inputs(inputs if inputs is not None else {})) + # Required to return list since np.ndarray forces all of tensors data to match in + # dimensions. This results in errors when running ops like variadic split. + return [copy.deepcopy(tensor.data) for tensor in res] + # flake8: noqa: D102 def start_async(request: InferRequest, inputs: dict = None) -> None: # type: ignore request._start_async(inputs=normalize_inputs(inputs if inputs is not None else {})) diff --git a/runtime/bindings/python/src/pyopenvino/core/common.hpp b/runtime/bindings/python/src/pyopenvino/core/common.hpp index fb7a47e6be15af..d4be9bd2a77995 100644 --- a/runtime/bindings/python/src/pyopenvino/core/common.hpp +++ b/runtime/bindings/python/src/pyopenvino/core/common.hpp @@ -61,5 +61,4 @@ namespace Common void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary); uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual); - }; // namespace Common diff --git a/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp b/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp index a5b04aab2c18cb..389a865aea1b0a 100644 --- a/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp +++ b/runtime/bindings/python/src/pyopenvino/core/executable_network.cpp @@ -1,5 +1,6 @@ // Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 +// #include "openvino/runtime/executable_network.hpp" @@ -7,9 +8,11 @@ #include "common.hpp" #include "pyopenvino/core/containers.hpp" -#include "pyopenvino/core/ie_input_info.hpp" #include "pyopenvino/core/infer_request.hpp" +PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap); +PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap); + namespace py = pybind11; void regclass_ExecutableNetwork(py::module m) { @@ -21,10 +24,36 @@ void regclass_ExecutableNetwork(py::module m) { return InferRequestWrapper(self.create_infer_request(), self.inputs(), self.outputs()); }); - // cls.def("infer_new_request", [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) { - // TODO: implment after https://github.com/openvinotoolkit/openvino/pull/7962 - // will be merged as a seperate ticket - // }); + cls.def( + "_infer_new_request", + [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) { + auto request = self.create_infer_request(); + const auto key = inputs.begin()->first; + if (!inputs.empty()) { + if (py::isinstance(key)) { + auto inputs_map = Common::cast_to_tensor_name_map(inputs); + for (auto&& input : inputs_map) { + request.set_tensor(input.first, input.second); + } + } else if (py::isinstance(key)) { + auto inputs_map = Common::cast_to_tensor_index_map(inputs); + for (auto&& input : inputs_map) { + request.set_input_tensor(input.first, input.second); + } + } else { + throw py::type_error("Incompatible key type! Supported types are string and int."); + } + } + + request.infer(); + + Containers::InferResults results; + for (const auto out : self.outputs()) { + results.push_back(request.get_tensor(out)); + } + return results; + }, + py::arg("inputs")); cls.def("export_model", &ov::runtime::ExecutableNetwork::export_model, py::arg("network_model")); diff --git a/runtime/bindings/python/tests/test_inference_engine/test_executable_network.py b/runtime/bindings/python/tests/test_inference_engine/test_executable_network.py index ec1eaf2f8112b9..99a5c8e96f12d9 100644 --- a/runtime/bindings/python/tests/test_inference_engine/test_executable_network.py +++ b/runtime/bindings/python/tests/test_inference_engine/test_executable_network.py @@ -2,13 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 import os +from numpy.core.fromnumeric import argmax import pytest import numpy as np from ..conftest import model_path, image_path from openvino.impl import Function, ConstOutput, Shape -from openvino import Core +from openvino import Core, Tensor is_myriad = os.environ.get("TEST_DEVICE") == "MYRIAD" test_net_xml, test_net_bin = model_path(is_myriad) @@ -236,3 +237,60 @@ def test_inputs_docs(device): input_0 = inputs[0] expected_string = "openvino.impl.ConstOutput wraps ov::Output" assert input_0.__doc__ == expected_string + + +def test_infer_new_request_numpy(device): + ie = Core() + func = ie.read_model(model=test_net_xml, weights=test_net_bin) + img = read_image() + exec_net = ie.compile_model(func, device) + res = exec_net.infer_new_request({'data': img}) + assert np.argmax(res) == 2 + + +def test_infer_new_request_tensor_numpy_copy(device): + ie = Core() + func = ie.read_model(model=test_net_xml, weights=test_net_bin) + img = read_image() + tensor = Tensor(img) + exec_net = ie.compile_model(func, device) + res_tensor = exec_net.infer_new_request({'data': tensor}) + res_img = exec_net.infer_new_request({'data': tensor}) + assert np.argmax(res_tensor) == 2 + assert np.argmax(res_tensor) == np.argmax(res_img) + + +def test_infer_tensor_numpy_shared_memory(device): + ie = Core() + func = ie.read_model(model=test_net_xml, weights=test_net_bin) + img = read_image() + img = np.ascontiguousarray(img) + tensor = Tensor(img, shared_memory=True) + exec_net = ie.compile_model(func, device) + res_tensor = exec_net.infer_new_request({'data': tensor}) + res_img = exec_net.infer_new_request({'data': tensor}) + assert np.argmax(res_tensor) == 2 + assert np.argmax(res_tensor) == np.argmax(res_img) + + +def test_infer_new_request_wrong_port_name(device): + ie = Core() + func = ie.read_model(model=test_net_xml, weights=test_net_bin) + img = read_image() + tensor = Tensor(img) + exec_net = ie.compile_model(func, device) + with pytest.raises(RuntimeError) as e: + exec_net.infer_new_request({'_data_': tensor}) + assert "Port for tensor name _data_ was not found." in str(e.value) + + +def test_infer_tensor_wrong_input_data(device): + ie = Core() + func = ie.read_model(model=test_net_xml, weights=test_net_bin) + img = read_image() + img = np.ascontiguousarray(img) + tensor = Tensor(img, shared_memory=True) + exec_net = ie.compile_model(func, device) + with pytest.raises(TypeError) as e: + exec_net.infer_new_request({4.5: tensor}) + assert "Incompatible key type!" in str(e.value)