Infer new request method

runtime/bindings/python/src/openvino/__init__.py

@@ -16,8 +16,8 @@
 from openvino.ie_api import BlobWrapper
 from openvino.ie_api import infer
 from openvino.ie_api import start_async
-from openvino.ie_api import blob_from_file
 from openvino.ie_api import tensor_from_file
+from openvino.ie_api import infer_new_request
 
 from openvino.impl import Dimension
 from openvino.impl import Function
@@ -78,7 +78,7 @@
 # this class will be removed
 Blob = BlobWrapper
 # Patching ExecutableNetwork
-ExecutableNetwork.infer = infer
+ExecutableNetwork.infer_new_request = infer_new_request
 # Patching InferRequest
 InferRequest.infer = infer
 InferRequest.start_async = start_async

runtime/bindings/python/src/openvino/ie_api.py

@@ -17,6 +17,7 @@
 from openvino.pyopenvino import TensorDesc
 from openvino.pyopenvino import InferRequest
 from openvino.pyopenvino import Tensor
+from openvino.pyopenvino import ExecutableNetwork
 
 
 precision_map = {"FP32": np.float32,
@@ -45,6 +46,11 @@ def infer(request: InferRequest, inputs: dict = None) -> np.ndarray:
     res = request._infer(inputs=normalize_inputs(inputs if inputs is not None else {}))
     return np.asarray([copy.deepcopy(tensor.data) for tensor in res])
 
+
+def infer_new_request(exec_net: ExecutableNetwork, inputs: dict = None) -> np.ndarray:
+    res = exec_net._infer_new_request(inputs=normalize_inputs(inputs if inputs is not None else {}))
+    return np.asarray([copy.deepcopy(tensor.data) for tensor in res])
+
 # flake8: noqa: D102
 def start_async(request: InferRequest, inputs: dict = None) -> None:  # type: ignore
     request._start_async(inputs=normalize_inputs(inputs if inputs is not None else {}))
@@ -104,11 +110,6 @@ def __new__(cls, tensor_desc: TensorDesc, arr: np.ndarray = None):  # type: igno
         else:
             raise AttributeError(f"Unsupported precision {precision} for Blob")
 
-# flake8: noqa: D102
-def blob_from_file(path_to_bin_file: str) -> BlobWrapper:
-    array = np.fromfile(path_to_bin_file, dtype=np.uint8)
-    tensor_desc = TensorDesc("U8", array.shape, "C")
-    return BlobWrapper(tensor_desc, array)
 
 # flake8: noqa: D102
 def tensor_from_file(path: str) -> Tensor:

runtime/bindings/python/src/pyopenvino/core/common.cpp

@@ -185,32 +185,6 @@ PyObject* parse_parameter(const InferenceEngine::Parameter& param) {
     }
 }
 
-bool is_TBlob(const py::handle& blob) {
-    if (py::isinstance<InferenceEngine::TBlob<float>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<double>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<int8_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<int16_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<int32_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<int64_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<uint8_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<uint16_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<uint32_t>>(blob)) {
-        return true;
-    } else if (py::isinstance<InferenceEngine::TBlob<uint64_t>>(blob)) {
-        return true;
-    } else {
-        return false;
-    }
-}
-
 const ov::runtime::Tensor& cast_to_tensor(const py::handle& tensor) {
     return tensor.cast<const ov::runtime::Tensor&>();
 }
@@ -280,45 +254,6 @@ const std::shared_ptr<InferenceEngine::Blob> cast_to_blob(const py::handle& blob
     }
 }
 
-void blob_from_numpy(const py::handle& arr, InferenceEngine::Blob::Ptr blob) {
-    if (py::isinstance<py::array_t<float>>(arr)) {
-        Common::fill_blob<float>(arr, blob);
-    } else if (py::isinstance<py::array_t<double>>(arr)) {
-        Common::fill_blob<double>(arr, blob);
-    } else if (py::isinstance<py::array_t<int8_t>>(arr)) {
-        Common::fill_blob<int8_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<int16_t>>(arr)) {
-        Common::fill_blob<int16_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<int32_t>>(arr)) {
-        Common::fill_blob<int32_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<int64_t>>(arr)) {
-        Common::fill_blob<int64_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<uint8_t>>(arr)) {
-        Common::fill_blob<uint8_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<uint16_t>>(arr)) {
-        Common::fill_blob<uint16_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<uint32_t>>(arr)) {
-        Common::fill_blob<uint32_t>(arr, blob);
-    } else if (py::isinstance<py::array_t<uint64_t>>(arr)) {
-        Common::fill_blob<uint64_t>(arr, blob);
-    } else {
-        IE_THROW() << "Unsupported data type for when filling blob!";
-    }
-}
-
-void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary) {
-    for (auto&& pair : dictonary) {
-        const std::string& name = pair.first.cast<std::string>();
-        if (py::isinstance<py::array>(pair.second)) {
-            Common::blob_from_numpy(pair.second, request.GetBlob(name));
-        } else if (is_TBlob(pair.second)) {
-            request.SetBlob(name, Common::cast_to_blob(pair.second));
-        } else {
-            IE_THROW() << "Unable to set blob " << name << "!";
-        }
-    }
-}
-
 uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual) {
     try {
         auto parameter_value = actual.GetMetric(METRIC_KEY(SUPPORTED_METRICS));

runtime/bindings/python/src/pyopenvino/core/common.hpp

@@ -46,8 +46,6 @@ namespace Common
 
     PyObject* parse_parameter(const InferenceEngine::Parameter& param);
 
-    bool is_TBlob(const py::handle& blob);
-
     const std::shared_ptr<InferenceEngine::Blob> cast_to_blob(const py::handle& blob);
 
     const Containers::TensorNameMap cast_to_tensor_name_map(const py::dict& inputs);
@@ -56,10 +54,5 @@ namespace Common
 
     const ov::runtime::Tensor& cast_to_tensor(const py::handle& tensor);
 
-    void blob_from_numpy(const py::handle& _arr, InferenceEngine::Blob::Ptr &blob);
-
-    void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary);
-
     uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual);
-
 }; // namespace Common

runtime/bindings/python/src/pyopenvino/core/executable_network.cpp

@@ -1,15 +1,18 @@
 // Copyright (C) 2021 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
+//
 
 #include "openvino/runtime/executable_network.hpp"
 
 #include <pybind11/stl.h>
 
 #include "common.hpp"
 #include "pyopenvino/core/containers.hpp"
-#include "pyopenvino/core/ie_input_info.hpp"
 #include "pyopenvino/core/infer_request.hpp"
 
+PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap);
+PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap);
+
 namespace py = pybind11;
 
 void regclass_ExecutableNetwork(py::module m) {
@@ -21,26 +24,52 @@ void regclass_ExecutableNetwork(py::module m) {
         return InferRequestWrapper(self.create_infer_request(), self.inputs(), self.outputs());
     });
 
-    // cls.def("infer_new_request", [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) {
-    // TODO: implment after https://github.com/openvinotoolkit/openvino/pull/7962
-    // will be merged as a seperate ticket
-    // });
-
-    cls.def("export_model", &ov::runtime::ExecutableNetwork::export_model, py::arg("network_model"));
-
     cls.def(
-        "get_config",
-        [](ov::runtime::ExecutableNetwork& self, const std::string& name) -> py::handle {
-            return Common::parse_parameter(self.get_config(name));
+        "_infer_new_request",
+        [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) {
+            auto request = self.create_infer_request();
+            const auto key = inputs.begin()->first;
+            if (!inputs.empty()) {
+                if (py::isinstance<py::str>(key)) {
+                    auto inputs_map = Common::cast_to_tensor_name_map(inputs);
+                    for (auto&& input : inputs_map) {
+                        request.set_tensor(input.first, input.second);
+                    }
+                } else if (py::isinstance<py::int_>(key)) {
+                    auto inputs_map = Common::cast_to_tensor_index_map(inputs);
+                    for (auto&& input : inputs_map) {
+                        request.set_input_tensor(input.first, input.second);
+                    }
+                } else {
+                    throw py::type_error("Incompatible key type! Supported types are string and int.");
+                }
+            }
+
+            request.infer();
+
+            Containers::InferResults results;
+            for (const auto out : self.outputs()) {
+                results.push_back(request.get_tensor(out));
+            }
+            return results;
         },
-        py::arg("name"));
+        py::arg("inputs"));
 
-    cls.def(
-        "get_metric",
-        [](ov::runtime::ExecutableNetwork& self, const std::string& name) -> py::handle {
-            return Common::parse_parameter(self.get_metric(name));
-        },
-        py::arg("name"));
+    cls.def("export_model", &ov::runtime::ExecutableNetwork::export_model, py::arg("network_model"));
+
+    // cls.def(
+    //     "get_config",
+    //     [](ov::runtime::ExecutableNetwork& self, const std::string& name) -> py::handle {
+    //         return Common::parse_parameter(self.get_config(name));
+    //     },
+    //     py::arg("name"));
+
+    // cls.def(
+    //     "get_metric",
+    //     [](ov::runtime::ExecutableNetwork& self, const std::string& name) -> py::handle {
+    //         return Common::parse_parameter(self.get_metric(name));
+    //     },
+    //     py::arg("name"));
 
     cls.def("get_runtime_function", &ov::runtime::ExecutableNetwork::get_runtime_function);
 

runtime/bindings/python/tests/test_inference_engine/test_executable_network.py

@@ -2,13 +2,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
+from numpy.core.fromnumeric import argmax
 import pytest
 import numpy as np
 
 from ..conftest import model_path, image_path
 from openvino.impl import Function, ConstOutput, Shape
 
-from openvino import Core
+from openvino import Core, Tensor
 
 is_myriad = os.environ.get("TEST_DEVICE") == "MYRIAD"
 test_net_xml, test_net_bin = model_path(is_myriad)
@@ -27,25 +28,6 @@ def read_image():
     return image
 
 
-def test_get_metric(device):
-    core = Core()
-    func = core.read_model(model=test_net_xml, weights=test_net_bin)
-    exec_net = core.compile_model(func, device)
-    network_name = exec_net.get_metric("NETWORK_NAME")
-    assert network_name == "test_model"
-
-
-@pytest.mark.skipif(os.environ.get("TEST_DEVICE", "CPU") != "CPU", reason="Device dependent test")
-def test_get_config(device):
-    core = Core()
-    if core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
-        pytest.skip("Can't run on ARM plugin due-to CPU dependent test")
-    func = core.read_model(model=test_net_xml, weights=test_net_bin)
-    exec_net = core.compile_model(func, device)
-    config = exec_net.get_config("PERF_COUNT")
-    assert config == "NO"
-
-
 def test_get_runtime_function(device):
     core = Core()
     func = core.read_model(model=test_net_xml, weights=test_net_bin)
@@ -236,3 +218,60 @@ def test_inputs_docs(device):
     input_0 = inputs[0]
     expected_string = "openvino.impl.ConstOutput wraps ov::Output<Const ov::Node >"
     assert input_0.__doc__ == expected_string
+
+
+def test_infer_new_request_numpy(device):
+    ie = Core()
+    func = ie.read_model(model=test_net_xml, weights=test_net_bin)
+    img = read_image()
+    exec_net = ie.compile_model(func, device)
+    res = exec_net.infer_new_request({'data': img})
+    assert np.argmax(res) == 2
+
+
+def test_infer_new_request_tensor_numpy_copy(device):
+    ie = Core()
+    func = ie.read_model(model=test_net_xml, weights=test_net_bin)
+    img = read_image()
+    tensor = Tensor(img)
+    exec_net = ie.compile_model(func, device)
+    res_tensor = exec_net.infer_new_request({'data': tensor})
+    res_img = exec_net.infer_new_request({'data': tensor})
+    assert np.argmax(res_tensor) == 2
+    assert np.argmax(res_tensor) == np.argmax(res_img)
+
+
+def test_infer_tensor_numpy_shared_memory(device):
+    ie = Core()
+    func = ie.read_model(model=test_net_xml, weights=test_net_bin)
+    img = read_image()
+    img = np.ascontiguousarray(img)
+    tensor = Tensor(img, shared_memory=True)
+    exec_net = ie.compile_model(func, device)
+    res_tensor = exec_net.infer_new_request({'data': tensor})
+    res_img = exec_net.infer_new_request({'data': tensor})
+    assert np.argmax(res_tensor) == 2
+    assert np.argmax(res_tensor) == np.argmax(res_img)
+
+
+def test_infer_new_request_wrong_port_name(device):
+    ie = Core()
+    func = ie.read_model(model=test_net_xml, weights=test_net_bin)
+    img = read_image()
+    tensor = Tensor(img)
+    exec_net = ie.compile_model(func, device)
+    with pytest.raises(RuntimeError) as e:
+        exec_net.infer_new_request({'_data_': tensor})
+    assert "Port for tensor name _data_ was not found." in str(e.value)
+
+
+def test_infer_tensor_wrong_input_data(device):
+    ie = Core()
+    func = ie.read_model(model=test_net_xml, weights=test_net_bin)
+    img = read_image()
+    img = np.ascontiguousarray(img)
+    tensor = Tensor(img, shared_memory=True)
+    exec_net = ie.compile_model(func, device)
+    with pytest.raises(TypeError) as e:
+        exec_net.infer_new_request({4.5: tensor})
+    assert "Incompatible key type!" in str(e.value)