Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Infer new request method #56

Merged
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions runtime/bindings/python/src/openvino/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
from openvino.ie_api import BlobWrapper
from openvino.ie_api import infer
from openvino.ie_api import start_async
from openvino.ie_api import blob_from_file
from openvino.ie_api import tensor_from_file
from openvino.ie_api import infer_new_request

from openvino.impl import Dimension
from openvino.impl import Function
Expand Down Expand Up @@ -78,7 +78,7 @@
# this class will be removed
Blob = BlobWrapper
# Patching ExecutableNetwork
ExecutableNetwork.infer = infer
ExecutableNetwork.infer_new_request = infer_new_request
# Patching InferRequest
InferRequest.infer = infer
InferRequest.start_async = start_async
Expand Down
11 changes: 6 additions & 5 deletions runtime/bindings/python/src/openvino/ie_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from openvino.pyopenvino import TensorDesc
from openvino.pyopenvino import InferRequest
from openvino.pyopenvino import Tensor
from openvino.pyopenvino import ExecutableNetwork


precision_map = {"FP32": np.float32,
Expand Down Expand Up @@ -45,6 +46,11 @@ def infer(request: InferRequest, inputs: dict = None) -> np.ndarray:
res = request._infer(inputs=normalize_inputs(inputs if inputs is not None else {}))
return np.asarray([copy.deepcopy(tensor.data) for tensor in res])


def infer_new_request(exec_net: ExecutableNetwork, inputs: dict = None) -> np.ndarray:
res = exec_net._infer_new_request(inputs=normalize_inputs(inputs if inputs is not None else {}))
return np.asarray([copy.deepcopy(tensor.data) for tensor in res])
bszmelcz marked this conversation as resolved.
Show resolved Hide resolved

# flake8: noqa: D102
def start_async(request: InferRequest, inputs: dict = None) -> None: # type: ignore
request._start_async(inputs=normalize_inputs(inputs if inputs is not None else {}))
Expand Down Expand Up @@ -104,11 +110,6 @@ def __new__(cls, tensor_desc: TensorDesc, arr: np.ndarray = None): # type: igno
else:
raise AttributeError(f"Unsupported precision {precision} for Blob")

# flake8: noqa: D102
def blob_from_file(path_to_bin_file: str) -> BlobWrapper:
bszmelcz marked this conversation as resolved.
Show resolved Hide resolved
array = np.fromfile(path_to_bin_file, dtype=np.uint8)
tensor_desc = TensorDesc("U8", array.shape, "C")
return BlobWrapper(tensor_desc, array)

# flake8: noqa: D102
def tensor_from_file(path: str) -> Tensor:
Expand Down
65 changes: 0 additions & 65 deletions runtime/bindings/python/src/pyopenvino/core/common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -185,32 +185,6 @@ PyObject* parse_parameter(const InferenceEngine::Parameter& param) {
}
}

bool is_TBlob(const py::handle& blob) {
if (py::isinstance<InferenceEngine::TBlob<float>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<double>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<int8_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<int16_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<int32_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<int64_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<uint8_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<uint16_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<uint32_t>>(blob)) {
return true;
} else if (py::isinstance<InferenceEngine::TBlob<uint64_t>>(blob)) {
return true;
} else {
return false;
}
}

const ov::runtime::Tensor& cast_to_tensor(const py::handle& tensor) {
return tensor.cast<const ov::runtime::Tensor&>();
}
Expand Down Expand Up @@ -280,45 +254,6 @@ const std::shared_ptr<InferenceEngine::Blob> cast_to_blob(const py::handle& blob
}
}

void blob_from_numpy(const py::handle& arr, InferenceEngine::Blob::Ptr blob) {
if (py::isinstance<py::array_t<float>>(arr)) {
Common::fill_blob<float>(arr, blob);
} else if (py::isinstance<py::array_t<double>>(arr)) {
Common::fill_blob<double>(arr, blob);
} else if (py::isinstance<py::array_t<int8_t>>(arr)) {
Common::fill_blob<int8_t>(arr, blob);
} else if (py::isinstance<py::array_t<int16_t>>(arr)) {
Common::fill_blob<int16_t>(arr, blob);
} else if (py::isinstance<py::array_t<int32_t>>(arr)) {
Common::fill_blob<int32_t>(arr, blob);
} else if (py::isinstance<py::array_t<int64_t>>(arr)) {
Common::fill_blob<int64_t>(arr, blob);
} else if (py::isinstance<py::array_t<uint8_t>>(arr)) {
Common::fill_blob<uint8_t>(arr, blob);
} else if (py::isinstance<py::array_t<uint16_t>>(arr)) {
Common::fill_blob<uint16_t>(arr, blob);
} else if (py::isinstance<py::array_t<uint32_t>>(arr)) {
Common::fill_blob<uint32_t>(arr, blob);
} else if (py::isinstance<py::array_t<uint64_t>>(arr)) {
Common::fill_blob<uint64_t>(arr, blob);
} else {
IE_THROW() << "Unsupported data type for when filling blob!";
}
}

void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary) {
for (auto&& pair : dictonary) {
const std::string& name = pair.first.cast<std::string>();
if (py::isinstance<py::array>(pair.second)) {
Common::blob_from_numpy(pair.second, request.GetBlob(name));
} else if (is_TBlob(pair.second)) {
request.SetBlob(name, Common::cast_to_blob(pair.second));
} else {
IE_THROW() << "Unable to set blob " << name << "!";
}
}
}

uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual) {
try {
auto parameter_value = actual.GetMetric(METRIC_KEY(SUPPORTED_METRICS));
Expand Down
7 changes: 0 additions & 7 deletions runtime/bindings/python/src/pyopenvino/core/common.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ namespace Common

PyObject* parse_parameter(const InferenceEngine::Parameter& param);

bool is_TBlob(const py::handle& blob);

const std::shared_ptr<InferenceEngine::Blob> cast_to_blob(const py::handle& blob);

const Containers::TensorNameMap cast_to_tensor_name_map(const py::dict& inputs);
Expand All @@ -56,10 +54,5 @@ namespace Common

const ov::runtime::Tensor& cast_to_tensor(const py::handle& tensor);

void blob_from_numpy(const py::handle& _arr, InferenceEngine::Blob::Ptr &blob);

void set_request_blobs(InferenceEngine::InferRequest& request, const py::dict& dictonary);

uint32_t get_optimal_number_of_requests(const InferenceEngine::ExecutableNetwork& actual);

}; // namespace Common
39 changes: 34 additions & 5 deletions runtime/bindings/python/src/pyopenvino/core/executable_network.cpp
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "openvino/runtime/executable_network.hpp"

#include <pybind11/stl.h>

#include "common.hpp"
#include "pyopenvino/core/containers.hpp"
#include "pyopenvino/core/ie_input_info.hpp"
#include "pyopenvino/core/infer_request.hpp"

PYBIND11_MAKE_OPAQUE(Containers::TensorIndexMap);
PYBIND11_MAKE_OPAQUE(Containers::TensorNameMap);

namespace py = pybind11;

void regclass_ExecutableNetwork(py::module m) {
Expand All @@ -21,10 +24,36 @@ void regclass_ExecutableNetwork(py::module m) {
return InferRequestWrapper(self.create_infer_request(), self.inputs(), self.outputs());
});

// cls.def("infer_new_request", [](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) {
// TODO: implment after https://github.com/openvinotoolkit/openvino/pull/7962
// will be merged as a seperate ticket
// });
cls.def(
"_infer_new_request",
[](ov::runtime::ExecutableNetwork& self, const py::dict& inputs) {
auto request = self.create_infer_request();
const auto key = inputs.begin()->first;
if (!inputs.empty()) {
if (py::isinstance<py::str>(key)) {
auto inputs_map = Common::cast_to_tensor_name_map(inputs);
for (auto&& input : inputs_map) {
request.set_tensor(input.first, input.second);
}
} else if (py::isinstance<py::int_>(key)) {
auto inputs_map = Common::cast_to_tensor_index_map(inputs);
for (auto&& input : inputs_map) {
request.set_input_tensor(input.first, input.second);
}
} else {
throw py::type_error("Incompatible key type! Supported types are string and int.");
}
}

request.infer();

Containers::InferResults results;
for (const auto out : self.outputs()) {
results.push_back(request.get_tensor(out));
}
return results;
},
py::arg("inputs"));

cls.def("export_model", &ov::runtime::ExecutableNetwork::export_model, py::arg("network_model"));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@
# SPDX-License-Identifier: Apache-2.0

import os
from numpy.core.fromnumeric import argmax
import pytest
import numpy as np

from ..conftest import model_path, image_path
from openvino.impl import Function, ConstOutput, Shape

from openvino import Core
from openvino import Core, Tensor

is_myriad = os.environ.get("TEST_DEVICE") == "MYRIAD"
test_net_xml, test_net_bin = model_path(is_myriad)
Expand Down Expand Up @@ -236,3 +237,60 @@ def test_inputs_docs(device):
input_0 = inputs[0]
expected_string = "openvino.impl.ConstOutput wraps ov::Output<Const ov::Node >"
assert input_0.__doc__ == expected_string


def test_infer_new_request_numpy(device):
ie = Core()
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
img = read_image()
exec_net = ie.compile_model(func, device)
res = exec_net.infer_new_request({'data': img})
assert np.argmax(res) == 2


def test_infer_new_request_tensor_numpy_copy(device):
ie = Core()
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
img = read_image()
tensor = Tensor(img)
exec_net = ie.compile_model(func, device)
res_tensor = exec_net.infer_new_request({'data': tensor})
res_img = exec_net.infer_new_request({'data': tensor})
assert np.argmax(res_tensor) == 2
assert np.argmax(res_tensor) == np.argmax(res_img)


def test_infer_tensor_numpy_shared_memory(device):
ie = Core()
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
img = read_image()
img = np.ascontiguousarray(img)
tensor = Tensor(img, shared_memory=True)
exec_net = ie.compile_model(func, device)
res_tensor = exec_net.infer_new_request({'data': tensor})
res_img = exec_net.infer_new_request({'data': tensor})
assert np.argmax(res_tensor) == 2
assert np.argmax(res_tensor) == np.argmax(res_img)


def test_infer_new_request_wrong_port_name(device):
ie = Core()
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
img = read_image()
tensor = Tensor(img)
exec_net = ie.compile_model(func, device)
with pytest.raises(RuntimeError) as e:
exec_net.infer_new_request({'_data_': tensor})
assert "Port for tensor name _data_ was not found." in str(e.value)


def test_infer_tensor_wrong_input_data(device):
ie = Core()
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
img = read_image()
img = np.ascontiguousarray(img)
tensor = Tensor(img, shared_memory=True)
exec_net = ie.compile_model(func, device)
with pytest.raises(TypeError) as e:
exec_net.infer_new_request({4.5: tensor})
assert "Incompatible key type!" in str(e.value)