Skip to content

Commit

Permalink
[IE PYTHON] wait for infer callback to complete (#7374)
Browse files Browse the repository at this point in the history
* test commit

* Add test

* Remove threading.event from InferRequest

* Fix wait for ExecutableNetwork
Alexey Lebedev authored Sep 15, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 5b285ed commit 6df94af
Showing 5 changed files with 43 additions and 28 deletions.
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ cdef class InferRequest:
cpdef get_perf_counts(self)
cdef void user_callback(self, int status) with gil
cdef public:
_inputs_list, _outputs_list, _py_callback, _py_data, _py_callback_used, _py_callback_called, _user_blobs, _inputs_is_dynamic
_inputs_list, _outputs_list, _py_callback, _py_data, _user_blobs, _inputs_is_dynamic

cdef class IENetwork:
cdef C.IENetwork impl
Original file line number Diff line number Diff line change
@@ -1071,15 +1071,11 @@ cdef class InferRequest:
self._inputs_list = []
self._outputs_list = []
self._py_callback = lambda *args, **kwargs: None
self._py_callback_used = False
self._py_callback_called = threading.Event()
self._py_data = None
self._inputs_is_dynamic = {}

cdef void user_callback(self, int status) with gil:
if self._py_callback:
# Set flag at first since user can call wait in callback
self._py_callback_called.set()
self._py_callback(status, self._py_data)

## Description: Sets a callback function that is called on success or failure of an asynchronous request
@@ -1103,7 +1099,6 @@ cdef class InferRequest:
def set_completion_callback(self, py_callback, py_data = None):
self._py_callback = py_callback
self._py_data = py_data
self._py_callback_used = True
deref(self.impl).setCyCallback(<cb_type> self.user_callback, <void *> self)

cpdef BlobBuffer _get_blob_buffer(self, const string & blob_name):
@@ -1221,8 +1216,6 @@ cdef class InferRequest:
cpdef async_infer(self, inputs=None):
if inputs is not None:
self._fill_inputs(inputs)
if self._py_callback_used:
self._py_callback_called.clear()
with nogil:
deref(self.impl).infer_async()

@@ -1242,24 +1235,6 @@ cdef class InferRequest:
cpdef wait(self, timeout=None):
cdef int status
cdef int64_t c_timeout
cdef int c_wait_mode
if self._py_callback_used:
# check request status to avoid blocking for idle requests
c_wait_mode = WaitMode.STATUS_ONLY
with nogil:
status = deref(self.impl).wait(c_wait_mode)
if status != StatusCode.RESULT_NOT_READY:
return status
if not self._py_callback_called.is_set():
if timeout == WaitMode.RESULT_READY:
timeout = None
if timeout is not None:
# Convert milliseconds to seconds
timeout = float(timeout)/1000
if not self._py_callback_called.wait(timeout):
return StatusCode.REQUEST_BUSY
return StatusCode.OK

if timeout is None:
timeout = WaitMode.RESULT_READY
c_timeout = <int64_t> timeout
Original file line number Diff line number Diff line change
@@ -545,10 +545,10 @@ void InferenceEnginePython::IEExecNetwork::createInferRequests(int num_requests)
auto end_time = Time::now();
auto execTime = std::chrono::duration_cast<ns>(end_time - infer_request.start_time);
infer_request.exec_time = static_cast<double>(execTime.count()) * 0.000001;
infer_request.request_queue_ptr->setRequestIdle(infer_request.index);
if (infer_request.user_callback) {
infer_request.user_callback(infer_request.user_data, code);
}
infer_request.request_queue_ptr->setRequestIdle(infer_request.index);
});
}
}
21 changes: 21 additions & 0 deletions inference-engine/ie_bridges/python/tests/test_ExecutableNetwork.py
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@
import os
import pytest
import warnings
import time

from openvino.inference_engine import ie_api as ie
from conftest import model_path, image_path
@@ -173,6 +174,26 @@ def test_wait_before_start(device):
del ie_core


def test_wait_for_callback(device):
def callback(status, callbacks_info):
time.sleep(0.01)
callbacks_info['finished'] += 1

ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
num_requests = 3
exec_net = ie_core.load_network(net, device, num_requests=num_requests)
callbacks_info = {}
callbacks_info['finished'] = 0
img = read_image()
for request in exec_net.requests:
request.set_completion_callback(callback, callbacks_info)
request.async_infer({'data': img})

exec_net.wait(num_requests)
assert callbacks_info['finished'] == num_requests


def test_wrong_request_id(device):
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
21 changes: 20 additions & 1 deletion inference-engine/ie_bridges/python/tests/test_InferRequest.py
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
import warnings
import threading
from datetime import datetime
import time

from openvino.inference_engine import ie_api as ie
from conftest import model_path, image_path
@@ -349,7 +350,7 @@ def execute(self, input_data):
self.cv.release()
status = self.request.wait(ie.WaitMode.RESULT_READY)
assert status == ie.StatusCode.OK
assert self.status_code == ie.StatusCode.OK
assert self.status_code == ie.StatusCode.RESULT_NOT_READY

ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
@@ -361,6 +362,24 @@ def execute(self, input_data):
del ie_core


def test_async_infer_wait_while_callback_will_not_finish(device):
def callback(status, callback_status):
time.sleep(0.01)
callback_status['finished'] = True

ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=1)
callback_status = {}
callback_status['finished'] = False
request = exec_net.requests[0]
request.set_completion_callback(callback, py_data=callback_status)
img = read_image()
request.async_infer({'data': img})
request.wait()
assert callback_status['finished'] == True


@pytest.mark.ngraph_dependent_test
def test_get_perf_counts(device):
ie_core = ie.IECore()

0 comments on commit 6df94af

Please sign in to comment.