Auto plugin async infer request implementation (openvinotoolkit#5707)

* Async auto-request, now with revamped SetCallback (after openvinotoolkit#5645 merged) it is safe to set.
Also test modification to verify that the callback is called on the same (user's) request and e.g. not on the actual device's request

* Override CreateInferRequestImpl() instead of CreateInferRequest()

Signed-off-by: Shoujiang Ma <[email protected]>

Co-authored-by: myshevts <[email protected]>

inference-engine/include/cpp/ie_infer_request.hpp

@@ -217,6 +217,18 @@ class INFERENCE_ENGINE_API_CLASS(InferRequest) {
      * @return true if current InferRequest object is initialized, false - otherwise
      */
     explicit operator bool() const noexcept;
+
+    /**
+     * @brief Compares whether this request wraps the same impl underneath
+     * @return true if current InferRequest object doesn't wrap the same impl as the operator's arg
+     */
+    bool operator!=(const InferRequest&) const noexcept;
+
+    /**
+     * @brief Compares whether this request wraps the same impl underneath
+     * @return true if current InferRequest object wraps the same impl as the operator's arg
+     */
+    bool operator==(const InferRequest&) const noexcept;
 };
 
 template<>

inference-engine/src/auto_plugin/auto_exec_network.cpp

@@ -21,10 +21,10 @@ AutoExecutableNetwork::AutoExecutableNetwork(const SoExecutableNetworkInternal&
 
 AutoExecutableNetwork::~AutoExecutableNetwork() = default;
 
-IInferRequestInternal::Ptr AutoExecutableNetwork::CreateInferRequestImpl(InputsDataMap networkInputs,
+InferenceEngine::IInferRequestInternal::Ptr AutoExecutableNetwork::CreateInferRequestImpl(InputsDataMap networkInputs,
                                                                          OutputsDataMap networkOutputs) {
-    SoIInferRequestInternal inferRequest = { _network, _network->CreateInferRequest() };
-    return std::make_shared<AutoInferRequest>(networkInputs, networkOutputs, inferRequest);
+    SoIInferRequestInternal inferRequest = {_network, _network->CreateInferRequest()};
+    return std::make_shared<AutoInferRequest>(_networkInputs, _networkOutputs, inferRequest);
 }
 
 void AutoExecutableNetwork::Export(std::ostream& networkModel) {

inference-engine/src/auto_plugin/auto_exec_network.hpp

@@ -24,7 +24,7 @@ struct DeviceInformation {
     std::map<std::string, std::string> config;
 };
 
-class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafeDefault {
+class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkInternal {
 public:
     using Ptr = std::shared_ptr<AutoExecutableNetwork>;
 
@@ -38,6 +38,7 @@ class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSaf
     InferenceEngine::Parameter GetConfig(const std::string& name) const override;
     InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
                                                                        InferenceEngine::OutputsDataMap networkOutputs) override;
+
     ~AutoExecutableNetwork() override;
 
 private:

inference-engine/src/auto_plugin/auto_infer_request.cpp

@@ -36,4 +36,16 @@ void AutoInferRequest::Cancel() {
     _inferRequest->Cancel();
 }
 
+void AutoInferRequest::StartAsync() {
+    _inferRequest->StartAsync();
+}
+
+InferenceEngine::StatusCode AutoInferRequest::Wait(int64_t millis_timeout) {
+    return _inferRequest->Wait(millis_timeout);
+}
+
+void AutoInferRequest::SetCallback(Callback callback) {
+    _inferRequest->SetCallback(callback);
+}
+
 }  // namespace AutoPlugin

inference-engine/src/auto_plugin/auto_infer_request.hpp

@@ -30,6 +30,10 @@ class AutoInferRequest : public InferenceEngine::IInferRequestInternal {
     void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& data) override;
     InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override;
     void Cancel() override;
+    //async impl
+    void StartAsync() override;
+    InferenceEngine::StatusCode Wait(int64_t millis_timeout) override;
+    void SetCallback(Callback callback) override;
 
 private:
     InferenceEngine::SoIInferRequestInternal _inferRequest;

inference-engine/src/inference_engine/cpp/ie_infer_request.cpp

@@ -206,4 +206,12 @@ bool InferRequest::operator!() const noexcept {
 InferRequest::operator bool() const noexcept {
     return !!_impl;
 }
+
+bool InferRequest::operator!=(const InferRequest& r) const noexcept {
+    return !(r == *this);
+}
+
+bool InferRequest::operator==(const InferRequest& r) const noexcept {
+    return r._impl == _impl;
+}
 }  // namespace InferenceEngine

inference-engine/tests/functional/plugin/shared/include/behavior/infer_request_callback.hpp

@@ -36,8 +36,9 @@ TEST_P(CallbackTests, canCallSyncAndAsyncWithCompletionCallback) {
     // Create InferRequest
     InferenceEngine::InferRequest req = execNet.CreateInferRequest();
     bool isCalled = false;
-    req.SetCompletionCallback<std::function<void(InferenceEngine::InferRequest, InferenceEngine::StatusCode)>>(
+    req.SetCompletionCallback<std::function<void(InferenceEngine::InferRequest r, InferenceEngine::StatusCode)>>(
             [&](InferenceEngine::InferRequest request, InferenceEngine::StatusCode status) {
+                ASSERT_TRUE(req == request); //the callback is called on the same impl of the request
                 // HSD_1805940120: Wait on starting callback return HDDL_ERROR_INVAL_TASK_HANDLE
                 if (targetDevice != CommonTestUtils::DEVICE_HDDL) {
                     ASSERT_EQ(static_cast<int>(InferenceEngine::StatusCode::OK), status);