Skip to content

Commit

Permalink
Update AsyncInferRequest for InferRequestLegacy compatibility
Browse files Browse the repository at this point in the history
Signed-off-by: Andrew Park <[email protected]>
  • Loading branch information
andrew-k-park committed May 18, 2022
1 parent 16a60b5 commit f57a7e4
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <string>
#include <map>
#include <cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp>
#include "intel_gpu/plugin/infer_request_legacy.hpp"
#include "intel_gpu/plugin/infer_request.hpp"

namespace ov {
Expand All @@ -16,19 +17,21 @@ namespace intel_gpu {
class AsyncInferRequest : public InferenceEngine::AsyncInferRequestThreadSafeDefault {
public:
using Parent = InferenceEngine::AsyncInferRequestThreadSafeDefault;
AsyncInferRequest(const InferRequest::Ptr &inferRequest,
AsyncInferRequest(const IInferRequestInternal::Ptr &inferRequest,
const InferenceEngine::ITaskExecutor::Ptr& taskExecutor,
const InferenceEngine::ITaskExecutor::Ptr& waitExecutor,
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor);
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor,
const bool isLegacy);

~AsyncInferRequest();

void Infer_ThreadUnsafe() override;
void StartAsync_ThreadUnsafe() override;

private:
InferRequest::Ptr _inferRequest;
IInferRequestInternal::Ptr _inferRequest;
InferenceEngine::ITaskExecutor::Ptr _waitExecutor;
bool _isLegacy;
};

} // namespace intel_gpu
Expand Down
87 changes: 63 additions & 24 deletions src/plugins/intel_gpu/src/plugin/async_infer_request.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,45 +10,84 @@ namespace ov {
namespace runtime {
namespace intel_gpu {

AsyncInferRequest::AsyncInferRequest(const InferRequest::Ptr &inferRequest,
AsyncInferRequest::AsyncInferRequest(const IInferRequestInternal::Ptr &inferRequest,
const InferenceEngine::ITaskExecutor::Ptr& taskExecutor,
const InferenceEngine::ITaskExecutor::Ptr& waitExecutor,
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor)
: AsyncInferRequestThreadSafeDefault(inferRequest, taskExecutor, callbackExecutor), _inferRequest(inferRequest), _waitExecutor(waitExecutor) {
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor,
const bool isLegacy)
: AsyncInferRequestThreadSafeDefault(inferRequest, taskExecutor, callbackExecutor),
_inferRequest(inferRequest),
_waitExecutor(waitExecutor),
_isLegacy(isLegacy) {
_pipeline = {};

if (!_inferRequest->use_external_queue()) {
_pipeline.push_back({taskExecutor,
[this] {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "AsyncInferRequest::PreprocessingAndStartPipeline");
_inferRequest->setup_stream_graph();
_inferRequest->preprocess();
_inferRequest->enqueue();
_inferRequest->wait();
} });
if (_isLegacy) {
if (!std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->use_external_queue()) {
_pipeline.push_back({taskExecutor,
[this] {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "AsyncInferRequest::PreprocessingAndStartPipeline");
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->setup_stream_graph();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->preprocess();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->enqueue();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->wait();
} });
} else {
_pipeline.push_back({ _waitExecutor,
[this] {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "AsyncInferRequest::WaitPipeline");
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->wait_notify();
} });
}
} else {
_pipeline.push_back({ _waitExecutor,
if (!std::static_pointer_cast<InferRequest>(_inferRequest)->use_external_queue()) {
_pipeline.push_back({taskExecutor,
[this] {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "AsyncInferRequest::WaitPipeline");
_inferRequest->wait_notify();
} });
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "AsyncInferRequest::PreprocessingAndStartPipeline");
std::static_pointer_cast<InferRequest>(_inferRequest)->setup_stream_graph();
std::static_pointer_cast<InferRequest>(_inferRequest)->preprocess();
std::static_pointer_cast<InferRequest>(_inferRequest)->enqueue();
std::static_pointer_cast<InferRequest>(_inferRequest)->wait();
} });
} else {
_pipeline.push_back({ _waitExecutor,
[this] {
OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "AsyncInferRequest::WaitPipeline");
std::static_pointer_cast<InferRequest>(_inferRequest)->wait_notify();
} });
}
}
}

void AsyncInferRequest::Infer_ThreadUnsafe() {
if (_inferRequest->use_external_queue()) {
_inferRequest->setup_stream_graph();
_inferRequest->preprocess_notify();
_inferRequest->enqueue_notify();
if (_isLegacy) {
if (std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->use_external_queue()) {
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->setup_stream_graph();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->preprocess_notify();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->enqueue_notify();
}
} else {
if (std::static_pointer_cast<InferRequest>(_inferRequest)->use_external_queue()) {
std::static_pointer_cast<InferRequest>(_inferRequest)->setup_stream_graph();
std::static_pointer_cast<InferRequest>(_inferRequest)->preprocess_notify();
std::static_pointer_cast<InferRequest>(_inferRequest)->enqueue_notify();
}
}
Parent::Infer_ThreadUnsafe();
}

void AsyncInferRequest::StartAsync_ThreadUnsafe() {
if (_inferRequest->use_external_queue()) {
_inferRequest->setup_stream_graph();
_inferRequest->preprocess_notify();
_inferRequest->enqueue_notify();
if (_isLegacy) {
if (std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->use_external_queue()) {
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->setup_stream_graph();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->preprocess_notify();
std::static_pointer_cast<InferRequestLegacy>(_inferRequest)->enqueue_notify();
}
} else {
if (std::static_pointer_cast<InferRequest>(_inferRequest)->use_external_queue()) {
std::static_pointer_cast<InferRequest>(_inferRequest)->setup_stream_graph();
std::static_pointer_cast<InferRequest>(_inferRequest)->preprocess_notify();
std::static_pointer_cast<InferRequest>(_inferRequest)->enqueue_notify();
}
}
Parent::StartAsync_ThreadUnsafe();
}
Expand Down
10 changes: 7 additions & 3 deletions src/plugins/intel_gpu/src/plugin/compiled_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -119,18 +119,22 @@ IInferRequestInternal::Ptr CompiledModel::CreateInferRequest() {
}
}

bool is_legacy = false;
if (this->_plugin) {
const auto& core = _plugin->GetCore();
if (core && core->isNewAPI())
internalRequest = CreateInferRequestImpl(_parameters, _results);
}
if (!internalRequest)
if (!internalRequest) {
internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs);
is_legacy = true;
}
internalRequest->setPointerToExecutableNetworkInternal(shared_from_this());
return std::make_shared<AsyncInferRequest>(std::static_pointer_cast<InferRequest>(internalRequest),
return std::make_shared<AsyncInferRequest>(internalRequest,
m_taskExecutor,
m_waitExecutor,
_callbackExecutor);
_callbackExecutor,
is_legacy);
}

std::shared_ptr<ngraph::Function> CompiledModel::GetExecGraphInfo() {
Expand Down

0 comments on commit f57a7e4

Please sign in to comment.