Fixed issue with run stateful network with several infer requests on …

…MKLDNNPlugin (openvinotoolkit#3711)
ilya-lavrenov · Jan 21, 2021 · 05d97fa · 05d97fa
1 parent 88b200e
commit 05d97fa
Show file tree

Hide file tree

Showing 6 changed files with 232 additions and 43 deletions.
diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2018-2020 Intel Corporation
+// Copyright (C) 2018-2021 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
 
@@ -16,6 +16,7 @@
 #include "nodes/common/cpu_convert.h"
 #include "mkldnn_memory_state.h"
 #include "nodes/mkldnn_memory_node.hpp"
+#include "nodes/common/cpu_memcpy.h"
 
 MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsDataMap     networkInputs,
                                                      InferenceEngine::OutputsDataMap    networkOutputs,
@@ -42,7 +43,7 @@ MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsData
     // of MemoryLayer implementation. It uses output edge of MemoryLayer
     // producer as storage for tensor to keep it between infer calls.
     IE_SUPPRESS_DEPRECATED_START
-    if (execNetwork->QueryState().size() == 0) {
+    if (execNetwork->_numRequests > 1 || execNetwork->QueryState().size() == 0) {
         for (auto &node : graph->GetNodes()) {
             if (node->getType() == MemoryInput) {
                 auto memoryNode = dynamic_cast<MKLDNNMemoryInputNode*>(node.get());
@@ -132,6 +133,49 @@ void MKLDNNPlugin::MKLDNNInferRequest::PushInputData() {
     }
 }
 
+void MKLDNNPlugin::MKLDNNInferRequest::PushStates() {
+    for (auto &node : graph->GetNodes()) {
+        if (node->getType() == MemoryInput) {
+            auto cur_node = dynamic_cast<MKLDNNMemoryInputNode*>(node.get());
+            auto cur_id = cur_node->getId();
+            for (const auto& state : memoryStates) {
+                if (state->GetName() == cur_id) {
+                    auto cur_state_mem = cur_node->getStore();
+                    auto data_ptr = state->GetState()->cbuffer().as<void*>();
+                    auto data_size = state->GetState()->byteSize();
+                    auto elemSize = MKLDNNExtensionUtils::sizeOfDataType(cur_state_mem->GetDataType());
+                    auto padSize = cur_state_mem->GetDescriptor().data.layout_desc.blocking.offset_padding;
+                    auto cur_state_mem_buf = static_cast<uint8_t*>(cur_state_mem->GetData()) + padSize * elemSize;
+
+                    cpu_memcpy(cur_state_mem_buf, data_ptr, data_size);
+                }
+            }
+        }
+    }
+}
+
+void MKLDNNPlugin::MKLDNNInferRequest::PullStates() {
+    for (auto &node : graph->GetNodes()) {
+        if (node->getType() == MemoryInput) {
+            auto cur_node = dynamic_cast<MKLDNNMemoryInputNode*>(node.get());
+            auto cur_id = cur_node->getId();
+            for (const auto& state : memoryStates) {
+                if (state->GetName() == cur_id) {
+                    auto cur_state_mem = cur_node->getStore();
+                    auto data_ptr = state->GetState()->cbuffer().as<void*>();
+                    auto data_size = state->GetState()->byteSize();
+                    auto elemSize = MKLDNNExtensionUtils::sizeOfDataType(cur_state_mem->GetDataType());
+                    auto padSize = cur_state_mem->GetDescriptor().data.layout_desc.blocking.offset_padding;
+                    auto cur_state_mem_buf = static_cast<uint8_t*>(cur_state_mem->GetData()) + padSize * elemSize;
+
+                    cpu_memcpy(data_ptr, cur_state_mem_buf, data_size);
+                }
+            }
+        }
+    }
+}
+
+
 void MKLDNNPlugin::MKLDNNInferRequest::InferImpl() {
     using namespace openvino::itt;
     OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, profilingTask);
@@ -144,8 +188,16 @@ void MKLDNNPlugin::MKLDNNInferRequest::InferImpl() {
 
     PushInputData();
 
+    if (memoryStates.size() != 0) {
+        PushStates();
+    }
+
     graph->Infer(m_curBatch);
 
+    if (memoryStates.size() != 0) {
+        PullStates();
+    }
+
     graph->PullOutputData(_outputs);
 }
 

diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2018-2020 Intel Corporation
+// Copyright (C) 2018-2021 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
 
@@ -49,6 +49,8 @@ class MKLDNNInferRequest : public InferenceEngine::InferRequestInternal {
 
 private:
     void PushInputData();
+    void PushStates();
+    void PullStates();
 
     void pushInput(const std::string& inputName, InferenceEngine::Blob::Ptr& inputBlob, InferenceEngine::Precision dataType);
 

diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_memory_state.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_memory_state.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2018-2020 Intel Corporation
+// Copyright (C) 2018-2021 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
 
@@ -15,24 +15,15 @@ std::string  MKLDNNVariableState::GetName() const {
 }
 
 void  MKLDNNVariableState::Reset() {
-    storage->FillZero();
+    std::memset(this->storage->buffer(), 0, storage->byteSize());
 }
 
 void  MKLDNNVariableState::SetState(Blob::Ptr newState) {
-    auto prec = newState->getTensorDesc().getPrecision();
-    auto data_type = MKLDNNExtensionUtils::IEPrecisionToDataType(prec);
-    auto data_layout = MKLDNNMemory::Convert(newState->getTensorDesc().getLayout());
-    auto data_ptr = newState->cbuffer().as<void*>();
-    auto data_size = newState->byteSize();
-
-    storage->SetData(data_type, data_layout, data_ptr, data_size);
+    storage = newState;
 }
 
 InferenceEngine::Blob::CPtr MKLDNNVariableState::GetState() const {
-    auto result_blob = make_blob_with_precision(MKLDNNMemoryDesc(storage->GetDescriptor()));
-    result_blob->allocate();
-    std::memcpy(result_blob->buffer(), storage->GetData(), storage->GetSize());
-    return result_blob;
+    return storage;
 }
 
 }  // namespace MKLDNNPlugin
diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_memory_state.h b/inference-engine/src/mkldnn_plugin/mkldnn_memory_state.h
@@ -1,11 +1,13 @@
-// Copyright (C) 2018-2020 Intel Corporation
+// Copyright (C) 2018-2021 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
 
 #pragma once
 
 #include "cpp_interfaces/impl/ie_variable_state_internal.hpp"
+#include "blob_factory.hpp"
 #include "mkldnn_memory.h"
+#include "nodes/common/cpu_memcpy.h"
 
 #include <string>
 
@@ -14,7 +16,11 @@ namespace MKLDNNPlugin {
 class MKLDNNVariableState : public InferenceEngine::IVariableStateInternal {
 public:
     MKLDNNVariableState(std::string name, MKLDNNMemoryPtr storage) :
-            name(name), storage(storage) {}
+            name(name) {
+        this->storage = make_blob_with_precision(MKLDNNMemoryDesc(storage->GetDescriptor()));
+        this->storage->allocate();
+        cpu_memcpy(this->storage->buffer(), storage->GetData(), storage->GetSize());
+    }
 
     std::string GetName() const override;
     void Reset() override;
@@ -23,7 +29,7 @@ class MKLDNNVariableState : public InferenceEngine::IVariableStateInternal {
 
 private:
     std::string name;
-    MKLDNNMemoryPtr storage;
+    InferenceEngine::Blob::Ptr storage;
 };
 
 }  // namespace MKLDNNPlugin
diff --git a/inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 //
 
@@ -51,6 +51,8 @@ std::vector<std::string> disabledTestPatterns() {
         R"(.*(ConstantResultSubgraphTest).*)",
         // TODO: Issue: 29577
         R"(.*CoreThreadingTests.smoke_QueryNetwork.*)",
+        //TODO: Issue: 46416
+        R"(.*VariableStateTest.inferreq_smoke_VariableState_2infers*.*)",
         // TODO: Issue 24839
         R"(.*ConvolutionLayerTest.CompareWithRefs.*D=\(1.3\).*)",
         R"(.*ConvolutionLayerTest.CompareWithRefs.*D=\(3.1\).*)"