fix openvinotoolkit#18388 by recreate ov::Model using the main runt…

…ime and assign the shared objects to the recreated object
FredBill1 · Feb 17, 2024 · 8e32a92 · 8e32a92
1 parent 6accdd7
commit 8e32a92
Show file tree

Hide file tree

Showing 8 changed files with 52 additions and 9 deletions.
diff --git a/src/core/include/openvino/core/model.hpp b/src/core/include/openvino/core/model.hpp
@@ -27,6 +27,8 @@
 
 namespace ov {
 class Model;
+class CompiledModel;
+class ICompiledModel;
 
 std::shared_ptr<Model> clone_ov_model(const Model& func, std::unordered_map<Node*, std::shared_ptr<Node>>& node_map);
 
@@ -42,9 +44,11 @@ class ModelAccessor;
  */
 class OPENVINO_API Model : public std::enable_shared_from_this<Model> {
     friend class frontend::FrontEnd;
+    friend class ov::CompiledModel;
+    friend class ov::ICompiledModel;
     friend std::shared_ptr<Model> clone_ov_model(const Model& func,
                                                  std::unordered_map<Node*, std::shared_ptr<Node>>& node_map);
-    std::shared_ptr<void> m_shared_object;  // Frontend plugin shared object handle.
+    std::shared_ptr<void> m_shared_object;  // plugin shared object handle.
 
 public:
     _OPENVINO_HIDDEN_METHOD static const ::ov::DiscreteTypeInfo& get_type_info_static() {

diff --git a/src/inference/dev_api/openvino/runtime/icompiled_model.hpp b/src/inference/dev_api/openvino/runtime/icompiled_model.hpp
@@ -179,6 +179,8 @@ class OPENVINO_RUNTIME_API ICompiledModel : public std::enable_shared_from_this<
     const std::shared_ptr<ov::threading::ITaskExecutor> get_callback_executor() const;
     void set_task_executor(const std::shared_ptr<ov::threading::ITaskExecutor> task_executor);
     void set_callback_executor(const std::shared_ptr<ov::threading::ITaskExecutor> callback_executor);
+
+    static void set_model_shared_object(ov::Model& model, const std::shared_ptr<void>& shared_object);
 };
 
 }  // namespace ov
diff --git a/src/inference/src/cpp/compiled_model.cpp b/src/inference/src/cpp/compiled_model.cpp
@@ -32,7 +32,27 @@ CompiledModel::CompiledModel(const std::shared_ptr<ov::ICompiledModel>& impl, co
 }
 
 std::shared_ptr<const Model> CompiledModel::get_runtime_model() const {
-    OV_COMPILED_MODEL_CALL_STATEMENT(return _impl->get_runtime_model());
+    OV_COMPILED_MODEL_CALL_STATEMENT({
+        auto model = _impl->get_runtime_model();
+        // Recreate ov::Model using main runtime, not Plugin's one
+        auto copy = std::make_shared<Model>(model->get_results(),
+                                            model->get_sinks(),
+                                            model->get_parameters(),
+                                            model->get_variables(),
+                                            model->get_friendly_name());
+        struct SharedObject {
+            // destroy the shared object from runtime model before it from compiled model
+            std::shared_ptr<void> compiled_model_so;
+            std::shared_ptr<void> model_so;
+            SharedObject(const std::shared_ptr<void>& compiled_model_so, const std::shared_ptr<void>& model_so)
+                : compiled_model_so(compiled_model_so),
+                  model_so(model_so) {}
+        };
+        auto shared_object = std::make_shared<SharedObject>(_so, model->m_shared_object);
+        copy->m_shared_object = std::reinterpret_pointer_cast<void>(shared_object);
+        copy->get_rt_info() = model->get_rt_info();
+        return copy;
+    });
 }
 
 const std::vector<ov::Output<const ov::Node>>& CompiledModel::inputs() const {

diff --git a/src/inference/src/dev/icompiled_model.cpp b/src/inference/src/dev/icompiled_model.cpp
@@ -143,3 +143,7 @@ ov::SoPtr<ov::IRemoteContext> ov::ICompiledModel::get_context() const {
         return m_context;
     return m_plugin->get_default_context({});
 }
+
+void ov::ICompiledModel::set_model_shared_object(ov::Model& model, const std::shared_ptr<void>& shared_object) {
+    model.m_shared_object = shared_object;
+}
diff --git a/src/plugins/auto/src/auto_compiled_model.cpp b/src/plugins/auto/src/auto_compiled_model.cpp
@@ -32,7 +32,9 @@ void AutoCompiledModel::set_property(const ov::AnyMap& properties) {
 
 std::shared_ptr<const ov::Model> AutoCompiledModel::get_runtime_model() const {
     OPENVINO_ASSERT(m_context->m_hw_compiled_model);
-    return m_context->m_hw_compiled_model->get_runtime_model();
+    auto model = m_context->m_hw_compiled_model->get_runtime_model();
+    set_model_shared_object(const_cast<ov::Model&>(*model), m_context->m_hw_compiled_model._so);
+    return model;
 }
 
 ov::Any AutoCompiledModel::get_property(const std::string& name) const {

diff --git a/src/plugins/auto/src/cumulative_compiled_model.cpp b/src/plugins/auto/src/cumulative_compiled_model.cpp
@@ -29,8 +29,11 @@ void AutoCumuCompiledModel::set_property(const ov::AnyMap& properties) {
 }
 
 std::shared_ptr<const ov::Model> AutoCumuCompiledModel::get_runtime_model() const {
-    if (m_context->m_hw_compiled_model)
-        return m_context->m_hw_compiled_model->get_runtime_model();
+    if (m_context->m_hw_compiled_model) {
+        auto model = m_context->m_hw_compiled_model->get_runtime_model();
+        set_model_shared_object(const_cast<ov::Model&>(*model), m_context->m_hw_compiled_model._so);
+        return model;
+    }
     OPENVINO_NOT_IMPLEMENTED;
 }
 

diff --git a/src/plugins/auto_batch/src/compiled_model.cpp b/src/plugins/auto_batch/src/compiled_model.cpp
@@ -154,8 +154,10 @@ std::shared_ptr<ov::IAsyncInferRequest> CompiledModel::create_infer_request() co
 }
 
 std::shared_ptr<const ov::Model> CompiledModel::get_runtime_model() const {
-    return m_compiled_model_with_batch ? m_compiled_model_with_batch->get_runtime_model()
-                                       : m_compiled_model_without_batch->get_runtime_model();
+    auto& compiled_model = m_compiled_model_with_batch ? m_compiled_model_with_batch : m_compiled_model_without_batch;
+    auto model = compiled_model->get_runtime_model();
+    set_model_shared_object(const_cast<ov::Model&>(*model), compiled_model._so);
+    return model;
 }
 
 void CompiledModel::set_property(const ov::AnyMap& properties) {

diff --git a/src/plugins/hetero/src/compiled_model.cpp b/src/plugins/hetero/src/compiled_model.cpp
@@ -248,13 +248,19 @@ void ov::hetero::CompiledModel::set_property(const ov::AnyMap& properties) {
 
 std::shared_ptr<const ov::Model> ov::hetero::CompiledModel::get_runtime_model() const {
     std::vector<std::shared_ptr<ov::Model>> rt_models;
+    std::vector<std::shared_ptr<void>> shared_objects;
     // Collect runtime subgraphs
-    for (size_t i = 0; i < m_compiled_submodels.size(); i++) {
-        rt_models.push_back(m_compiled_submodels.at(i).compiled_model->get_runtime_model()->clone());
+    rt_models.reserve(m_compiled_submodels.size());
+    shared_objects.reserve(m_compiled_submodels.size());
+    for (auto& compiled_submodel: m_compiled_submodels) {
+        rt_models.push_back(compiled_submodel.compiled_model->get_runtime_model()->clone());
+        shared_objects.push_back(compiled_submodel.compiled_model._so);
     }
     ov::hetero::merge_submodels(rt_models, m_mapping_info._submodels_input_to_prev_output);
     auto& runtime_graph = rt_models[0];
     OPENVINO_ASSERT(runtime_graph->inputs().size() == inputs().size());
+    auto merged_shared_object = std::make_shared<std::vector<std::shared_ptr<void>>>(std::move(shared_objects));
+    set_model_shared_object(*runtime_graph, std::reinterpret_pointer_cast<void>(merged_shared_object));
     return runtime_graph;
 }