From 8e32a92c5e29e4cf74fc1599574a2a5ea4849d0b Mon Sep 17 00:00:00 2001 From: FredBill1 <36622430+FredBill1@users.noreply.github.com> Date: Sat, 17 Feb 2024 13:55:36 +0000 Subject: [PATCH] fix #18388 by recreate `ov::Model` using the main runtime and assign the shared objects to the recreated object --- src/core/include/openvino/core/model.hpp | 6 ++++- .../openvino/runtime/icompiled_model.hpp | 2 ++ src/inference/src/cpp/compiled_model.cpp | 22 ++++++++++++++++++- src/inference/src/dev/icompiled_model.cpp | 4 ++++ src/plugins/auto/src/auto_compiled_model.cpp | 4 +++- .../auto/src/cumulative_compiled_model.cpp | 7 ++++-- src/plugins/auto_batch/src/compiled_model.cpp | 6 +++-- src/plugins/hetero/src/compiled_model.cpp | 10 +++++++-- 8 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/core/include/openvino/core/model.hpp b/src/core/include/openvino/core/model.hpp index 2375f8d60c09ee..609da76a9ed39f 100644 --- a/src/core/include/openvino/core/model.hpp +++ b/src/core/include/openvino/core/model.hpp @@ -27,6 +27,8 @@ namespace ov { class Model; +class CompiledModel; +class ICompiledModel; std::shared_ptr clone_ov_model(const Model& func, std::unordered_map>& node_map); @@ -42,9 +44,11 @@ class ModelAccessor; */ class OPENVINO_API Model : public std::enable_shared_from_this { friend class frontend::FrontEnd; + friend class ov::CompiledModel; + friend class ov::ICompiledModel; friend std::shared_ptr clone_ov_model(const Model& func, std::unordered_map>& node_map); - std::shared_ptr m_shared_object; // Frontend plugin shared object handle. + std::shared_ptr m_shared_object; // plugin shared object handle. public: _OPENVINO_HIDDEN_METHOD static const ::ov::DiscreteTypeInfo& get_type_info_static() { diff --git a/src/inference/dev_api/openvino/runtime/icompiled_model.hpp b/src/inference/dev_api/openvino/runtime/icompiled_model.hpp index b4b2e472deba80..3144f89dbdc363 100644 --- a/src/inference/dev_api/openvino/runtime/icompiled_model.hpp +++ b/src/inference/dev_api/openvino/runtime/icompiled_model.hpp @@ -179,6 +179,8 @@ class OPENVINO_RUNTIME_API ICompiledModel : public std::enable_shared_from_this< const std::shared_ptr get_callback_executor() const; void set_task_executor(const std::shared_ptr task_executor); void set_callback_executor(const std::shared_ptr callback_executor); + + static void set_model_shared_object(ov::Model& model, const std::shared_ptr& shared_object); }; } // namespace ov diff --git a/src/inference/src/cpp/compiled_model.cpp b/src/inference/src/cpp/compiled_model.cpp index a3c9f2ccdb37b3..c8bc22368ba2b2 100644 --- a/src/inference/src/cpp/compiled_model.cpp +++ b/src/inference/src/cpp/compiled_model.cpp @@ -32,7 +32,27 @@ CompiledModel::CompiledModel(const std::shared_ptr& impl, co } std::shared_ptr CompiledModel::get_runtime_model() const { - OV_COMPILED_MODEL_CALL_STATEMENT(return _impl->get_runtime_model()); + OV_COMPILED_MODEL_CALL_STATEMENT({ + auto model = _impl->get_runtime_model(); + // Recreate ov::Model using main runtime, not Plugin's one + auto copy = std::make_shared(model->get_results(), + model->get_sinks(), + model->get_parameters(), + model->get_variables(), + model->get_friendly_name()); + struct SharedObject { + // destroy the shared object from runtime model before it from compiled model + std::shared_ptr compiled_model_so; + std::shared_ptr model_so; + SharedObject(const std::shared_ptr& compiled_model_so, const std::shared_ptr& model_so) + : compiled_model_so(compiled_model_so), + model_so(model_so) {} + }; + auto shared_object = std::make_shared(_so, model->m_shared_object); + copy->m_shared_object = std::reinterpret_pointer_cast(shared_object); + copy->get_rt_info() = model->get_rt_info(); + return copy; + }); } const std::vector>& CompiledModel::inputs() const { diff --git a/src/inference/src/dev/icompiled_model.cpp b/src/inference/src/dev/icompiled_model.cpp index d4e91b296074d4..e0ea91c58b7a2d 100644 --- a/src/inference/src/dev/icompiled_model.cpp +++ b/src/inference/src/dev/icompiled_model.cpp @@ -143,3 +143,7 @@ ov::SoPtr ov::ICompiledModel::get_context() const { return m_context; return m_plugin->get_default_context({}); } + +void ov::ICompiledModel::set_model_shared_object(ov::Model& model, const std::shared_ptr& shared_object) { + model.m_shared_object = shared_object; +} diff --git a/src/plugins/auto/src/auto_compiled_model.cpp b/src/plugins/auto/src/auto_compiled_model.cpp index e35fdc5a8c9004..0b22c13ee254c0 100644 --- a/src/plugins/auto/src/auto_compiled_model.cpp +++ b/src/plugins/auto/src/auto_compiled_model.cpp @@ -32,7 +32,9 @@ void AutoCompiledModel::set_property(const ov::AnyMap& properties) { std::shared_ptr AutoCompiledModel::get_runtime_model() const { OPENVINO_ASSERT(m_context->m_hw_compiled_model); - return m_context->m_hw_compiled_model->get_runtime_model(); + auto model = m_context->m_hw_compiled_model->get_runtime_model(); + set_model_shared_object(const_cast(*model), m_context->m_hw_compiled_model._so); + return model; } ov::Any AutoCompiledModel::get_property(const std::string& name) const { diff --git a/src/plugins/auto/src/cumulative_compiled_model.cpp b/src/plugins/auto/src/cumulative_compiled_model.cpp index 3ec7a820256d67..c0e6e3e2516fb6 100644 --- a/src/plugins/auto/src/cumulative_compiled_model.cpp +++ b/src/plugins/auto/src/cumulative_compiled_model.cpp @@ -29,8 +29,11 @@ void AutoCumuCompiledModel::set_property(const ov::AnyMap& properties) { } std::shared_ptr AutoCumuCompiledModel::get_runtime_model() const { - if (m_context->m_hw_compiled_model) - return m_context->m_hw_compiled_model->get_runtime_model(); + if (m_context->m_hw_compiled_model) { + auto model = m_context->m_hw_compiled_model->get_runtime_model(); + set_model_shared_object(const_cast(*model), m_context->m_hw_compiled_model._so); + return model; + } OPENVINO_NOT_IMPLEMENTED; } diff --git a/src/plugins/auto_batch/src/compiled_model.cpp b/src/plugins/auto_batch/src/compiled_model.cpp index 8cc37d612c8518..66f8f170143f21 100644 --- a/src/plugins/auto_batch/src/compiled_model.cpp +++ b/src/plugins/auto_batch/src/compiled_model.cpp @@ -154,8 +154,10 @@ std::shared_ptr CompiledModel::create_infer_request() co } std::shared_ptr CompiledModel::get_runtime_model() const { - return m_compiled_model_with_batch ? m_compiled_model_with_batch->get_runtime_model() - : m_compiled_model_without_batch->get_runtime_model(); + auto& compiled_model = m_compiled_model_with_batch ? m_compiled_model_with_batch : m_compiled_model_without_batch; + auto model = compiled_model->get_runtime_model(); + set_model_shared_object(const_cast(*model), compiled_model._so); + return model; } void CompiledModel::set_property(const ov::AnyMap& properties) { diff --git a/src/plugins/hetero/src/compiled_model.cpp b/src/plugins/hetero/src/compiled_model.cpp index d44c8be8259b50..750f462242f929 100644 --- a/src/plugins/hetero/src/compiled_model.cpp +++ b/src/plugins/hetero/src/compiled_model.cpp @@ -248,13 +248,19 @@ void ov::hetero::CompiledModel::set_property(const ov::AnyMap& properties) { std::shared_ptr ov::hetero::CompiledModel::get_runtime_model() const { std::vector> rt_models; + std::vector> shared_objects; // Collect runtime subgraphs - for (size_t i = 0; i < m_compiled_submodels.size(); i++) { - rt_models.push_back(m_compiled_submodels.at(i).compiled_model->get_runtime_model()->clone()); + rt_models.reserve(m_compiled_submodels.size()); + shared_objects.reserve(m_compiled_submodels.size()); + for (auto& compiled_submodel: m_compiled_submodels) { + rt_models.push_back(compiled_submodel.compiled_model->get_runtime_model()->clone()); + shared_objects.push_back(compiled_submodel.compiled_model._so); } ov::hetero::merge_submodels(rt_models, m_mapping_info._submodels_input_to_prev_output); auto& runtime_graph = rt_models[0]; OPENVINO_ASSERT(runtime_graph->inputs().size() == inputs().size()); + auto merged_shared_object = std::make_shared>>(std::move(shared_objects)); + set_model_shared_object(*runtime_graph, std::reinterpret_pointer_cast(merged_shared_object)); return runtime_graph; }