[CONFORMANCE][SUBGRAPHS DUMPER] Rework subgraphs_dumper graphs extr…

…action algo feedback by plugins (openvinotoolkit#19669) * [CONFORMANCE][SUBGRAPHS DUMPER] Change repeat pattern extractor to avoid duplications and reduce graphs size * Small change * temporary * merge * try to handle large models * Fixes + tests * Remove extra * Exclude models after const folding in case dynamic modesl * shapes to meta * Fix tests * Fix test + is_subgraph * Fix issue with default output * change hashing * Check memory * Hash algo * correct modelsize check * Log large models * tmp disable fused_names extractor * add device for fused_names * remove extra * fix vuild * Disable fused_names extractor
cavusmustafa · Oct 4, 2023 · cdcbb1d · cdcbb1d
1 parent d7be40b
commit cdcbb1d
Show file tree

Hide file tree

Showing 33 changed files with 764 additions and 171 deletions.
diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/cache.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/cache.hpp
@@ -28,9 +28,33 @@ class ICache {
         m_serialization_dir = serialization_dir;
     }
 
+    bool is_model_large_to_read(const std::shared_ptr<ov::Model>& model, const std::string& model_path) {
+        // ov::Model + ov::CompiledModel
+        auto model_bytesize = model->get_graph_size();
+        if (2 * model_bytesize >= mem_size) {
+            auto model_bytesize_gb = model_bytesize;
+            model_bytesize_gb >>= 30;
+            auto mem_size_gb = mem_size;
+            mem_size_gb >>= 30;
+            // std::cout << "[ WARNING ] Model " << model_path << " bytesize is " << model_bytesize_gb <<
+            // "is larger than RAM size: " << mem_size_gb << ". Model will be skipped!" << std::endl;
+            return true;
+        }
+        return false;
+    }
+
+    bool is_model_large_to_store_const(const std::shared_ptr<ov::Model>& model) {
+        auto model_bytesize = model->get_graph_size();
+        if (mem_size < model_bytesize * 4) {
+            return true;
+        }
+        return false;
+    }
+
 protected:
     size_t m_serialization_timeout = 60;
     std::string m_serialization_dir = ".";
+    static size_t mem_size;
 
     ICache() = default;
 

diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/graph_cache.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/graph_cache.hpp
@@ -19,12 +19,13 @@ class GraphCache : public ICache {
 public:
     void update_cache(const std::shared_ptr<ov::Model>& model,
                       const std::string& model_meta_data,
-                      bool extract_body, bool from_cache = false) override;
+                      bool extract_body,
+                      bool from_cache = false) override;
     void serialize_cache() override;
 
-    static std::shared_ptr<GraphCache>& get() {
+    static std::shared_ptr<GraphCache>& get(const std::string& device = "") {
         if (m_cache_instance == nullptr) {
-            m_cache_instance = std::shared_ptr<GraphCache>(new GraphCache);
+            m_cache_instance = std::shared_ptr<GraphCache>(new GraphCache(device));
         }
         return m_cache_instance;
     }
@@ -46,19 +47,21 @@ class GraphCache : public ICache {
     // cache byte size
     uint64_t m_graph_cache_bytesize = 0;
 
-    GraphCache() {
+    GraphCache(const std::string& device = "") {
         ExtractorsManager::ExtractorsMap matchers = {
             // temporary disabling according mem leaks in CI and not using swap mem
-            { "fused_names", FusedNamesExtractor::Ptr(new FusedNamesExtractor) },
+            // { "fused_names", FusedNamesExtractor::Ptr(new FusedNamesExtractor(device)) },
             { "repeat_pattern", RepeatPatternExtractor::Ptr(new RepeatPatternExtractor) },
         };
         m_manager.set_extractors(matchers);
         m_cache_subdir = "subgraph";
     }
 
-    void update_cache(const std::shared_ptr<ov::Model>& model, const std::string& model_path,
-                      std::map<std::string, InputInfo>& input_info, const std::string& extractor_name,
-                      size_t model_op_cnt, bool from_cache = false);
+    void update_cache(const std::shared_ptr<ov::Model>& model,
+                      const std::string& model_path,
+                      std::map<std::string, InputInfo>& input_info,
+                      const std::string& extractor_name,
+                      size_t model_op_cnt);
 };
 
 }  // namespace subgraph_dumper

diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/input_info.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/input_info.hpp
@@ -40,15 +40,33 @@ struct InputInfo {
 
     Range ranges;
     bool is_const;
+    ov::PartialShape max_shape, min_shape;
 
-    InputInfo(double in_min = DEFAULT_MIN_VALUE,
+    InputInfo(const ov::PartialShape& shape = {},
+              double in_min = DEFAULT_MIN_VALUE,
               double in_max = DEFAULT_MAX_VALUE,
               bool in_is_const = false) :
               is_const(in_is_const),
-              ranges(Range(in_min, in_max)) {}
+              ranges(Range(in_min, in_max)),
+              max_shape(shape),
+              min_shape(shape) {}
 
     bool operator==(const InputInfo& input_info_ref) const {
-        return this->is_const == input_info_ref.is_const && this->ranges == input_info_ref.ranges;
+        return this->is_const == input_info_ref.is_const &&
+               this->ranges == input_info_ref.ranges &&
+               this->max_shape == input_info_ref.max_shape &&
+               this->min_shape == input_info_ref.min_shape;
+    }
+
+    InputInfo operator=(const InputInfo& input_info) {
+        this->ranges = input_info.ranges;
+        if (ov::shape_size(this->max_shape.get_max_shape()) < ov::shape_size(input_info.max_shape.get_max_shape())) {
+            this->max_shape = input_info.max_shape;
+        }
+        if (ov::shape_size(this->min_shape.get_min_shape()) > ov::shape_size(input_info.min_shape.get_min_shape())) {
+            this->min_shape = input_info.min_shape;
+        }
+        return *this;
     }
 };
 

diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/meta_info.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/meta_info.hpp
@@ -13,18 +13,27 @@ namespace subgraph_dumper {
 
 class MetaInfo {
 public:
-    MetaInfo(const std::string& model_path = "", const std::map<std::string, InputInfo>& _input_info = {},
-             size_t total_op_cnt = 1, size_t this_op_cnt = 1, const std::string& extractor = "", size_t model_priority = 1);
+    MetaInfo(const std::string& model_path = "",
+             const std::map<std::string, InputInfo>& _input_info = {},
+             size_t total_op_cnt = 1,
+             size_t this_op_cnt = 1,
+             const std::string& extractor = "",
+             size_t model_priority = 1);
     MetaInfo(std::map<std::string, InputInfo> _in_info,
              std::map<std::string, ModelInfo> _model_info,
              std::unordered_set<std::string> _extractors) :
              model_info(_model_info),
              input_info(_in_info),
              extractors(_extractors) {};
     void serialize(const std::string& serialization_path);
-    void update(const std::string& model_path, const std::map<std::string, InputInfo>& _input_info, size_t _total_op_cnt = 1,
-                size_t _this_op_cnt = 1, const std::string& extractor = "", const std::vector<std::string>& ignored_inputs = {});
+    void update(const std::string& model_path,
+                const std::map<std::string, InputInfo>& _input_info,
+                size_t _total_op_cnt = 1,
+                size_t _this_op_cnt = 1,
+                const std::string& extractor = "",
+                const std::vector<std::string>& ignored_inputs = {});
     std::map<std::string, InputInfo> get_input_info() const;
+    void set_input_info(const std::map<std::string, InputInfo>& new_in_info) { input_info = new_in_info; };
     std::map<std::string, ModelInfo> get_model_info() const;
     std::string get_any_extractor() const { return *extractors.begin(); }
 

diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/model_info.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/model_info.hpp
@@ -15,7 +15,10 @@ struct ModelInfo {
     std::set<std::string> model_paths;
     size_t this_op_cnt, total_op_cnt, model_priority;
 
-    ModelInfo(const std::string& model_path = "", size_t total_ops_in_model = 1, size_t this_ops_in_model = 1, size_t _model_priority = 1) :
+    ModelInfo(const std::string& model_path = "",
+              size_t total_ops_in_model = 1,
+              size_t this_ops_in_model = 1,
+              size_t _model_priority = 1) :
         total_op_cnt(total_ops_in_model), this_op_cnt(this_ops_in_model), model_priority(_model_priority) {
             model_paths = model_path.empty() ? std::set<std::string>() : std::set<std::string>({ model_path }) ;
     }

diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/gflag_config.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/gflag_config.hpp
@@ -7,6 +7,8 @@
 #include <gflags/gflags.h>
 #include <iostream>
 
+#include "common_test_utils/test_constants.hpp"
+
 static const char help_message[] = "Print a usage message.";
 static const char input_folders_message[] = "Required. Comma separated paths to the input folders with IRs";
 static const char local_cache_message[] = "Optional. Comma separated paths to the local cache folders with IRs";
@@ -15,11 +17,13 @@ static const char path_regex_message[] = "Optional. regular expression to be app
                                          "folders recursive discovery";
 static const char extract_body_message[] = "Optional. Allow to extract operation bodies to operation cache.";
 static const char cache_type_message[] = "Optional. Specify caching type: OP, GRAPH. The default value is both";
+static const char device_message[] = "Optional. Specify device to compile model for `fused_names` extractor. Default is `TEMPLATE` ";
 
 DEFINE_bool(h, false, help_message);
 DEFINE_string(input_folders, "", local_cache_message);
 DEFINE_string(local_cache, "", input_folders_message);
 DEFINE_string(output_folder, "output", output_folder_message);
+DEFINE_string(device, ov::test::utils::DEVICE_TEMPLATE, device_message);
 DEFINE_string(path_regex, ".*", output_folder_message);
 DEFINE_bool(extract_body, true, extract_body_message);
 DEFINE_string(cache_type, "", cache_type_message);
@@ -38,6 +42,7 @@ static void showUsage() {
     std::cout << "    --output_folder \"<path>\"             " << output_folder_message << "\n";
     std::cout << "    --path_regex \"<path>\"                " << path_regex_message << "\n";
     std::cout << "    --extract_body \"<value>\"             " << extract_body_message << "\n";
-    std::cout << "    --cache_type \"<value>\"               " << extract_body_message << "\n";
+    std::cout << "    --cache_type \"<value>\"               " << cache_type_message << "\n";
+    std::cout << "    --device \"<value>\"                   " << device_message << "\n";
     std::cout << std::flush;
 }
diff --git a/...sts/functional/plugin/conformance/subgraphs_dumper/include/matchers/single_op/manager.hpp b/...sts/functional/plugin/conformance/subgraphs_dumper/include/matchers/single_op/manager.hpp
@@ -17,7 +17,7 @@ class MatchersManager {
     explicit MatchersManager(const MatchersMap& matchers = {}) : m_matchers(matchers) {}
 
     bool match(const std::shared_ptr<ov::Node> &node,
-               const std::shared_ptr<ov::Node> &ref);
+               const std::shared_ptr<ov::Node> &ref) const;
 
     void set_matchers(const MatchersMap& matchers = {}) { m_matchers = matchers; }
     const MatchersMap& get_matchers() { return m_matchers; }

diff --git a/.../functional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/fused_names.hpp b/.../functional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/fused_names.hpp
@@ -14,15 +14,16 @@ namespace subgraph_dumper {
 
 class FusedNamesExtractor final : public SubgraphExtractor {
 public:
-    FusedNamesExtractor();
+    FusedNamesExtractor(const std::string& device = "");
     ~FusedNamesExtractor();
 
     std::list<ExtractedPattern> extract(const std::shared_ptr<ov::Model> &model,
-                                        bool is_extract_body = true) override;
-    void set_target_device(const std::string& _device) { device = _device; }
+                                        bool is_extract_body = true,
+                                        bool is_copy_constants = true) override;
 
 protected:
     std::unordered_set<std::string> extract_compiled_model_names(const std::shared_ptr<ov::Model>& model);
+    void set_target_device(const std::string& _device);
 
     std::string device;
     std::shared_ptr<ov::Core> core;

diff --git a/...ests/functional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/manager.hpp b/...ests/functional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/manager.hpp
@@ -12,23 +12,32 @@ namespace subgraph_dumper {
 
 class ExtractorsManager {
 public:
+    // { model, subgraph, model_in_info, subgraph_in_info }
+    using ExtractedSubgraphTuple = std::tuple<bool, std::shared_ptr<ov::Model>, std::shared_ptr<ov::Model>, std::map<std::string, InputInfo>, std::map<std::string, InputInfo>>;
     using ExtractorsMap = std::map<std::string, SubgraphExtractor::Ptr>;
+
     explicit ExtractorsManager(const ExtractorsMap& extractors = {}) : m_extractors(extractors) {}
 
     bool match(const std::shared_ptr<ov::Model> &model,
-               const std::shared_ptr<ov::Model> &ref,
+               const std::shared_ptr<ov::Model> &ref_model,
                std::map<std::string, InputInfo> &in_info,
                const std::map<std::string, InputInfo> &in_info_ref);
+    ExtractedSubgraphTuple is_subgraph(const std::shared_ptr<ov::Model> &model,
+                                       const std::shared_ptr<ov::Model> &ref_model,
+                                       const std::map<std::string, InputInfo> &in_info = {},
+                                       const std::map<std::string, InputInfo> &in_info_ref = {});
     std::list<ExtractedPattern> extract(const std::shared_ptr<ov::Model> &model,
-                                        bool is_extract_body = true);
+                                        bool is_extract_body = true,
+                                        bool is_copy_constants = true);
 
     void set_extractors(const ExtractorsMap& extractors = {}) { m_extractors = extractors; }
     ExtractorsMap get_extractors() { return m_extractors; }
 
     std::map<std::string, InputInfo> align_input_info(const std::shared_ptr<ov::Model>& model,
                                                       const std::shared_ptr<ov::Model>& model_ref,
                                                       const std::map<std::string, InputInfo> &in_info,
-                                                      const std::map<std::string, InputInfo> &in_info_ref);
+                                                      const std::map<std::string, InputInfo> &in_info_ref,
+                                                      const std::map<std::string, std::string> &matched_op = {});
 
 protected:
     ExtractorsMap m_extractors = {};

diff --git a/...nctional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/repeat_pattern.hpp b/...nctional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/repeat_pattern.hpp
@@ -25,7 +25,8 @@ class RepeatPatternExtractor final : public SubgraphExtractor {
     }
 
     std::list<ExtractedPattern> extract(const std::shared_ptr<ov::Model> &model,
-                                        bool is_extract_body = true) override;
+                                        bool is_extract_body = true,
+                                        bool is_copy_constants = true) override;
 
 private:
     MatchersManager manager;

diff --git a/...sts/functional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/subgraph.hpp b/...sts/functional/plugin/conformance/subgraphs_dumper/include/matchers/subgraph/subgraph.hpp
@@ -8,21 +8,38 @@
 
 #include "openvino/op/util/op_types.hpp"
 #include "common_test_utils/graph_comparator.hpp"
+
 #include "cache/meta/input_info.hpp"
+#include "matchers/single_op/single_op.hpp"
+#include "matchers/single_op/convolutions.hpp"
+#include "matchers/single_op/manager.hpp"
 
 namespace ov {
 namespace tools {
 namespace subgraph_dumper {
 
 class SubgraphExtractor {
 public:
+    // { is_subgraph, model, subgraph, matched_ops{ model_op_name, graph_op_name }}
+    using IsSubgraphTuple = std::tuple<bool, std::shared_ptr<ov::Model>, std::shared_ptr<ov::Model>, std::map<std::string, std::string>>;
     using Ptr = std::shared_ptr<SubgraphExtractor>;
 
+    SubgraphExtractor() {
+        MatchersManager::MatchersMap matchers = {
+            { "generic_single_op", SingleOpMatcher::Ptr(new SingleOpMatcher) },
+            { "convolutions", ConvolutionsMatcher::Ptr(new ConvolutionsMatcher) },
+        };
+        m_manager.set_matchers(matchers);
+    }
+
     bool match(const std::shared_ptr<ov::Model> &model,
                const std::shared_ptr<ov::Model> &ref_model) const;
+    IsSubgraphTuple is_subgraph(const std::shared_ptr<ov::Model> &model,
+                                const std::shared_ptr<ov::Model> &ref_model) const;
 
     virtual std::list<ExtractedPattern> extract(const std::shared_ptr<ov::Model> &model,
-                                                bool is_extract_body = true) {
+                                                bool is_extract_body = true,
+                                                bool is_copy_constants = true) {
         return std::list<ExtractedPattern>{};
     };
 
@@ -34,6 +51,7 @@ class SubgraphExtractor {
         .enable(FunctionsComparator::ATTRIBUTES)
         .enable(FunctionsComparator::NODES)
         .enable(FunctionsComparator::PRECISIONS);
+    MatchersManager m_manager = MatchersManager();
 
     inline bool is_node_to_skip(const std::shared_ptr<ov::Node>& node) const {
         return ov::op::util::is_parameter(node) ||

diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/memory.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/memory.hpp
@@ -0,0 +1,48 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#if defined(_WIN32)
+#include <Windows.h>
+#else
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#endif
+
+namespace ov {
+namespace tools {
+namespace subgraph_dumper {
+
+static size_t get_ram_size() {
+    size_t ram_mem_size_bytes = 0;
+#ifdef _WIN32
+    MEMORYSTATUSEX status;
+    status.dwLength = sizeof(status);
+    GlobalMemoryStatusEx( &status );
+    ram_mem_size_bytes = status.ullTotalPhys;
+#elif defined(CTL_HW) && defined(HW_MEMSIZE)
+    int mib[2];
+    mib[0] = CTL_HW;
+#if defined(HW_MEMSIZE)
+    mib[1] = HW_MEMSIZE;
+#endif
+    int64_t size = 0;
+    size_t len = sizeof( size );
+    if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 )
+        ram_mem_size_bytes = size;
+#elif defined(_SC_AIX_REALMEM)
+    ram_mem_size_bytes = sysconf( _SC_AIX_REALMEM ) * (size_t)1024L;
+
+#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
+    ram_mem_size_bytes =  static_cast<size_t>(sysconf( _SC_PHYS_PAGES )) *
+        static_cast<size_t>(sysconf(_SC_PAGE_SIZE));
+#endif
+    return ram_mem_size_bytes;
+}
+
+}  // namespace subgraph_dumper
+}  // namespace tools
+}  // namespace ov