[CONFORMANCE] Add relative weights for conformance (#15799)

* Add Weights by ops

* Upgrade conformance tools

* api_conformance

* Change prefix

* Reorg meta info

* Chnage base algo

* fix all other

* return summary

* Update the report

* wa

* review

src/tests/functional/plugin/conformance/subgraphs_dumper/include/ops_cache.hpp

@@ -12,6 +12,8 @@
 #include "matchers/matchers_manager.hpp"
 #include "functional_test_utils/include/functional_test_utils/summary/op_info.hpp"
 
+#include "utils/model_wrap_struct.hpp"
+
 namespace SubgraphsDumper {
 
 class OPCache {
@@ -23,9 +25,9 @@ class OPCache {
         return std::unique_ptr<OPCache>(new OPCache());
     }
 
-    void update_ops_cache(const std::shared_ptr<ov::Node> &op, const std::string &source_model = {});
+    void update_ops_cache(const std::shared_ptr<ov::Node> &op, const Model& source_model);
 
-    void update_ops_cache(const std::shared_ptr<ov::Model> &func, const bool extract_body = true, const std::string &source_model = {});
+    void update_ops_cache(const std::shared_ptr<ov::Model> &func,  const Model& source_model, const bool extract_body = true);
 
     void serialize_cached_ops(const std::string &serialization_dir);
 

src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/model_wrap_struct.hpp

@@ -4,18 +4,25 @@
 
 #pragma once
 
+#include "common_test_utils/file_utils.hpp"
 #include "functional_test_utils/ov_plugin_cache.hpp"
 
 namespace SubgraphsDumper {
 
 struct Model {
     std::string path;
     size_t size = 0;
+    std::string name;
+    size_t op_cnt = 0;
 
     Model(std::string model) {
         path = model;
+        auto pos = model.rfind(CommonTestUtils::FileSeparator);
+        name = pos == std::string::npos ? model : CommonTestUtils::replaceExt(model.substr(pos + 1), "");
         try {
-            size = ov::test::utils::PluginCache::get().core()->read_model(path)->get_graph_size();
+            auto ov_model = ov::test::utils::PluginCache::get().core()->read_model(path);
+            size = ov_model->get_graph_size();
+            op_cnt = ov_model->get_ops().size() - (ov_model->inputs().size() + ov_model->outputs().size());
         } catch (...) {
             std::cout << "Impossible to read network: " << path << std::endl;
         }

src/tests/functional/plugin/conformance/subgraphs_dumper/src/main.cpp

@@ -112,7 +112,7 @@ void cacheModels(std::unique_ptr<SubgraphsDumper::OPCache> &cache,
                     ret_code = 1;
                     continue;
                 }
-                cache->update_ops_cache(function, extract_body, model.path);
+                cache->update_ops_cache(function, model, extract_body);
                 successful_models_file << model.path << std::endl;
             } catch (std::exception &e) {
                 not_fully_cached_models_file << model.path << std::endl;

src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp

@@ -14,11 +14,12 @@
 using namespace SubgraphsDumper;
 
 void OPCache::update_ops_cache(const std::shared_ptr<ov::Node> &op,
-                               const std::string &source_model) {
+                               const Model& source_model) {
     const std::shared_ptr<ov::Node> cachedOp = [&] {
         for (auto &&it : m_ops_cache) {
             if (manager.match_any(it.first, op, it.second)) {
-                it.second.found_in_models[source_model] += 1;
+                it.second.found_in_models[source_model.name].unique_op_cnt += 1;
+                it.second.found_in_models[source_model.name].model_paths.insert({{source_model.path, source_model.op_cnt}});
                 return it.first;
             }
         }
@@ -28,7 +29,7 @@ void OPCache::update_ops_cache(const std::shared_ptr<ov::Node> &op,
     auto saveOpToCash = [&] {
         try {
             const auto& clone_fn = SubgraphsDumper::ClonersMap::cloners.at(op->get_type_info());
-            LayerTestsUtils::OPInfo meta(source_model);
+            LayerTestsUtils::OPInfo meta(source_model.name, source_model.path, source_model.op_cnt);
             const std::shared_ptr<ov::Node> op_clone = clone_fn(op, meta);
             if (!op_clone) {
                 return;
@@ -62,7 +63,7 @@ void OPCache::update_ops_cache(const std::shared_ptr<ov::Node> &op,
     }
 }
 
-void OPCache::update_ops_cache(const std::shared_ptr<ov::Model> &func, const bool extract_body, const std::string &source_model) {
+void OPCache::update_ops_cache(const std::shared_ptr<ov::Model> &func, const Model& source_model, const bool extract_body) {
     size_t cached_ops_count = m_ops_cache.size();
     for (const auto &op : func->get_ordered_ops()) {
         if (std::dynamic_pointer_cast<ov::op::v0::Parameter>(op) ||
@@ -81,16 +82,16 @@ void OPCache::update_ops_cache(const std::shared_ptr<ov::Model> &func, const boo
                 std::vector<std::shared_ptr<ov::Model>> bodies;
                 for (size_t i = 0; i < if_op->get_internal_subgraphs_size(); i++) {
                     auto if_body = if_op->get_function(i);
-                    update_ops_cache(if_body, extract_body, source_model);
+                    update_ops_cache(if_body, source_model, extract_body);
                 }
             } else if (std::dynamic_pointer_cast<ov::op::v5::Loop>(op)) {
                 auto loop = std::dynamic_pointer_cast<ov::op::v5::Loop>(op);
                 auto loop_body = loop->get_function();
-                update_ops_cache(loop_body, extract_body, source_model);
+                update_ops_cache(loop_body, source_model, extract_body);
             } else if (std::dynamic_pointer_cast<ov::op::v0::TensorIterator>(op)) {
                 auto ti = std::dynamic_pointer_cast<ov::op::v0::TensorIterator>(op);
                 auto ti_body = ti->get_body();
-                update_ops_cache(ti_body, extract_body, source_model);
+                update_ops_cache(ti_body, source_model, extract_body);
             }
         }
         update_ops_cache(op, source_model);
@@ -122,12 +123,25 @@ void OPCache::serialize_meta_info(const LayerTestsUtils::OPInfo &info, const std
     pugi::xml_document doc;
     pugi::xml_node root = doc.append_child("meta_info");
     pugi::xml_node models = root.append_child("models");
-    models.append_child("initial_model").append_attribute("name").set_value(info.source_model.c_str());
+    double k = 0;
     for (const auto &model : info.found_in_models) {
         pugi::xml_node model_node = models.append_child("model");
         model_node.append_attribute("name").set_value(model.first.c_str());
-        model_node.append_attribute("count").set_value(static_cast<unsigned long long>(model.second));
+        double model_k = model.second.unique_op_cnt;
+        model_node.append_attribute("count").set_value(static_cast<unsigned long long>(model.second.unique_op_cnt));
+        size_t tmp = 0;
+        for (const auto& model_path : model.second.model_paths) {
+            if (model_path.second) {
+                model_node.append_child("path").append_attribute("model").set_value(model_path.first.c_str());
+                tmp += model_path.second;
+            }
+        }
+        model_k /= tmp;
+        model_k /= model.second.model_paths.size();
+        k += model_k;
     }
+    k *=  info.found_in_models.size();
+    root.append_child("graph_priority").append_attribute("value").set_value(k);
     auto ports_info = root.append_child("ports_info");
     for (const auto &port : info.ports_info) {
         auto port_node = ports_info.append_child("port");
@@ -164,7 +178,7 @@ OPCache::serialize_function(const std::pair<std::shared_ptr<ov::Node>, LayerTest
                             const std::string &serialization_dir) {
     try {
         std::cout << "Serializing function wrapping op " << op.first << std::endl;
-        std::cout << "Taken from model: " << op.second.source_model << std::endl;
+        std::cout << "Taken from model: " << op.second.found_in_models.begin()->first << std::endl;
 
         ov::ParameterVector params;
         bool is_dynamic = false;

src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/read_ir_test/read_ir.hpp

@@ -34,7 +34,7 @@ class ReadIRTest : public testing::WithParamInterface<ReadIRParams>,
     void SetUp() override;
 
 private:
-    std::string path_to_model, path_to_cache, source_model;
+    std::string path_to_model, path_to_cache;
     std::vector<std::pair<std::string, size_t>> ocurance_in_models;
 };
 } // namespace subgraph

src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp

@@ -145,9 +145,16 @@ void ReadIRTest::SetUp() {
         pugi::xml_document doc;
         doc.load_file(metaFile.c_str());
         auto models = doc.child("meta_info").child("models");
-        source_model = models.child("initial_model").attribute("name").as_string();
+        size_t model_len = 0, occurance = 0;
         for (const auto &model : models.children("model")) {
             ocurance_in_models.push_back({model.attribute("name").as_string(), model.attribute("count").as_uint()});
+            model_len++;
+            occurance += model.attribute("count").as_uint();
+        }
+        rel_influence_coef = doc.child("meta_info").child("graph_priority").attribute("value").as_double();
+        // TODO: remove after cache update w/a
+        if (rel_influence_coef == 0) {
+            rel_influence_coef = 1.f;
         }
         auto portsInfo = doc.child("meta_info").child("ports_info");
         auto getPortInfo = [&](size_t id) {
@@ -264,6 +271,7 @@ void ReadIRTest::SetUp() {
     if (inputShapes.empty()) {
         GTEST_SKIP() << "The graph is constant. The case is not applicable for Operation conformance scenario";
     }
+    std::cout << "[ CONFORMANCE ] Influence coefficient: " << rel_influence_coef << std::endl;
     init_input_shapes(inputShapes);
     is_report_stages = true;
 }

src/tests/functional/plugin/shared/include/base/ov_behavior_test_utils.hpp

@@ -50,6 +50,7 @@ class APIBaseTest : public CommonTestUtils::TestsCommon {
     const std::unique_ptr<CommonTestUtils::CrashHandler> crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
 
 protected:
+    size_t k = 1;
     std::string target_device = "";
     ov::test::utils::ov_entity api_entity = ov::test::utils::ov_entity::undefined;
     ov::test::utils::ApiSummary& api_summary = ov::test::utils::ApiSummary::getInstance();
@@ -61,7 +62,10 @@ class APIBaseTest : public CommonTestUtils::TestsCommon {
 
     void SetUp() override {
         set_api_entity();
-        api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::CRASHED);
+        auto test_name = this->GetTestName();
+        k = test_name.find("_mandatory") != std::string::npos || test_name.find("mandatory_") != std::string::npos ? 1 : 0;
+        std::cout << "[ CONFORMANCE ] Influence coefficient: " << k << std::endl;
+        api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::CRASHED, k);
 #ifdef _WIN32
         jmpRes = setjmp(CommonTestUtils::env);
 #else
@@ -70,7 +74,7 @@ class APIBaseTest : public CommonTestUtils::TestsCommon {
         if (jmpRes == CommonTestUtils::JMP_STATUS::ok) {
             crashHandler->StartTimer();
         } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) {
-            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::HANGED);
+            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::HANGED, k);
             GTEST_FAIL();
         }
     }
@@ -80,11 +84,11 @@ class APIBaseTest : public CommonTestUtils::TestsCommon {
             set_api_entity();
         }
         if (this->HasFailure()) {
-            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::FAILED);
+            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::FAILED, k);
         } else if (this->IsSkipped()) {
-            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::SKIPPED);
+            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::SKIPPED, k);
         } else {
-            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::PASSED);
+            api_summary.updateStat(api_entity, target_device, ov::test::utils::PassRate::Statuses::PASSED, k);
         }
     }
 };

src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp

@@ -57,6 +57,7 @@ class SubgraphBaseTest : public CommonTestUtils::TestsCommon {
 
     ov::test::utils::OpSummary& summary = ov::test::utils::OpSummary::getInstance();
     bool is_report_stages = false;
+    double rel_influence_coef = 1.f;
 
     virtual std::vector<ov::Tensor> calculate_refs();
     virtual std::vector<ov::Tensor> get_plugin_outputs();

src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp

@@ -47,7 +47,7 @@ void SubgraphBaseTest::run() {
          ov::test::utils::PassRate::Statuses::SKIPPED :
          ov::test::utils::PassRate::Statuses::CRASHED;
     summary.setDeviceName(targetDevice);
-    summary.updateOPsStats(function, status);
+    summary.updateOPsStats(function, status, rel_influence_coef);
 
     if (isCurrentTestDisabled)
         GTEST_SKIP() << "Disabled test due to configuration" << std::endl;
@@ -92,14 +92,14 @@ void SubgraphBaseTest::run() {
             status = ov::test::utils::PassRate::Statuses::FAILED;
             errorMessage = "Unknown failure occurred.";
         }
-        summary.updateOPsStats(function, status);
+        summary.updateOPsStats(function, status, rel_influence_coef);
         if (status != ov::test::utils::PassRate::Statuses::PASSED) {
             GTEST_FATAL_FAILURE_(errorMessage.c_str());
         }
     } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) {
         IE_THROW() << "Crash happens";
     } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) {
-        summary.updateOPsStats(function, ov::test::utils::PassRate::Statuses::HANGED);
+        summary.updateOPsStats(function, ov::test::utils::PassRate::Statuses::HANGED, rel_influence_coef);
         IE_THROW() << "Crash happens";
     }
 }

src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/summary/api_summary.hpp

@@ -48,7 +48,7 @@ class ApiSummary : public virtual Summary {
     static ApiSummary &getInstance();
     inline void getStatisticFromReport(const std::string& filePath);
     std::map<ov_entity, std::map<std::string, PassRate>> getApiStats() { return apiStats; }
-    void updateStat(ov_entity, const std::string& device, PassRate::Statuses);
+    void updateStat(ov_entity, const std::string& device, PassRate::Statuses, double rel_influence_coef = 1);
     void saveReport() override;
 };
 

src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/summary/op_info.hpp

@@ -5,6 +5,18 @@
 #pragma once
 
 namespace LayerTestsUtils {
+
+struct ModelInfo {
+    size_t unique_op_cnt;
+    // model_path, op_cnt
+    std::map<std::string, size_t> model_paths;
+
+
+    ModelInfo(size_t _op_cnt = 0, const std::map<std::string, size_t>& _model_paths = {{}})
+        : unique_op_cnt(_op_cnt),
+          model_paths(_model_paths) {}
+};
+
 struct PortInfo {
     double min;
     double max;
@@ -20,12 +32,11 @@ struct PortInfo {
 };
 
 struct OPInfo {
-    std::string source_model;
-    std::map<std::string, size_t> found_in_models;
+    std::map<std::string, ModelInfo> found_in_models;
     std::map<size_t, PortInfo> ports_info;
 
-    OPInfo(const std::string &source_model) : source_model(source_model) {
-        found_in_models = {{source_model, 1}};
+    OPInfo(const std::string& source_model, const std::string& model_path, size_t total_op_cnt = 0) {
+        found_in_models = {{source_model, ModelInfo(1, {{model_path, total_op_cnt}})}};
         ports_info = {};
     }
 

src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp

@@ -49,10 +49,10 @@ class OpSummary : public virtual Summary {
     std::map<std::string, PassRate> getStatisticFromReport();
     void saveReport() override;
 
-    void updateOPsStats(const std::shared_ptr<ov::Model> &model, const PassRate::Statuses &status);
+    void updateOPsStats(const std::shared_ptr<ov::Model> &model, const PassRate::Statuses &status, double rel_influence_coef = 1);
     void updateOPsImplStatus(const std::shared_ptr<ov::Model> &model, const bool implStatus);
 
-    void updateOPsStats(const ov::NodeTypeInfo &op, const PassRate::Statuses &status);
+    void updateOPsStats(const ov::NodeTypeInfo &op, const PassRate::Statuses &status, double rel_influence_coef = 1);
     void updateOPsImplStatus(const ov::NodeTypeInfo &op, const bool implStatus);
 };
 

src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp

@@ -31,14 +31,19 @@ struct PassRate {
     unsigned long hanged = 0;
     bool isImplemented = false;
 
+    double rel_passed = 0;
+    double rel_all = 0;
+
     PassRate() = default;
 
-    PassRate(unsigned long p, unsigned long f, unsigned long s, unsigned long c, unsigned long h) {
+    PassRate(unsigned long p, unsigned long f, unsigned long s, unsigned long c, unsigned long h, double rel_p = 0, double rel_a = 0) {
         passed = p;
         failed = f;
         skipped = s;
         crashed = c;
         hanged = h;
+        rel_passed = rel_p;
+        rel_all = rel_a;
         if (!isImplemented && passed > 0) {
             isImplemented = true;
         }
@@ -55,6 +60,14 @@ struct PassRate {
             return passed * 100.f / (passed + failed + skipped + crashed + hanged);
         }
     }
+
+    double getRelPassrate() const {
+        if (rel_all == 0) {
+            return 100.f;
+        } else {
+            return rel_passed * 100.f / rel_all;
+        }
+    }
 };
 
 class Summary {

src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py

@@ -37,16 +37,16 @@ def parse_arguments():
 
 def update_result_node(xml_node: SubElement, aggregated_res: SubElement):
     for attr_name in xml_node.attrib:
-        if attr_name == "passrate":
+        if attr_name == "passrate" or attr_name == "relative_passrate":
             continue
         if attr_name == "implemented":
             xml_value = xml_node.attrib.get(attr_name) == "true"
             aggregated_value = aggregated_res.attrib.get(attr_name) == "true"
             str_value = "true" if xml_value or aggregated_value else "false"
             aggregated_res.set(attr_name, str_value)
             continue
-        xml_value = int(xml_node.attrib.get(attr_name))
-        aggregated_value = int(aggregated_res.attrib.get(attr_name))
+        xml_value = float(xml_node.attrib.get(attr_name)) if "relative_" in attr_name else int(xml_node.attrib.get(attr_name))
+        aggregated_value = float(aggregated_res.attrib.get(attr_name)) if "relative_" in attr_name else int(aggregated_res.attrib.get(attr_name))
         # if attr_name == "crashed" and xml_value > 0:
             # print("f")
         aggregated_res.set(attr_name, str(xml_value + aggregated_value))