From 061d6b5b6228c794b33acb41c26b891632fc75a2 Mon Sep 17 00:00:00 2001 From: Sofya Balandina Date: Mon, 30 Oct 2023 13:45:46 +0000 Subject: [PATCH] [apiConformance] Move sw plugin to run with Template only (#20084) * [apiConformance] Move sw plugin to run over TEMPLATE * Fix set_property * update * add configs for sw plugins --- .../include/api_conformance_helpers.hpp | 111 ------------------ .../include/ov_api_conformance_helpers.hpp | 53 ++++++--- .../ov_executable_network/exec_graph_info.cpp | 4 +- .../exec_network_base.cpp | 4 +- .../ov_executable_network/get_metric.cpp | 8 +- .../ov_executable_network/properties.cpp | 34 ++---- .../behavior/ov_infer_request/callback.cpp | 2 +- .../ov_infer_request/cancellation.cpp | 2 +- .../infer_request_dynamic.cpp | 4 +- .../ov_infer_request/inference_chaining.cpp | 4 +- .../behavior/ov_infer_request/io_tensor.cpp | 10 +- .../ov_infer_request/multithreading.cpp | 2 +- .../ov_infer_request/perf_counters.cpp | 2 +- .../src/behavior/ov_infer_request/wait.cpp | 2 +- .../src/behavior/ov_plugin/caching_tests.cpp | 12 +- .../behavior/ov_plugin/core_integration.cpp | 6 +- .../src/behavior/ov_plugin/life_time.cpp | 4 +- .../src/behavior/ov_plugin/properties.cpp | 50 +++----- .../config/batch_config.txt | 1 + .../config/hetero_config.txt | 1 + .../config/multi_config.txt | 1 + 21 files changed, 93 insertions(+), 224 deletions(-) delete mode 100644 src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/api_conformance_helpers.hpp create mode 100644 src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/batch_config.txt create mode 100644 src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/hetero_config.txt create mode 100644 src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/multi_config.txt diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/api_conformance_helpers.hpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/api_conformance_helpers.hpp deleted file mode 100644 index bdde3438192bd6..00000000000000 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/api_conformance_helpers.hpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "conformance.hpp" -#include "common_test_utils/test_constants.hpp" - -namespace ov { -namespace test { -namespace conformance { - -inline const std::string get_plugin_lib_name_by_device(const std::string& deviceName) { - const std::map devices{ - { "AUTO", "openvino_auto_plugin" }, - { "HETERO", "openvino_hetero_plugin" }, - { "BATCH", "openvino_auto_batch_plugin" }, - { "MULTI", "openvino_auto_plugin" }, - { "NPU", "openvino_intel_npu_plugin" }, - { "CPU", "openvino_intel_cpu_plugin" }, - { "GNA", "openvino_intel_gna_plugin" }, - { "GPU", "openvino_intel_gpu_plugin" }, - { "TEMPLATE", "openvino_template_plugin" }, - { "NVIDIA", "openvino_nvidia_gpu_plugin" }, - }; - if (devices.find(deviceName) == devices.end()) { - if (std::string(targetPluginName) != "") { - return targetPluginName; - } - throw std::runtime_error("Incorrect device name"); - } - return devices.at(deviceName); -} - -inline const std::pair generate_default_multi_config() { - return {MULTI_CONFIG_KEY(DEVICE_PRIORITIES), ov::test::conformance::targetDevice}; -} - -inline const std::pair generate_default_hetero_config() { - return { "TARGET_FALLBACK" , ov::test::conformance::targetDevice }; -} - -inline const std::pair generate_default_batch_config() { - // auto-batching with batch 1 (no real batching in fact, but full machinery is in action) - return { CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , ov::test::conformance::targetDevice }; -} - -inline const std::vector> generate_configs(const std::string& target_plugin, - const std::vector>& config = {}) { - std::pair default_config; - if (target_plugin == std::string(ov::test::utils::DEVICE_MULTI) || target_plugin == std::string(ov::test::utils::DEVICE_AUTO)) { - default_config = generate_default_multi_config(); - } else if (target_plugin == std::string(ov::test::utils::DEVICE_HETERO)) { - default_config = generate_default_hetero_config(); - } else if (target_plugin == std::string(ov::test::utils::DEVICE_BATCH)) { - default_config = generate_default_batch_config(); - } else { - throw std::runtime_error("Incorrect target device: " + target_plugin); - } - - std::vector> resultConfig; - if (config.empty()) { - return {{default_config}}; - } - for (auto configItem : config) { - configItem.insert(default_config); - resultConfig.push_back(configItem); - } - return resultConfig; -} - -inline const std::string generate_complex_device_name(const std::string& deviceName) { - return deviceName + ":" + ov::test::conformance::targetDevice; -} - -inline const std::vector return_all_possible_device_combination(bool enable_complex_name = true) { - std::vector res{ov::test::conformance::targetDevice}; - std::vector devices{ov::test::utils::DEVICE_HETERO, ov::test::utils::DEVICE_AUTO, - ov::test::utils::DEVICE_BATCH, ov::test::utils::DEVICE_MULTI}; - for (const auto& device : devices) { - res.emplace_back(enable_complex_name ? generate_complex_device_name(device) : device); - } - return res; -} - -inline std::vector> generate_pairs_plugin_name_by_device() { - std::vector> res; - for (const auto& device : return_all_possible_device_combination()) { - std::string real_device = device.substr(0, device.find(':')); - res.push_back(std::make_pair(get_plugin_lib_name_by_device(real_device), - real_device)); - } - return res; -} - -inline std::map AnyMap2StringMap(const AnyMap& config) { - if (config.empty()) - return {}; - std::map result; - for (const auto& configItem : config) { - result.insert({configItem.first, configItem.second.as()}); - } - return result; -} - -const std::map ie_config = AnyMap2StringMap(ov::test::conformance::pluginConfig); - -} // namespace conformance -} // namespace test -} // namespace ov diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/ov_api_conformance_helpers.hpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/ov_api_conformance_helpers.hpp index 1f6278be78116c..6dd7a82928cff6 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/ov_api_conformance_helpers.hpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/ov_api_conformance_helpers.hpp @@ -4,36 +4,53 @@ #pragma once -#include "api_conformance_helpers.hpp" +#include "conformance.hpp" +#include "common_test_utils/test_constants.hpp" namespace ov { namespace test { namespace conformance { -inline const std::vector generate_ov_configs(const std::string& target_plugin, - const std::vector& config = {}) { - std::pair default_config; - if (target_plugin == std::string(ov::test::utils::DEVICE_MULTI) || - target_plugin == std::string(ov::test::utils::DEVICE_AUTO) || - target_plugin == std::string(ov::test::utils::DEVICE_HETERO)) { - default_config = ov::device::priorities(ov::test::conformance::targetDevice); - } else if (target_plugin == std::string(ov::test::utils::DEVICE_BATCH)) { - default_config = { CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(ov::test::conformance::targetDevice)}; - } else { - throw std::runtime_error("Incorrect target device: " + target_plugin); - } - +inline const std::vector generate_ov_configs(const std::vector& config = {}) { std::vector resultConfig; - if (config.empty()) { - return {{default_config}}; - } for (auto configItem : config) { - configItem.insert(default_config); + configItem.insert(ov::test::conformance::pluginConfig.begin(), ov::test::conformance::pluginConfig.end()); resultConfig.push_back(configItem); } return resultConfig; } +inline const std::string get_plugin_lib_name_by_device(const std::string& deviceName) { + const std::map devices{ + { "AUTO", "openvino_auto_plugin" }, + { "HETERO", "openvino_hetero_plugin" }, + { "BATCH", "openvino_auto_batch_plugin" }, + { "MULTI", "openvino_auto_plugin" }, + { "NPU", "openvino_intel_npu_plugin" }, + { "CPU", "openvino_intel_cpu_plugin" }, + { "GNA", "openvino_intel_gna_plugin" }, + { "GPU", "openvino_intel_gpu_plugin" }, + { "TEMPLATE", "openvino_template_plugin" }, + { "NVIDIA", "openvino_nvidia_gpu_plugin" }, + }; + if (devices.find(deviceName) == devices.end()) { + if (std::string(targetPluginName) != "") { + return targetPluginName; + } + throw std::runtime_error("Incorrect device name"); + } + return devices.at(deviceName); +} + +inline std::vector> generate_ov_pairs_plugin_name_by_device() { + std::vector> res; + std::string device(ov::test::conformance::targetDevice); + std::string real_device = device.substr(0, device.find(':')); + res.push_back(std::make_pair(get_plugin_lib_name_by_device(real_device), + real_device)); + return res; +} + } // namespace conformance } // namespace test } // namespace ov diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_graph_info.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_graph_info.cpp index 71f0bd1175b6aa..3fc385eb81701c 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_graph_info.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_graph_info.cpp @@ -32,12 +32,12 @@ INSTANTIATE_TEST_SUITE_P(ov_compiled_model, OVCompiledGraphImportExportTest, ::testing::Combine( ::testing::ValuesIn(ovExecGraphInfoElemTypes), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVCompiledGraphImportExportTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P( ov_compiled_model, OVClassCompiledModelImportExportTestP, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_network_base.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_network_base.cpp index fcf1952c55af4c..b7abf26114f916 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_network_base.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/exec_network_base.cpp @@ -13,13 +13,13 @@ using namespace ov::test::conformance; INSTANTIATE_TEST_SUITE_P(ov_compiled_model_mandatory, OVCompiledModelBaseTest, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVCompiledModelBaseTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_compiled_model, OVCompiledModelBaseTestOptional, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVCompiledModelBaseTestOptional::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/get_metric.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/get_metric.cpp index 3448f8d355161f..aac7e09a380cf4 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/get_metric.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/get_metric.cpp @@ -16,7 +16,7 @@ using namespace ov::test::conformance; INSTANTIATE_TEST_SUITE_P( ov_compiled_model_mandatory, OVClassCompiledModelGetPropertyTest, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); // // Executable Network GetConfig / SetConfig @@ -24,15 +24,15 @@ INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P( ov_compiled_model_mandatory, OVClassCompiledModelGetIncorrectPropertyTest, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); INSTANTIATE_TEST_SUITE_P( ov_compiled_model_mandatory, OVClassCompiledModelGetConfigTest, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); INSTANTIATE_TEST_SUITE_P( ov_compiled_model, OVClassCompiledModelSetIncorrectConfigTest, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/properties.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/properties.cpp index bb361ab03088d8..5639a0360eaec7 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/properties.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_executable_network/properties.cpp @@ -15,55 +15,35 @@ const std::vector inproperties = { {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, }; -const std::vector auto_batch_inproperties = {}; - INSTANTIATE_TEST_SUITE_P(ov_compiled_model_mandatory, OVClassCompiledModelPropertiesIncorrectTests, ::testing::Combine( - ::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination()), - ::testing::ValuesIn(inproperties)), + ::testing::Values(targetDevice), + ::testing::ValuesIn(generate_ov_configs(inproperties))), OVClassCompiledModelPropertiesIncorrectTests::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(ov_compiled_model_AutoBatch, OVClassCompiledModelPropertiesIncorrectTests, - ::testing::Combine( - ::testing::Values(ov::test::utils::DEVICE_BATCH), - ::testing::ValuesIn(generate_ov_configs(ov::test::utils::DEVICE_BATCH, auto_batch_inproperties))), - OVClassCompiledModelPropertiesIncorrectTests::getTestCaseName); - - const std::vector default_properties = { {ov::enable_profiling(false)} }; INSTANTIATE_TEST_SUITE_P(ov_compiled_model_mandatory, OVClassCompiledModelPropertiesDefaultTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(default_properties)), OVClassCompiledModelPropertiesDefaultTests::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_compiled_model_mandatory, OVCompiledModelPropertiesDefaultSupportedTests, - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), OVCompiledModelPropertiesDefaultSupportedTests::getTestCaseName); -const std::vector auto_batch_properties = { - {{CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, - {{ov::auto_batch_timeout(10)}}, -}; - INSTANTIATE_TEST_SUITE_P(ov_compiled_model_mandatory, OVClassCompiledModelPropertiesTests, ::testing::Combine( - ::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(default_properties)), OVClassCompiledModelPropertiesTests::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(ov_compiled_model_AutoBatch, OVClassCompiledModelPropertiesTests, - ::testing::Combine( - ::testing::Values(ov::test::utils::DEVICE_BATCH), - ::testing::ValuesIn(ov::test::conformance::generate_ov_configs(ov::test::utils::DEVICE_BATCH, auto_batch_properties))), - OVClassCompiledModelPropertiesTests::getTestCaseName); - INSTANTIATE_TEST_SUITE_P( ov_compiled_model, OVClassCompiledModelEmptyPropertiesTests, - ::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination())); + ::testing::Values(targetDevice)); // IE Class Load network @@ -78,7 +58,7 @@ const std::vector multiModelPriorityConfigs = { INSTANTIATE_TEST_SUITE_P(ov_compiled_model_mandatory, OVClassCompiledModelGetPropertyTest_MODEL_PRIORITY, - ::testing::Combine(::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Combine(::testing::Values(targetDevice), ::testing::ValuesIn(multiModelPriorityConfigs))); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/callback.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/callback.cpp index a32f200024b7e8..02631d2f897ecb 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/callback.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/callback.cpp @@ -14,7 +14,7 @@ using namespace ov::test::conformance; INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestCallbackTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestCallbackTests::getTestCaseName); diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/cancellation.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/cancellation.cpp index 76d64baef6f6f2..5ffdbc0f354fa4 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/cancellation.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/cancellation.cpp @@ -11,7 +11,7 @@ using namespace ov::test::conformance; INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestCancellationTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestCancellationTests::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/infer_request_dynamic.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/infer_request_dynamic.cpp index 0ebf894b6e516e..b8749687c3458e 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/infer_request_dynamic.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/infer_request_dynamic.cpp @@ -60,7 +60,7 @@ INSTANTIATE_TEST_SUITE_P(ov_infer_request_1, OVInferRequestDynamicTests, ::testing::Values(std::vector, std::vector>>{ {{1, 4, 20, 20}, {1, 4, 20, 20}}, {{2, 4, 20, 20}, {2, 4, 20, 20}}}), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestDynamicTests::getTestCaseName); @@ -70,7 +70,7 @@ INSTANTIATE_TEST_SUITE_P(ov_infer_request_2, OVInferRequestDynamicTests, ::testing::Values(std::vector, std::vector>>{ {{1, 4, 20, 20}, {1, 2, 20, 40}}, {{2, 4, 20, 20}, {2, 2, 20, 40}}}), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestDynamicTests::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/inference_chaining.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/inference_chaining.cpp index f45c529015ecd0..d485e404d3195e 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/inference_chaining.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/inference_chaining.cpp @@ -12,13 +12,13 @@ using namespace ov::test::conformance; INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferenceChainingStatic, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferenceChaining::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_infer_request, OVInferenceChaining, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferenceChaining::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/io_tensor.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/io_tensor.cpp index 5d741f5f8148e7..779ae92c921cac 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/io_tensor.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/io_tensor.cpp @@ -14,7 +14,7 @@ using namespace ov::test::conformance; namespace { INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestIOTensorTest, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestIOTensorTest::getTestCaseName); @@ -36,14 +36,14 @@ std::vector ovIOTensorElemTypes = { INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestIOTensorSetPrecisionTest, ::testing::Combine( ::testing::ValuesIn(ovIOTensorElemTypes), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestIOTensorSetPrecisionTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestCheckTensorPrecision, ::testing::Combine( ::testing::ValuesIn(ovIOTensorElemTypes), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestCheckTensorPrecision::getTestCaseName); @@ -54,14 +54,14 @@ std::vector ovIOTensorElemTypesOptional = { INSTANTIATE_TEST_SUITE_P(ov_infer_request, OVInferRequestIOTensorSetPrecisionTest, ::testing::Combine( ::testing::ValuesIn(ovIOTensorElemTypesOptional), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestIOTensorSetPrecisionTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_infer_request, OVInferRequestCheckTensorPrecision, ::testing::Combine( ::testing::ValuesIn(ovIOTensorElemTypesOptional), - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestCheckTensorPrecision::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/multithreading.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/multithreading.cpp index 5b319775c7a72d..3c9a2a8e50e58b 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/multithreading.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/multithreading.cpp @@ -15,7 +15,7 @@ namespace { INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestMultithreadingTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestMultithreadingTests::getTestCaseName); diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/perf_counters.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/perf_counters.cpp index b31428ff9edb41..fae5dad32cfbe3 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/perf_counters.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/perf_counters.cpp @@ -12,7 +12,7 @@ namespace { INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestPerfCountersTest, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestPerfCountersTest::getTestCaseName); diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/wait.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/wait.cpp index 6b996615578ff5..538a45b5d721e4 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/wait.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_infer_request/wait.cpp @@ -15,7 +15,7 @@ namespace { INSTANTIATE_TEST_SUITE_P(ov_infer_request_mandatory, OVInferRequestWaitTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(pluginConfig)), OVInferRequestWaitTests::getTestCaseName); diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/caching_tests.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/caching_tests.cpp index 38cb32b4887993..2a0d49b97de6df 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/caching_tests.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/caching_tests.cpp @@ -42,8 +42,8 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin, CompileModelCacheTestBase, ::testing::ValuesIn(CompileModelCacheTestBase::getAnyTypeOnlyFunctions()), ::testing::ValuesIn(ovElemTypesTemplate), ::testing::ValuesIn(ovBatchSizesTemplate), - ::testing::ValuesIn(return_all_possible_device_combination()), - ::testing::Values(ov::AnyMap{})), + ::testing::Values(targetDevice), + ::testing::Values(pluginConfig)), CompileModelCacheTestBase::getTestCaseName); // Convolution/UnaryElementwiseArithmetic/BinaryElementwiseArithmetic is not supported boolean elemnt type @@ -52,8 +52,8 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin_numeric, CompileModelCacheTestBase, ::testing::ValuesIn(CompileModelCacheTestBase::getNumericTypeOnlyFunctions()), ::testing::ValuesIn(ovElemAnyNumericTypesTemplate), ::testing::ValuesIn(ovBatchSizesTemplate), - ::testing::ValuesIn(return_all_possible_device_combination()), - ::testing::Values(ov::AnyMap{})), + ::testing::Values(targetDevice), + ::testing::Values(pluginConfig)), CompileModelCacheTestBase::getTestCaseName); // LSTMcell supported floating-point element type @@ -62,8 +62,8 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin_floating_point, CompileModelCacheTestBase, ::testing::ValuesIn(CompileModelCacheTestBase::getFloatingPointOnlyFunctions()), ::testing::ValuesIn(ovElemAnyFloatingPointTypesTemplate), ::testing::ValuesIn(ovBatchSizesTemplate), - ::testing::ValuesIn(return_all_possible_device_combination()), - ::testing::Values(ov::AnyMap{})), + ::testing::Values(targetDevice), + ::testing::Values(pluginConfig)), CompileModelCacheTestBase::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/core_integration.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/core_integration.cpp index 0c0cee4632233a..ca47910e9e3d95 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/core_integration.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/core_integration.cpp @@ -18,15 +18,15 @@ namespace { INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVClassModelTestP, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); INSTANTIATE_TEST_SUITE_P(ov_plugin, OVClassModelOptionalTestP, - ::testing::ValuesIn(return_all_possible_device_combination())); + ::testing::Values(targetDevice)); // IE Class Query network INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVClassQueryModelTest, - ::testing::ValuesIn(return_all_possible_device_combination(false))); + ::testing::Values(targetDevice)); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/life_time.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/life_time.cpp index 49cd957bb82e65..85b7ab4bdfa945 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/life_time.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/life_time.cpp @@ -11,10 +11,10 @@ using namespace ov::test::conformance; namespace { INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVHoldersTest, - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), OVHoldersTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVHoldersTestOnImportedNetwork, - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), OVHoldersTestOnImportedNetwork::getTestCaseName); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/properties.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/properties.cpp index e2ca8e79edaf10..517f8363f0327f 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/properties.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/ov_plugin/properties.cpp @@ -16,18 +16,10 @@ const std::vector inproperties = { {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, }; -const std::vector auto_batch_inproperties = {}; - INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVPropertiesIncorrectTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), - ::testing::ValuesIn(inproperties)), - OVPropertiesIncorrectTests::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(ov_plugin_AutoBatch, OVPropertiesIncorrectTests, - ::testing::Combine( - ::testing::Values(ov::test::utils::DEVICE_BATCH), - ::testing::ValuesIn(auto_batch_inproperties)), + ::testing::Values(targetDevice), + ::testing::ValuesIn(generate_ov_configs(inproperties))), OVPropertiesIncorrectTests::getTestCaseName); const std::vector default_properties = { @@ -35,63 +27,51 @@ const std::vector default_properties = { {ov::enable_profiling(true)}, }; -const std::vector auto_batch_properties = { - {}, - {{CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, - {{ov::auto_batch_timeout(10)}}, -}; - INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVPropertiesTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination(false)), + ::testing::Values(targetDevice), ::testing::ValuesIn(default_properties)), OVPropertiesTests::getTestCaseName); -INSTANTIATE_TEST_SUITE_P(ov_plugin_AutoBatch, OVPropertiesTests, - ::testing::Combine( - ::testing::Values(ov::test::utils::DEVICE_BATCH), - ::testing::ValuesIn(ov::test::conformance::generate_ov_configs(ov::test::utils::DEVICE_BATCH, auto_batch_properties))), - OVPropertiesTests::getTestCaseName); - INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVCheckGetSupportedROMetricsPropsTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(OVCheckGetSupportedROMetricsPropsTests::getROMandatoryProperties())), OVCheckGetSupportedROMetricsPropsTests::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckGetSupportedROMetricsPropsTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(OVCheckGetSupportedROMetricsPropsTests::getROOptionalProperties())), OVCheckGetSupportedROMetricsPropsTests::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVCheckSetSupportedRWMetricsPropsTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(OVCheckSetSupportedRWMetricsPropsTests::getRWMandatoryPropertiesValues())), OVCheckSetSupportedRWMetricsPropsTests::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckSetSupportedRWMetricsPropsTests, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(OVCheckSetSupportedRWMetricsPropsTests::getRWOptionalPropertiesValues())), OVCheckSetSupportedRWMetricsPropsTests::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVCheckChangePropComplieModleGetPropTests_DEVICE_ID, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(ov::AnyMap({}))), OVCheckChangePropComplieModleGetPropTests_DEVICE_ID::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVCheckChangePropComplieModleGetPropTests_InferencePrecision, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::Values(ov::AnyMap({}))), OVCheckChangePropComplieModleGetPropTests_InferencePrecision::getTestCaseName); INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckMetricsPropsTests_ModelDependceProps, ::testing::Combine( - ::testing::ValuesIn(return_all_possible_device_combination()), + ::testing::Values(targetDevice), ::testing::ValuesIn(OVCheckMetricsPropsTests_ModelDependceProps::getModelDependcePropertiesValues())), OVCheckMetricsPropsTests_ModelDependceProps::getTestCaseName); @@ -101,15 +81,15 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckMetricsPropsTests_ModelDependceProps, INSTANTIATE_TEST_SUITE_P( ov_plugin_mandatory, OVGetMetricPropsTest, - ::testing::ValuesIn(return_all_possible_device_combination(false))); + ::testing::Values(targetDevice)); INSTANTIATE_TEST_SUITE_P( ov_plugin, OVGetMetricPropsOptionalTest, - ::testing::ValuesIn(return_all_possible_device_combination(false))); + ::testing::Values(targetDevice)); INSTANTIATE_TEST_SUITE_P( ov_plugin_mandatory, OVGetAvailableDevicesPropsTest, - ::testing::ValuesIn(return_all_possible_device_combination(false))); + ::testing::Values(targetDevice)); // // IE Class GetConfig @@ -117,9 +97,9 @@ INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P( ov_plugin, OVPropertiesDefaultSupportedTests, - ::testing::ValuesIn(return_all_possible_device_combination(false))); + ::testing::Values(targetDevice)); INSTANTIATE_TEST_SUITE_P( ov_plugin_remove_mandatory, OVBasicPropertiesTestsP, - ::testing::ValuesIn(generate_pairs_plugin_name_by_device())); + ::testing::ValuesIn(generate_ov_pairs_plugin_name_by_device())); } // namespace diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/batch_config.txt b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/batch_config.txt new file mode 100644 index 00000000000000..e51aed2fc05f4c --- /dev/null +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/batch_config.txt @@ -0,0 +1 @@ +AUTO_BATCH_DEVICE_CONFIG TEMPLATE \ No newline at end of file diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/hetero_config.txt b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/hetero_config.txt new file mode 100644 index 00000000000000..4c8a8807b6aba4 --- /dev/null +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/hetero_config.txt @@ -0,0 +1 @@ +TARGET_FALLBACK TEMPLATE \ No newline at end of file diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/multi_config.txt b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/multi_config.txt new file mode 100644 index 00000000000000..af5c0ce5e5f5b5 --- /dev/null +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/config/multi_config.txt @@ -0,0 +1 @@ +MULTI_DEVICE_PRIORITIES TEMPLATE \ No newline at end of file