Skip to content

Commit

Permalink
Tests for dynamic preprocessing in SetBlob (openvinotoolkit#5798)
Browse files Browse the repository at this point in the history
* Corrected tests to match CVS-53713

* Fixed tests configs

* Skip tests on GPU

* Commented condition inside main SetBlob because of MYRIAD

* Adopted tests
  • Loading branch information
ilya-lavrenov authored and yekruglov committed Jun 7, 2021
1 parent 77236a1 commit eeac22f
Show file tree
Hide file tree
Showing 33 changed files with 815 additions and 290 deletions.
2 changes: 1 addition & 1 deletion docs/template_plugin/tests/functional/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ addIeTargetTest(
IE::funcSharedTests
INCLUDES
"${IE_MAIN_TEMPLATE_PLUGIN_SOURCE_DIR}/include"
ADD_CLANG_FORMAT
ADD_CPPLINT
LABELS
TEMPLATE
)
Expand Down
3 changes: 2 additions & 1 deletion docs/template_plugin/tests/functional/core_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@

#include "functional_test_utils/core_config.hpp"

void CoreConfiguration(LayerTestsUtils::LayerTestsCommon* test) {}
void CoreConfiguration(LayerTestsUtils::LayerTestsCommon* test) {
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,19 @@
using namespace LayerTestsDefinitions;

namespace {
static const std::vector<ngraph::element::Type> precisionsTemplate = {
ngraph::element::f32,
};
static const std::vector<ngraph::element::Type> precisionsTemplate = {
ngraph::element::f32,
};

static const std::vector<std::size_t> batchSizesTemplate = {1, 2};
static const std::vector<std::size_t> batchSizesTemplate = {
1, 2
};

INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_Template, LoadNetworkCacheTestBase,
::testing::Combine(::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()), ::testing::ValuesIn(precisionsTemplate),
::testing::ValuesIn(batchSizesTemplate), ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
LoadNetworkCacheTestBase::getTestCaseName);
} // namespace
INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_Template, LoadNetworkCacheTestBase,
::testing::Combine(
::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
::testing::ValuesIn(precisionsTemplate),
::testing::ValuesIn(batchSizesTemplate),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
LoadNetworkCacheTestBase::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,19 @@
// SPDX-License-Identifier: Apache-2.0
//

#include "behavior/config.hpp"
#include "multi-device/multi_device_config.hpp"

#include "behavior/config.hpp"
#include <template/template_config.hpp>

#include "multi-device/multi_device_config.hpp"

using namespace BehaviorTestsDefinitions;

namespace {

const std::vector<InferenceEngine::Precision> netPrecisions = {InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16};
const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::FP32,
InferenceEngine::Precision::FP16
};

const std::vector<std::map<std::string, std::string>> configs = {
{{TEMPLATE_CONFIG_KEY(THROUGHPUT_STREAMS), InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}},
Expand All @@ -25,23 +27,32 @@ const std::vector<std::map<std::string, std::string>> inconfigs = {
};

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigTests,
::testing::Combine(::testing::ValuesIn(netPrecisions), ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(inconfigs)),
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(inconfigs)),
IncorrectConfigTests::getTestCaseName);

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigAPITests,
::testing::Combine(::testing::ValuesIn(netPrecisions), ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(inconfigs)),
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(inconfigs)),
IncorrectConfigAPITests::getTestCaseName);


INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, CorrectConfigAPITests,
::testing::Combine(::testing::ValuesIn(netPrecisions), ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
CorrectConfigAPITests::getTestCaseName);

INSTANTIATE_TEST_CASE_P(smoke_Multi_BehaviorTests, CorrectConfigTests,
::testing::Combine(::testing::ValuesIn(netPrecisions), ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
CorrectConfigAPITests::getTestCaseName);

} // namespace
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
// SPDX-License-Identifier: Apache-2.0
//

#include "behavior/core_integration.hpp"

#include <string>
#include <utility>
#include <string>
#include <vector>

#include "behavior/core_integration.hpp"

using namespace BehaviorTestsDefinitions;

namespace {
Expand All @@ -16,31 +16,54 @@ namespace {
// IE Class Common tests with <pluginName, deviceName params>
//

INSTANTIATE_TEST_CASE_P(smoke_IEClassBasicTestP, IEClassBasicTestP, ::testing::Values(std::make_pair("templatePlugin", CommonTestUtils::DEVICE_TEMPLATE)));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassBasicTestP, IEClassBasicTestP,
::testing::Values(std::make_pair("templatePlugin", CommonTestUtils::DEVICE_TEMPLATE)));

INSTANTIATE_TEST_CASE_P(smoke_IEClassNetworkTestP, IEClassNetworkTestP, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassNetworkTestP, IEClassNetworkTestP,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

//
// IE Class GetMetric
//

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetAvailableDevices, IEClassGetAvailableDevices,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetAvailableDevices, IEClassGetAvailableDevices, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

//
// IE Class SetConfig
Expand Down Expand Up @@ -88,7 +111,9 @@ TEST_F(IEClassSetConfigTestHETERO, smoke_SetConfigNoThrow) {
// IE Class GetConfig
//

INSTANTIATE_TEST_CASE_P(smoke_IEClassGetConfigTest, IEClassGetConfigTest, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassGetConfigTest, IEClassGetConfigTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

using IEClassGetConfigTestTEMPLATE = IEClassNetworkTest;

Expand All @@ -100,7 +125,7 @@ TEST_F(IEClassGetConfigTestTEMPLATE, smoke_GetConfigNoThrow) {
ASSERT_NO_THROW(p = ie.GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS)));
std::vector<std::string> configValues = p;

for (auto&& confKey : configValues) {
for (auto &&confKey : configValues) {
if (CONFIG_KEY(DEVICE_ID) == confKey) {
std::string defaultDeviceID = ie.GetConfig(deviceName, CONFIG_KEY(DEVICE_ID));
std::cout << CONFIG_KEY(DEVICE_ID) << " : " << defaultDeviceID << std::endl;
Expand All @@ -118,55 +143,70 @@ TEST_F(IEClassGetConfigTestTEMPLATE, smoke_GetConfigNoThrow) {
// Executable Network GetMetric
//

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported, IEClassExecutableNetworkGetMetricTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported, IEClassExecutableNetworkGetMetricTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
//
// Executable Network GetConfig / SetConfig
//

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

// IE Class Query network

INSTANTIATE_TEST_CASE_P(smoke_IEClassQueryNetworkTest, IEClassQueryNetworkTest, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassQueryNetworkTest, IEClassQueryNetworkTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

// IE Class Load network

INSTANTIATE_TEST_CASE_P(smoke_IEClassLoadNetworkTest, IEClassLoadNetworkTest, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassLoadNetworkTest, IEClassLoadNetworkTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

//
// Hetero Executable Network GetMetric
//

#ifdef ENABLE_MKL_DNN

INSTANTIATE_TEST_CASE_P(smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

INSTANTIATE_TEST_CASE_P(smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
INSTANTIATE_TEST_CASE_P(
smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

#endif // ENABLE_MKL_DNN
} // namespace
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,32 @@ using namespace BehaviorTestsDefinitions;

namespace {

const std::vector<std::vector<int>> orders = {
const std::vector<std::vector<int >> orders = {
// 0 - plugin
// 1 - executable_network
// 2 - infer_request
{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}};

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTest, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), ::testing::ValuesIn(orders)),
HoldersTest::getTestCaseName);
{0, 1, 2},
{0, 2, 1},
{1, 0, 2},
{1, 2, 0},
{2, 0, 1},
{2, 1, 0}
};

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(orders)),
HoldersTest::getTestCaseName);

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTestImportNetwork,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "HETERO:TEMPLATE"), ::testing::ValuesIn(orders)),
HoldersTest::getTestCaseName);

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTestOnImportedNetwork, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "HETERO:TEMPLATE"),
HoldersTestOnImportedNetwork::getTestCaseName);
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "HETERO:TEMPLATE"),
::testing::ValuesIn(orders)),
HoldersTest::getTestCaseName);

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTestOnImportedNetwork,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "HETERO:TEMPLATE"),
HoldersTestOnImportedNetwork::getTestCaseName);

} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,28 @@
// SPDX-License-Identifier: Apache-2.0
//

#include "behavior/exec_graph_info.hpp"

#include <vector>

#include "behavior/exec_graph_info.hpp"

using namespace BehaviorTestsDefinitions;

namespace {

const std::vector<InferenceEngine::Precision> netPrecisions = {InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16};
const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::FP32,
InferenceEngine::Precision::FP16
};

const std::vector<std::map<std::string, std::string>> configs = {{}};
const std::vector<std::map<std::string, std::string>> configs = {
{}
};

INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, ExecGraphTests,
::testing::Combine(::testing::ValuesIn(netPrecisions), ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
ExecGraphTests::getTestCaseName);

} // namespace
Loading

0 comments on commit eeac22f

Please sign in to comment.