Skip to content

Commit

Permalink
API Conformance report: C++ & Merge XML Part (openvinotoolkit#11245)
Browse files Browse the repository at this point in the history
* Separate Summary, OpSummary, ApiSummary

* final separatin

* ChangeNamespaces

* git status

* filename

* link error

* Fix linking

* Fix compilation

* Report - ieplugin

* Next step

* Fix build

* Changing inheritance + fix build

* Fix run

* Summary

* Fix comments

* Fix the run

* fix build

* #Extend report

* Fix build

* fix template

* api_report flag

* rebase to master branch

* fix

* fix build

* myriad

* fix problem with crash

* Fix some mistakes

* python merge

* fix tests

* tmp

* Update Merge_xml script

* Fix op

* fix build

* Fix bug with --report_unique_name

* build

* remove extra

* gg

* gpu build

* c

* Fix issue with win

* infer_req

* compiled + exec net

* ov_plugin

* ie_plugin

* Fix comments

* ff

* fix last comment

* fix build

* fix template func

* Apply commnets

* Apply commnets

* fix ci

* build

* build

* build

* inl

* Remove extra

* fix merge_xml

* fix build

* remarks

* skip one test

Co-authored-by: Alexander Zhogov <[email protected]>
  • Loading branch information
iefode and Alexander Zhogov authored Aug 15, 2022
1 parent dd55f43 commit d04521a
Show file tree
Hide file tree
Showing 132 changed files with 2,552 additions and 2,006 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ TEST_P(OVInferRequestBatchedTests, SetInputTensors_Batch_Non_0) {
auto batch_shape = Shape{batch, 3, 3, 3};
auto model = OVInferRequestBatchedTests::create_n_inputs(1, element::f32, batch_shape, "CNHW");
const std::string tensor_name = "tensor_input0";
auto execNet = ie->compile_model(model, targetDevice);
auto execNet = ie->compile_model(model, target_device);
ov::InferRequest req;
req = execNet.create_infer_request();
std::vector<ov::Tensor> tensors(batch, ov::Tensor(element::f32, one_shape));
Expand All @@ -31,7 +31,7 @@ TEST_P(OVInferRequestBatchedTests, SetInputTensors_remote_tensor_default) {
auto batch_shape = Shape{batch, 4, 4, 4};
auto model = OVInferRequestBatchedTests::create_n_inputs(1, element::f32, batch_shape, "NCHW");
const std::string tensor_name = "tensor_input0";
auto execNet = ie->compile_model(model, targetDevice);
auto execNet = ie->compile_model(model, target_device);
ov::InferRequest req;
req = execNet.create_infer_request();
std::vector<ov::Tensor> tensors(batch - 1, ov::Tensor(element::f32, one_shape));
Expand All @@ -49,7 +49,7 @@ TEST_P(OVInferRequestBatchedTests, SetInputTensors_Strides) {
auto model = OVInferRequestBatchedTests::create_n_inputs(2, element::f32, batch_shape, "NCHW");
std::vector<float> buffer1(one_shape_size_stride, 10);
std::vector<float> buffer2(one_shape_size_stride, 20);
auto execNet = ie->compile_model(model, targetDevice);
auto execNet = ie->compile_model(model, target_device);
// Create InferRequest
ov::InferRequest req;
req = execNet.create_infer_request();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,13 @@ INSTANTIATE_TEST_SUITE_P(
// IE Class SetConfig
//

using IEClassSetConfigTestHETERO = BehaviorTestsUtils::IEClassNetworkTest;
class IEClassSetConfigTestHETERO : public BehaviorTestsUtils::IEClassNetworkTest,
public BehaviorTestsUtils::IEPluginTestBase {
void SetUp() override {
IEClassNetworkTest::SetUp();
IEPluginTestBase::SetUp();
}
};

TEST_F(IEClassSetConfigTestHETERO, smoke_SetConfigNoThrow) {
{
Expand Down Expand Up @@ -115,7 +121,13 @@ INSTANTIATE_TEST_SUITE_P(
smoke_IEClassGetConfigTest, IEClassGetConfigTest,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

using IEClassGetConfigTestTEMPLATE = BehaviorTestsUtils::IEClassNetworkTest;
class IEClassGetConfigTestTEMPLATE : public BehaviorTestsUtils::IEClassNetworkTest,
public BehaviorTestsUtils::IEPluginTestBase {
void SetUp() override {
IEClassNetworkTest::SetUp();
IEPluginTestBase::SetUp();
}
};

TEST_F(IEClassGetConfigTestTEMPLATE, smoke_GetConfigNoThrow) {
InferenceEngine::Core ie = BehaviorTestsUtils::createIECoreWithTemplate();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
#include "ngraph/node.hpp"
#include "pugixml.hpp"

#include "functional_test_utils/include/functional_test_utils/layer_test_utils/op_info.hpp"
#include "functional_test_utils/include/functional_test_utils/summary/op_info.hpp"

namespace SubgraphsDumper {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include <memory>
#include <ngraph/ngraph.hpp>
#include "matchers/matchers_manager.hpp"
#include "functional_test_utils/include/functional_test_utils/layer_test_utils/op_info.hpp"
#include "functional_test_utils/include/functional_test_utils/summary/op_info.hpp"

namespace SubgraphsDumper {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include "gtest/gtest.h"
#include "matchers/convolutions.hpp"
#include "ngraph/ops.hpp"
#include "functional_test_utils/include/functional_test_utils/layer_test_utils/op_info.hpp"
#include "functional_test_utils/include/functional_test_utils/summary/op_info.hpp"

class ConvolutionMatcherTest : public ::testing::Test {
protected:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include "gtest/gtest.h"
#include "matchers/single_op.hpp"
#include "ngraph/ops.hpp"
#include "functional_test_utils/include/functional_test_utils/layer_test_utils/op_info.hpp"
#include "functional_test_utils/include/functional_test_utils/summary/op_info.hpp"

class SingleOpMatcherTest : public ::testing::Test {
protected:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ inline const std::vector<std::map<std::string, std::string>> generate_configs(co
return resultConfig;
}

inline const std::string generate_complex_device_name(const std::string& deviceName) {
inline const std::string generate_complex_device_name(const std::string deviceName) {
return deviceName + ":" + ov::test::conformance::targetDevice;
}

Expand All @@ -85,9 +85,27 @@ inline const std::vector<std::string> return_all_possible_device_combination() {
return res;
}

const std::vector<std::map<std::string, std::string>> empty_config = {
{},
};
inline std::vector<std::pair<std::string, std::string>> generate_pairs_plugin_name_by_device() {
std::vector<std::pair<std::string, std::string>> res;
for (const auto& device : return_all_possible_device_combination()) {
std::string real_device = device.substr(0, device.find(':'));
res.push_back(std::make_pair(get_plugin_lib_name_by_device(ov::test::conformance::targetDevice),
real_device));
}
return res;
}

inline std::map<std::string, std::string> AnyMap2StringMap(const AnyMap& config) {
if (config.empty())
return {};
std::map<std::string, std::string> result;
for (const auto& configItem : config) {
result.insert({configItem.first, configItem.second.as<std::string>()});
}
return result;
}

const std::map<std::string, std::string> ie_config = AnyMap2StringMap(ov::test::conformance::pluginConfig);

} // namespace conformance
} // namespace test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@ inline const std::vector<ov::AnyMap> generate_ov_configs(const std::string& targ
return resultConfig;
}

const std::vector<ov::AnyMap> empty_ov_config = {
{},
};
const ov::AnyMap ov_config = ov::test::conformance::pluginConfig;

} // namespace conformance
} // namespace test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace {
using namespace ExecutionGraphTests;

INSTANTIATE_TEST_SUITE_P(ie_executable_network, ExecGraphSerializationTest,
::testing::Values(ov::test::conformance::targetDevice),
::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination()),
ExecGraphSerializationTest::getTestCaseName);

const std::vector<InferenceEngine::Precision> execGraphInfoElemTypes = {
Expand All @@ -22,7 +22,7 @@ INSTANTIATE_TEST_SUITE_P(ie_executable_network, ExecGraphUniqueNodeNames,
::testing::Combine(
::testing::ValuesIn(execGraphInfoElemTypes),
::testing::Values(InferenceEngine::SizeVector({1, 2, 5, 5})),
::testing::Values(ov::test::conformance::targetDevice)),
::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination())),
ExecGraphUniqueNodeNames::getTestCaseName);

} // namespace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace {
INSTANTIATE_TEST_SUITE_P(ie_executable_network, ExecutableNetworkBaseTest,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
ExecutableNetworkBaseTest::getTestCaseName);

const std::vector<InferenceEngine::Precision> execNetBaseElemTypes = {
Expand All @@ -28,6 +28,6 @@ namespace {
::testing::Combine(
::testing::ValuesIn(execNetBaseElemTypes),
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
ExecNetSetPrecision::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ INSTANTIATE_TEST_SUITE_P(

INSTANTIATE_TEST_SUITE_P(
smoke_IEClassHeteroExecutableNetworkGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values(ov::test::conformance::targetDevice));
::testing::ValuesIn(return_all_possible_device_combination()));

INSTANTIATE_TEST_SUITE_P(
ie_executable_network, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestCallbackTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestCallbackTests::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestCancellationTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestCancellationTests::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestIOBBlobTest,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestIOBBlobTest::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ using namespace BehaviorTestsDefinitions;
INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestMultithreadingTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestMultithreadingTests::getTestCaseName);

} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ using namespace BehaviorTestsDefinitions;
INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestPerfCountersTest,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestPerfCountersTest::getTestCaseName);

} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,14 @@ const std::vector<FuncTestUtils::BlobType> setBlobTypes = {
FuncTestUtils::BlobType::Compound,
FuncTestUtils::BlobType::Batched,
FuncTestUtils::BlobType::Memory,
// FuncTestUtils::BlobType::Remote,
FuncTestUtils::BlobType::Remote,
FuncTestUtils::BlobType::I420,
FuncTestUtils::BlobType::NV12
};

const std::map<std::string, std::string> ConfigBlobType{}; //nothing special

INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestSetBlobByType,
::testing::Combine(::testing::ValuesIn(setBlobTypes),
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestSetBlobByType::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ using namespace BehaviorTestsDefinitions;
INSTANTIATE_TEST_SUITE_P(ie_infer_request, InferRequestWaitTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_config)),
::testing::Values(ie_config)),
InferRequestWaitTests::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,6 @@ INSTANTIATE_TEST_SUITE_P(ov_compiled_model,
::testing::Combine(
::testing::ValuesIn(ovExecGraphInfoElemTypes),
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_ov_config)),
::testing::Values(ov_config)),
OVExecGraphImportExportTest::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(ov_compiled_model, OVExecutableNetworkBaseTest,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_ov_config)),
::testing::Values(ov_config)),
OVExecutableNetworkBaseTest::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ using namespace InferenceEngine::PluginConfigParams;


INSTANTIATE_TEST_SUITE_P(
ov_compiled_model, OVClassImportExportTestP,
ov_compiled_model, OVClassExecutableNetworkImportExportTestP,
::testing::ValuesIn(return_all_possible_device_combination()));

//
Expand Down Expand Up @@ -55,7 +55,7 @@ INSTANTIATE_TEST_SUITE_P(

INSTANTIATE_TEST_SUITE_P(
ov_compiled_model, OVClassExecutableNetworkSetConfigTest,
::testing::Values(ov::test::conformance::targetDevice));
::testing::ValuesIn(return_all_possible_device_combination()));

////
//// Hetero Executable Network GetMetric
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,15 @@ const std::vector<ov::AnyMap> inproperties = {
};

const std::vector<ov::AnyMap> auto_batch_inproperties = {
{{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}},
{{ov::auto_batch_timeout(-1)}},
};

INSTANTIATE_TEST_SUITE_P(ov_compiled_model, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(ov::test::conformance::targetDevice),
::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination()),
::testing::ValuesIn(inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_Hetero, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(generate_ov_configs(CommonTestUtils::DEVICE_HETERO, inproperties))),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_Multi, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(generate_ov_configs(CommonTestUtils::DEVICE_MULTI, inproperties))),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_Auto, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(generate_ov_configs(CommonTestUtils::DEVICE_AUTO, inproperties))),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_AutoBatch, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
Expand All @@ -63,35 +44,16 @@ INSTANTIATE_TEST_SUITE_P(ov_compiled_model, OVCompiledModelPropertiesDefaultTest
OVCompiledModelPropertiesDefaultTests::getTestCaseName);

const std::vector<ov::AnyMap> auto_batch_properties = {
{},
{{CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}},
{{ov::auto_batch_timeout(10)}},
};

INSTANTIATE_TEST_SUITE_P(ov_compiled_model, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(ov::test::conformance::targetDevice),
::testing::ValuesIn(ov::test::conformance::return_all_possible_device_combination()),
::testing::ValuesIn(default_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_Hetero, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(ov::test::conformance::generate_ov_configs(CommonTestUtils::DEVICE_HETERO, default_properties))),
OVCompiledModelPropertiesTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_Multi, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(ov::test::conformance::generate_ov_configs(CommonTestUtils::DEVICE_MULTI, default_properties))),
OVCompiledModelPropertiesTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_Auto, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(ov::test::conformance::generate_ov_configs(CommonTestUtils::DEVICE_AUTO, default_properties))),
OVCompiledModelPropertiesTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_compiled_model_AutoBatch, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(ov_infer_request, OVInferRequestCallbackTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_ov_config)),
::testing::Values(ov_config)),
OVInferRequestCallbackTests::getTestCaseName);

} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ using namespace ov::test::conformance;
INSTANTIATE_TEST_SUITE_P(ov_infer_request, OVInferRequestCancellationTests,
::testing::Combine(
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_ov_config)),
::testing::Values(ov_config)),
OVInferRequestCancellationTests::getTestCaseName);
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ INSTANTIATE_TEST_SUITE_P(ov_infer_request_1, OVInferRequestDynamicTests,
{{1, 4, 20, 20}, {1, 4, 20, 20}},
{{2, 4, 20, 20}, {2, 4, 20, 20}}}),
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_ov_config)),
::testing::Values(ov_config)),
OVInferRequestDynamicTests::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_infer_request_2, OVInferRequestDynamicTests,
Expand All @@ -71,6 +71,6 @@ INSTANTIATE_TEST_SUITE_P(ov_infer_request_2, OVInferRequestDynamicTests,
{{1, 4, 20, 20}, {1, 2, 20, 40}},
{{2, 4, 20, 20}, {2, 2, 20, 40}}}),
::testing::ValuesIn(return_all_possible_device_combination()),
::testing::ValuesIn(empty_ov_config)),
::testing::Values(ov_config)),
OVInferRequestDynamicTests::getTestCaseName);
} // namespace
Loading

0 comments on commit d04521a

Please sign in to comment.