Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable CPU accelerate FIL in MULTI #7380

Merged
merged 2 commits into from
Sep 17, 2021
Merged

Enable CPU accelerate FIL in MULTI #7380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6a52b3
Enable CPU accelerate FIL in MULTI
tiger100256-hu Sep 6, 2021
504de80
add configure to device
tiger100256-hu Sep 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
337 changes: 277 additions & 60 deletions inference-engine/src/multi_device/multi_device_exec_network.cpp
View file
Open in desktop

Large diffs are not rendered by default.

39 changes: 38 additions & 1 deletion inference-engine/src/multi_device/multi_device_exec_network.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,21 @@
#include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
#include <ie_parallel.hpp>
#include <threading/ie_itask_executor.hpp>
#include <threading/ie_executor_manager.hpp>
#include "ie_icore.hpp"

#if (IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO)
# include <tbb/concurrent_queue.h>
#endif


namespace MultiDevicePlugin {

class MultiDeviceInferencePlugin;

using DeviceName = std::string;
using NetworkFuture = std::future<InferenceEngine::SoExecutableNetworkInternal>;
using NetworkPromise = std::promise<InferenceEngine::SoExecutableNetworkInternal>;

struct DeviceInformation {
DeviceName deviceName;
Expand Down Expand Up @@ -105,10 +112,16 @@ class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkTh
};
using NotBusyWorkerRequests = ThreadSafeBoundedQueue<WorkerInferRequest*>;

explicit MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::SoExecutableNetworkInternal>& networksPerDevice,
explicit MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::SoExecutableNetworkInternal>& networksPerDevice,
const std::vector<DeviceInformation>& networkDevices,
const std::unordered_map<std::string, InferenceEngine::Parameter>& config,
const bool needPerfCounters = false);
MultiDeviceExecutableNetwork(const std::string& modelPath,
const InferenceEngine::CNNNetwork& network,
const std::vector<DeviceInformation>& metaDevices,
const std::string& strDevices,
MultiDeviceInferencePlugin* plugin,
const bool needPerfCounters = false);

void SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config) override;
InferenceEngine::Parameter GetConfig(const std::string &name) const override;
Expand Down Expand Up @@ -138,6 +151,30 @@ class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkTh
std::unordered_map<std::string, InferenceEngine::Parameter> _config;
bool _needPerfCounters = false;
std::atomic_size_t _numRequestsCreated = {0};

private:
void GenerateWorkers(const std::string& device, const InferenceEngine::SoExecutableNetworkInternal& executableNetwork);
void WaitActualNetworkReady() const;
void WaitFirstNetworkReady();
static bool RunPipelineTask(InferenceEngine::Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device);

private:
std::shared_ptr<InferenceEngine::ICore> _core;
InferenceEngine::IStreamsExecutor::Ptr _executor;
MultiDeviceInferencePlugin* _multiPlugin;
InferenceEngine::SoExecutableNetworkInternal _networkFirstReady;
mutable InferenceEngine::SoExecutableNetworkInternal _networkActualNeeded;
NetworkFuture _cpuFuture;
NetworkPromise _cpuPromise;
mutable NetworkFuture _acceleratorFuture;
mutable NetworkPromise _acceleratorPromise;
mutable bool _alreadyActualNetwork = {false};
bool _workModeIsAUTO = {false};
DeviceInformation _cpuDevice;
DeviceInformation _acceleratorDevice;
mutable std::once_flag _oc;
};

} // namespace MultiDevicePlugin
153 changes: 129 additions & 24 deletions inference-engine/src/multi_device/multi_device_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,34 +219,50 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
bool workModeAuto = workMode != fullConfig.end() && workMode->second == InferenceEngine::PluginConfigParams::YES;
auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);

// not found device priorities for -d AUTO use case
if (priorities == fullConfig.end()) {
if (workModeAuto) {
std::string allDevices;
auto availableDevices = GetCore()->GetAvailableDevices();
if (availableDevices.empty()) {
IE_THROW(NotFound) << "No available device found";
}
for (auto&& device : availableDevices) {
allDevices += device;
allDevices += ((device == availableDevices[availableDevices.size()-1]) ? "" : ",");
}
metaDevices = ParseMetaDevices(allDevices, fullConfig);
multiNetworkConfig.insert({MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, allDevices});
} else {
IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
// if workMode is AUTO
if (workModeAuto) {
// check the configure and check if need to set PerfCounters configure to device
// and set filter configure
bool needPerfCounters = false;
std::map<std::string, std::string> filterConfig;
CheckConfig(fullConfig, needPerfCounters, filterConfig);
// filter the device that supports filter configure
auto strDevices = GetDeviceList(fullConfig);
auto metaDevices = ParseMetaDevices(strDevices, fullConfig);
auto supportDevices = FilterDevice(metaDevices, filterConfig);
if (supportDevices.size() == 0) {
IE_THROW() << "there is no device support the configure";
}
// replace the configure with configure that auto want to pass to device
// and reset the strDevices to support devices
std::vector<std::string> validConfigKey;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor but better to do std::vectorstd::string validConfigKey = PerfHintsConfig::SupportedKeys(); rather than explicitly add the specific values in the lines 241-242 below ( since if the new config is added to the hints impl, you will need to explicitly add that)

validConfigKey.push_back(PluginConfigParams::KEY_PERF_COUNT);
validConfigKey.push_back(PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor comment: I assume this ASYNC_REQUESTS config key is only needed becuase OV has legacy impl and tests for that. but not sure of new devices like VPUX will support that, as the mode that is triggered by this key is obslolete.

So let's check that this checking this (evetually useless) config capability doesn't prevent us form using new devices like CUDA and KMB, and if this is the case- will need to remove maybe in the next PR

validConfigKey.push_back(PluginConfigParams::KEY_PERFORMANCE_HINT);
validConfigKey.push_back(PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS);
strDevices = "";
for (auto iter = supportDevices.begin(); iter != supportDevices.end(); iter++) {
std::map<std::string, std::string> deviceConfig;
auto& configs = iter->config;
for (auto& config : configs) {
if (std::find(validConfigKey.begin(), validConfigKey.end(), config.first) != validConfigKey.end()) {
deviceConfig.insert({config.first, config.second});
}
}
iter->config = deviceConfig;
strDevices = iter->deviceName;
strDevices += ((iter + 1) == supportDevices.end()) ? "" : ",";
}

return std::make_shared<MultiDeviceExecutableNetwork>(modelPath, network, supportDevices, strDevices, this, needPerfCounters);
}

if (priorities == fullConfig.end()) {
IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
} else { // for use case -d MULTI:xPU or -d AUTO:xPU
metaDevices = ParseMetaDevices(priorities->second, fullConfig);
multiNetworkConfig.insert(*priorities);
}
// check if it is -d AUTO or -d AUTO:xPU use case
if (workModeAuto) {
// select the device
auto device = SelectDevice(metaDevices, networkPrecision).deviceName;
// parse the config for the device
metaDevices = ParseMetaDevices(SelectDevice(metaDevices, networkPrecision).deviceName, fullConfig);
}

DeviceMap<SoExecutableNetworkInternal> executableNetworkPerDevice;
std::mutex load_mutex;
Expand Down Expand Up @@ -345,7 +361,6 @@ QueryNetworkResult MultiDeviceInferencePlugin::QueryNetwork(const CNNNetwork&
return queryResult;
}


DeviceInformation MultiDeviceInferencePlugin::SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision) {
if (metaDevices.empty()) {
IE_THROW(NotFound) << "No available device to select in " << GetName() << " plugin";
Expand Down Expand Up @@ -466,4 +481,94 @@ DeviceInformation MultiDeviceInferencePlugin::SelectDevice(const std::vector<Dev
return CPU[0];
}

std::string MultiDeviceInferencePlugin::GetDeviceList(const std::map<std::string, std::string>& config) const {
std::string allDevices;

auto deviceListConfig = config.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
if (deviceListConfig == config.end()) {
auto deviceList = GetCore()->GetAvailableDevices();
for (auto&& device : deviceList) {
allDevices += device;
allDevices += ((device == deviceList[deviceList.size()-1]) ? "" : ",");
}
} else {
allDevices = deviceListConfig->second;
}

if (allDevices.empty()) {
IE_THROW() << "Please, check environment due to no supported devices can be used";
}

return allDevices;
}

void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::string>& config,
bool& needPerfCounters, std::map<std::string, std::string>& filterConfig) {
// TODO need to optimize this code, too much duplicated code
const auto perf_hints_configs = PerfHintsConfig::SupportedKeys();
for (auto&& kvp : config) {
if (kvp.first.find("AUTO_") == 0) {
continue;
} else if (kvp.first == PluginConfigParams::KEY_PERF_COUNT) {
if (kvp.second == PluginConfigParams::YES) {
needPerfCounters = true;
filterConfig.insert({kvp.first, kvp.second});
} else if (kvp.second == PluginConfigParams::NO) {
needPerfCounters = false;
} else {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} else if (kvp.first == PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS) {
if (kvp.second == PluginConfigParams::YES ||
kvp.second == PluginConfigParams::NO) {
continue;
} else {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} else if (std::find(perf_hints_configs.begin(), perf_hints_configs.end(), kvp.first) != perf_hints_configs.end()) {
PerfHintsConfig::CheckConfigAndValue(kvp);
} else if (supported_configKeys.end() == std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) {
IE_THROW() << "Unsupported config key: " << kvp.first;
}
}
}

std::vector<DeviceInformation> MultiDeviceInferencePlugin::FilterDevice(const std::vector<DeviceInformation>& metaDevices,
const std::map<std::string, std::string>& config) {
if (metaDevices.empty()) {
IE_THROW(NotFound) << "No available device to filter " << GetName() << " plugin";
}

if (config.size() == 0) {
return metaDevices;
}

std::vector<DeviceInformation> filterDevice;
for (auto&& item : metaDevices) {
bool support = true;
std::vector<std::string> supportedMetrics = GetCore()->GetMetric(item.deviceName, METRIC_KEY(SUPPORTED_METRICS));
if (std::find(supportedMetrics.begin(), supportedMetrics.end(), METRIC_KEY(SUPPORTED_CONFIG_KEYS)) != supportedMetrics.end()) {
std::vector<std::string> supportKeys = GetCore()->GetMetric(item.deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS));
for (auto&& kvp : config) {
auto targetKey = std::find(supportKeys.begin(), supportKeys.end(), kvp.first);
// if device have the key, we think the device support it
if (targetKey != supportKeys.end()) {
continue;
} else {
support = false;
break;
}
}
} else {
support = false;
}

if (support) {
filterDevice.push_back(item);
}
}
return filterDevice;
}
} // namespace MultiDevicePlugin
8 changes: 7 additions & 1 deletion inference-engine/src/multi_device/multi_device_plugin.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ class MultiDeviceInferencePlugin : public InferenceEngine::IInferencePlugin {
std::vector<MultiDevicePlugin::DeviceInformation> ParseMetaDevices(const std::string & devicesRequestsCfg,
const std::map<std::string, std::string> & config) const;

std::string GetDeviceList(const std::map<std::string, std::string>& config) const;
DeviceInformation SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision = METRIC_VALUE(FP32));

protected:
std::map<std::string, std::string> GetSupportedConfig(const std::map<std::string, std::string>& config,
const MultiDevicePlugin::DeviceName & deviceName) const;
Expand All @@ -45,7 +48,10 @@ class MultiDeviceInferencePlugin : public InferenceEngine::IInferencePlugin {
InferenceEngine::CNNNetwork network,
const std::map<std::string, std::string>& config,
const std::string &networkPrecision = METRIC_VALUE(FP32));
DeviceInformation SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision = METRIC_VALUE(FP32));
static void CheckConfig(const std::map<std::string, std::string>& config, bool& needPerfCounters,
std::map<std::string, std::string>& filterConfig);
std::vector<DeviceInformation> FilterDevice(const std::vector<DeviceInformation>& metaDevices,
const std::map<std::string, std::string>& config);
};

} // namespace MultiDevicePlugin
11 changes: 11 additions & 0 deletions ...ts/functional/plugin/conformance/test_runner/src/behavior/infer_request/perf_counters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ const std::vector<std::map<std::string, std::string>> MulticonfigsPerfCounters =
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};

const std::vector<std::map<std::string, std::string>> AutoconfigsPerfCounters = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(targetDevice),
Expand All @@ -30,4 +34,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPerfCountersTest
::testing::ValuesIn(MulticonfigsPerfCounters)),
InferRequestPerfCountersTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(AutoconfigsPerfCounters)),
InferRequestPerfCountersTest::getTestCaseName);


} // namespace
13 changes: 0 additions & 13 deletions ...gine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,5 @@ namespace {
::testing::ValuesIn(MultiInConfigs)),
InferRequestConfigTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestConfigTest,
::testing::Combine(
::testing::Values(1u),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(multiConfigs)),
InferRequestConfigTest::getTestCaseName);


INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests_, InferRequestConfigTest,
::testing::Combine(
::testing::Values(1u),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(MultiInConfigs)),
InferRequestConfigTest::getTestCaseName);
} // namespace
11 changes: 11 additions & 0 deletions ...sts/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/perf_counters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ const std::vector<std::map<std::string, std::string>> Multiconfigs = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES) , CommonTestUtils::DEVICE_CPU}}
};

const std::vector<std::map<std::string, std::string>> Autoconfigs = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES) , CommonTestUtils::DEVICE_CPU}}
};

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_CPU),
Expand All @@ -48,4 +52,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPerfCountersTest
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(Multiconfigs)),
InferRequestPerfCountersTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(Autoconfigs)),
InferRequestPerfCountersTest::getTestCaseName);

} // namespace
8 changes: 6 additions & 2 deletions inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/test_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}
};

const std::vector<std::map<std::string, std::string>> AutoConfigsInputOutput = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}}
};

const std::vector<std::map<std::string, std::string>> configsOutput = {
{},
{{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}
Expand All @@ -56,7 +60,7 @@ namespace {
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(MultiConfigsInputOutput)),
::testing::ValuesIn(AutoConfigsInputOutput)),
BehaviorTestOutput::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, BehaviorTests,
Expand Down Expand Up @@ -98,7 +102,7 @@ namespace {
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(MultiConfigsInputOutput)),
::testing::ValuesIn(AutoConfigsInputOutput)),
BehaviorTestInput::getTestCaseName);

} // namespace
Loading