Skip to content

Commit

Permalink
Enable CPU accelerate FIL in MULTI
Browse files Browse the repository at this point in the history 
Signed-off-by: Hu, Yuan2 <[email protected]>
  • Loading branch information
@tiger100256-hu
tiger100256-hu committed Sep 16, 2021
1 parent 7654789 commit a6a52b3
Show file tree
Hide file tree
Showing 15 changed files with 539 additions and 114 deletions.
337 changes: 277 additions & 60 deletions inference-engine/src/multi_device/multi_device_exec_network.cpp
View file

Large diffs are not rendered by default.

39 changes: 38 additions & 1 deletion inference-engine/src/multi_device/multi_device_exec_network.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,21 @@
#include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
#include <ie_parallel.hpp>
#include <threading/ie_itask_executor.hpp>
#include <threading/ie_executor_manager.hpp>
#include "ie_icore.hpp"

#if (IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO)
# include <tbb/concurrent_queue.h>
#endif


namespace MultiDevicePlugin {

class MultiDeviceInferencePlugin;

using DeviceName = std::string;
using NetworkFuture = std::future<InferenceEngine::SoExecutableNetworkInternal>;
using NetworkPromise = std::promise<InferenceEngine::SoExecutableNetworkInternal>;

struct DeviceInformation {
DeviceName deviceName;
Expand Down Expand Up @@ -105,10 +112,16 @@ class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkTh
};
using NotBusyWorkerRequests = ThreadSafeBoundedQueue<WorkerInferRequest*>;

explicit MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::SoExecutableNetworkInternal>& networksPerDevice,
explicit MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::SoExecutableNetworkInternal>& networksPerDevice,
const std::vector<DeviceInformation>& networkDevices,
const std::unordered_map<std::string, InferenceEngine::Parameter>& config,
const bool needPerfCounters = false);
MultiDeviceExecutableNetwork(const std::string& modelPath,
const InferenceEngine::CNNNetwork& network,
const std::vector<DeviceInformation>& metaDevices,
const std::string& strDevices,
MultiDeviceInferencePlugin* plugin,
const bool needPerfCounters = false);

void SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config) override;
InferenceEngine::Parameter GetConfig(const std::string &name) const override;
Expand Down Expand Up @@ -138,6 +151,30 @@ class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkTh
std::unordered_map<std::string, InferenceEngine::Parameter> _config;
bool _needPerfCounters = false;
std::atomic_size_t _numRequestsCreated = {0};

private:
void GenerateWorkers(const std::string& device, const InferenceEngine::SoExecutableNetworkInternal& executableNetwork);
void WaitActualNetworkReady() const;
void WaitFirstNetworkReady();
static bool RunPipelineTask(InferenceEngine::Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device);

private:
std::shared_ptr<InferenceEngine::ICore> _core;
InferenceEngine::IStreamsExecutor::Ptr _executor;
MultiDeviceInferencePlugin* _multiPlugin;
InferenceEngine::SoExecutableNetworkInternal _networkFirstReady;
mutable InferenceEngine::SoExecutableNetworkInternal _networkActualNeeded;
NetworkFuture _cpuFuture;
NetworkPromise _cpuPromise;
mutable NetworkFuture _acceleratorFuture;
mutable NetworkPromise _acceleratorPromise;
mutable bool _alreadyActualNetwork = {false};
bool _workModeIsAUTO = {false};
DeviceInformation _cpuDevice;
DeviceInformation _acceleratorDevice;
mutable std::once_flag _oc;
};

} // namespace MultiDevicePlugin
152 changes: 128 additions & 24 deletions inference-engine/src/multi_device/multi_device_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,34 +219,49 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
bool workModeAuto = workMode != fullConfig.end() && workMode->second == InferenceEngine::PluginConfigParams::YES;
auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);

// not found device priorities for -d AUTO use case
if (priorities == fullConfig.end()) {
if (workModeAuto) {
std::string allDevices;
auto availableDevices = GetCore()->GetAvailableDevices();
if (availableDevices.empty()) {
IE_THROW(NotFound) << "No available device found";
}
for (auto&& device : availableDevices) {
allDevices += device;
allDevices += ((device == availableDevices[availableDevices.size()-1]) ? "" : ",");
}
metaDevices = ParseMetaDevices(allDevices, fullConfig);
multiNetworkConfig.insert({MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, allDevices});
} else {
IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
// if workMode is AUTO
if (workModeAuto) {
// check the configure and check if need to set PerfCounters configure to device
// and set filter configure
bool needPerfCounters = false;
std::map<std::string, std::string> filterConfig;
CheckConfig(fullConfig, needPerfCounters, filterConfig);
// filter the device that supports filter configure
auto strDevices = GetDeviceList(fullConfig);
auto metaDevices = ParseMetaDevices(strDevices, fullConfig);
auto supportDevices = FilterDevice(metaDevices, filterConfig);
if (supportDevices.size() == 0) {
IE_THROW() << "there is no device support the configure";
}
// replace the configure with configure that auto want to pass to device
// and reset the strDevices to support devices
std::vector<std::string> validConfigKey;
validConfigKey.push_back(PluginConfigParams::KEY_PERF_COUNT);
validConfigKey.push_back(PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS);
validConfigKey.push_back(PluginConfigParams::KEY_PERFORMANCE_HINT);
strDevices = "";
for (auto iter = supportDevices.begin(); iter != supportDevices.end(); iter++) {
std::map<std::string, std::string> deviceConfig;
auto& configs = iter->config;
for (auto& config : configs) {
if (std::find(validConfigKey.begin(), validConfigKey.end(), config.first) != validConfigKey.end()) {
deviceConfig.insert({config.first, config.second});
}
}
iter->config = deviceConfig;
strDevices = iter->deviceName;
strDevices += ((iter + 1) == supportDevices.end()) ? "" : ",";
}

return std::make_shared<MultiDeviceExecutableNetwork>(modelPath, network, supportDevices, strDevices, this, needPerfCounters);
}

if (priorities == fullConfig.end()) {
IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
} else { // for use case -d MULTI:xPU or -d AUTO:xPU
metaDevices = ParseMetaDevices(priorities->second, fullConfig);
multiNetworkConfig.insert(*priorities);
}
// check if it is -d AUTO or -d AUTO:xPU use case
if (workModeAuto) {
// select the device
auto device = SelectDevice(metaDevices, networkPrecision).deviceName;
// parse the config for the device
metaDevices = ParseMetaDevices(SelectDevice(metaDevices, networkPrecision).deviceName, fullConfig);
}

DeviceMap<SoExecutableNetworkInternal> executableNetworkPerDevice;
std::mutex load_mutex;
Expand Down Expand Up @@ -345,7 +360,6 @@ QueryNetworkResult MultiDeviceInferencePlugin::QueryNetwork(const CNNNetwork&
return queryResult;
}


DeviceInformation MultiDeviceInferencePlugin::SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision) {
if (metaDevices.empty()) {
IE_THROW(NotFound) << "No available device to select in " << GetName() << " plugin";
Expand Down Expand Up @@ -466,4 +480,94 @@ DeviceInformation MultiDeviceInferencePlugin::SelectDevice(const std::vector<Dev
return CPU[0];
}

std::string MultiDeviceInferencePlugin::GetDeviceList(const std::map<std::string, std::string>& config) const {
std::string allDevices;

auto deviceListConfig = config.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
if (deviceListConfig == config.end()) {
auto deviceList = GetCore()->GetAvailableDevices();
for (auto&& device : deviceList) {
allDevices += device;
allDevices += ((device == deviceList[deviceList.size()-1]) ? "" : ",");
}
} else {
allDevices = deviceListConfig->second;
}

if (allDevices.empty()) {
IE_THROW() << "Please, check environment due to no supported devices can be used";
}

return allDevices;
}

void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::string>& config,
bool& needPerfCounters, std::map<std::string, std::string>& filterConfig) {
// TODO need to optimize this code, too much duplicated code
const auto perf_hints_configs = PerfHintsConfig::SupportedKeys();
for (auto&& kvp : config) {
if (kvp.first.find("AUTO_") == 0) {
continue;
} else if (kvp.first == PluginConfigParams::KEY_PERF_COUNT) {
if (kvp.second == PluginConfigParams::YES) {
needPerfCounters = true;
filterConfig.insert({kvp.first, kvp.second});
} else if (kvp.second == PluginConfigParams::NO) {
needPerfCounters = false;
} else {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} else if (kvp.first == PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS) {
if (kvp.second == PluginConfigParams::YES ||
kvp.second == PluginConfigParams::NO) {
continue;
} else {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} else if (std::find(perf_hints_configs.begin(), perf_hints_configs.end(), kvp.first) != perf_hints_configs.end()) {
PerfHintsConfig::CheckConfigAndValue(kvp);
} else if (supported_configKeys.end() == std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) {
IE_THROW() << "Unsupported config key: " << kvp.first;
}
}
}

std::vector<DeviceInformation> MultiDeviceInferencePlugin::FilterDevice(const std::vector<DeviceInformation>& metaDevices,
const std::map<std::string, std::string>& config) {
if (metaDevices.empty()) {
IE_THROW(NotFound) << "No available device to filter " << GetName() << " plugin";
}

if (config.size() == 0) {
return metaDevices;
}

std::vector<DeviceInformation> filterDevice;
for (auto&& item : metaDevices) {
bool support = true;
std::vector<std::string> supportedMetrics = GetCore()->GetMetric(item.deviceName, METRIC_KEY(SUPPORTED_METRICS));
if (std::find(supportedMetrics.begin(), supportedMetrics.end(), METRIC_KEY(SUPPORTED_CONFIG_KEYS)) != supportedMetrics.end()) {
std::vector<std::string> supportKeys = GetCore()->GetMetric(item.deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS));
for (auto&& kvp : config) {
auto targetKey = std::find(supportKeys.begin(), supportKeys.end(), kvp.first);
// if device have the key, we think the device support it
if (targetKey != supportKeys.end()) {
continue;
} else {
support = false;
break;
}
}
} else {
support = false;
}

if (support) {
filterDevice.push_back(item);
}
}
return filterDevice;
}
} // namespace MultiDevicePlugin
8 changes: 7 additions & 1 deletion inference-engine/src/multi_device/multi_device_plugin.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ class MultiDeviceInferencePlugin : public InferenceEngine::IInferencePlugin {
std::vector<MultiDevicePlugin::DeviceInformation> ParseMetaDevices(const std::string & devicesRequestsCfg,
const std::map<std::string, std::string> & config) const;

std::string GetDeviceList(const std::map<std::string, std::string>& config) const;
DeviceInformation SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision = METRIC_VALUE(FP32));

protected:
std::map<std::string, std::string> GetSupportedConfig(const std::map<std::string, std::string>& config,
const MultiDevicePlugin::DeviceName & deviceName) const;
Expand All @@ -45,7 +48,10 @@ class MultiDeviceInferencePlugin : public InferenceEngine::IInferencePlugin {
InferenceEngine::CNNNetwork network,
const std::map<std::string, std::string>& config,
const std::string &networkPrecision = METRIC_VALUE(FP32));
DeviceInformation SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision = METRIC_VALUE(FP32));
static void CheckConfig(const std::map<std::string, std::string>& config, bool& needPerfCounters,
std::map<std::string, std::string>& filterConfig);
std::vector<DeviceInformation> FilterDevice(const std::vector<DeviceInformation>& metaDevices,
const std::map<std::string, std::string>& config);
};

} // namespace MultiDevicePlugin
11 changes: 11 additions & 0 deletions ...ts/functional/plugin/conformance/test_runner/src/behavior/infer_request/perf_counters.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ const std::vector<std::map<std::string, std::string>> MulticonfigsPerfCounters =
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};

const std::vector<std::map<std::string, std::string>> AutoconfigsPerfCounters = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES), targetDevice }}
};

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(targetDevice),
Expand All @@ -30,4 +34,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPerfCountersTest
::testing::ValuesIn(MulticonfigsPerfCounters)),
InferRequestPerfCountersTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(AutoconfigsPerfCounters)),
InferRequestPerfCountersTest::getTestCaseName);


} // namespace
13 changes: 0 additions & 13 deletions ...gine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,5 @@ namespace {
::testing::ValuesIn(MultiInConfigs)),
InferRequestConfigTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestConfigTest,
::testing::Combine(
::testing::Values(1u),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(multiConfigs)),
InferRequestConfigTest::getTestCaseName);


INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests_, InferRequestConfigTest,
::testing::Combine(
::testing::Values(1u),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(MultiInConfigs)),
InferRequestConfigTest::getTestCaseName);
} // namespace
11 changes: 11 additions & 0 deletions ...sts/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/perf_counters.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ const std::vector<std::map<std::string, std::string>> Multiconfigs = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES) , CommonTestUtils::DEVICE_CPU}}
};

const std::vector<std::map<std::string, std::string>> Autoconfigs = {
{{ MULTI_CONFIG_KEY(DEVICE_PRIORITIES) , CommonTestUtils::DEVICE_CPU}}
};

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_CPU),
Expand All @@ -48,4 +52,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, InferRequestPerfCountersTest
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(Multiconfigs)),
InferRequestPerfCountersTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, InferRequestPerfCountersTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(Autoconfigs)),
InferRequestPerfCountersTest::getTestCaseName);

} // namespace
8 changes: 6 additions & 2 deletions inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/test_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}
};

const std::vector<std::map<std::string, std::string>> AutoConfigsInputOutput = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}}
};

const std::vector<std::map<std::string, std::string>> configsOutput = {
{},
{{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}
Expand All @@ -56,7 +60,7 @@ namespace {
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(MultiConfigsInputOutput)),
::testing::ValuesIn(AutoConfigsInputOutput)),
BehaviorTestOutput::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, BehaviorTests,
Expand Down Expand Up @@ -98,7 +102,7 @@ namespace {
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(MultiConfigsInputOutput)),
::testing::ValuesIn(AutoConfigsInputOutput)),
BehaviorTestInput::getTestCaseName);

} // namespace
Loading

0 comments on commit a6a52b3

Please sign in to comment.