[AUTO] Implement auto-plugin limited devices feature

inference-engine/include/auto_plugin/auto_config.hpp

@@ -0,0 +1,38 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief A header that defines advanced related properties for Auto plugin.
+ * These properties should be used in SetConfig() and LoadNetwork() methods
+ *
+ * @file auto_config.hpp
+ */
+
+#pragma once
+
+#include "ie_plugin_config.hpp"
+
+namespace InferenceEngine {
+
+/**
+ * @brief Auto plugin configuration
+ */
+namespace AutoConfigParams {
+
+/**
+ * @def AUTO_CONFIG_KEY(name)
+ * @brief A macro which provides a AUTO-mangled name for configuration key with name `name`
+ */
+#define AUTO_CONFIG_KEY(name) InferenceEngine::AutoConfigParams::_CONFIG_KEY(AUTO_##name)
+
+#define DECLARE_AUTO_CONFIG_KEY(name) DECLARE_CONFIG_KEY(AUTO_##name)
+#define DECLARE_AUTO_CONFIG_VALUE(name) DECLARE_CONFIG_VALUE(AUTO_##name)
+
+/**
+ * @brief Limit device list config option, with comma-separated devices listed
+ */
+DECLARE_AUTO_CONFIG_KEY(DEVICE_LIST);
+
+}  // namespace AutoConfigParams
+}  // namespace InferenceEngine

inference-engine/src/auto_plugin/CMakeLists.txt

@@ -12,7 +12,7 @@ ie_add_plugin(NAME ${TARGET_NAME}
               SOURCES ${SOURCES} ${HEADERS}
               VERSION_DEFINES_FOR auto_plugin.cpp)
 
-target_link_libraries(${TARGET_NAME} PRIVATE inference_engine)
+target_link_libraries(${TARGET_NAME} PRIVATE ${NGRAPH_LIBRARIES} inference_engine inference_engine_transformations)
 
 ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME})
 

inference-engine/src/auto_plugin/auto_exec_network.cpp

@@ -13,16 +13,10 @@
 #include "auto_infer_request.hpp"
 
 namespace AutoPlugin {
-
 using namespace InferenceEngine;
 
-AutoExecutableNetwork::AutoExecutableNetwork(const SoExecutableNetworkInternal& network,
-                                             const DeviceInformation&           deviceInfo,
-                                             const bool                         needPerfCounters) :
-    _deviceInfo(deviceInfo),
-    _network(network),
-    _config(deviceInfo.config.begin(), deviceInfo.config.end()),
-    _needPerfCounters(needPerfCounters) {
+AutoExecutableNetwork::AutoExecutableNetwork(const SoExecutableNetworkInternal& network) :
+    _network(network) {
 }
 
 AutoExecutableNetwork::~AutoExecutableNetwork() = default;

inference-engine/src/auto_plugin/auto_exec_network.hpp

@@ -28,9 +28,7 @@ class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSaf
 public:
     using Ptr = std::shared_ptr<AutoExecutableNetwork>;
 
-    AutoExecutableNetwork(const InferenceEngine::SoExecutableNetworkInternal&  network,
-                          const DeviceInformation&                             deviceInfo,
-                          const bool                                           needPerfCounters = false);
+    explicit AutoExecutableNetwork(const InferenceEngine::SoExecutableNetworkInternal& network);
 
     void Export(std::ostream& networkModel) override;
     InferenceEngine::RemoteContext::Ptr GetContext() const override;
@@ -42,10 +40,8 @@ class AutoExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSaf
                                                                        InferenceEngine::OutputsDataMap networkOutputs) override;
     ~AutoExecutableNetwork() override;
 
-    DeviceInformation                                            _deviceInfo;
-    InferenceEngine::SoExecutableNetworkInternal                 _network;
-    std::unordered_map<std::string, InferenceEngine::Parameter>  _config;
-    bool                                                         _needPerfCounters = false;
+private:
+    InferenceEngine::SoExecutableNetworkInternal _network;
 };
 
 }  // namespace AutoPlugin

inference-engine/src/auto_plugin/auto_plugin.cpp

@@ -11,25 +11,37 @@
 #include <ie_metric_helpers.hpp>
 #include <threading/ie_executor_manager.hpp>
 #include <ie_algorithm.hpp>
+#include <ngraph/opsets/opset1.hpp>
+#include <transformations/utils/utils.hpp>
 
+#include <auto_plugin/auto_config.hpp>
 #include "auto_plugin.hpp"
+#include "ngraph_ops/convolution_ie.hpp"
+#include "ngraph_ops/deconvolution_ie.hpp"
 
 namespace AutoPlugin {
 namespace {
-    ConfigType mergeConfigs(ConfigType config, const ConfigType& local) {
-        for (auto && kvp : local) {
-            config[kvp.first] = kvp.second;
+    std::string GetNetworkPrecision(const InferenceEngine::CNNNetwork &network) {
+        auto nGraphFunc = network.getFunction();
+        bool isINTModel = ngraph::op::util::has_op_with_type<ngraph::op::FakeQuantize>(nGraphFunc);
+        if (isINTModel) {
+            return METRIC_VALUE(INT8);
         }
-        return config;
-    }
-
-    DeviceInformation SelectDevice(const std::vector<DeviceInformation>& metaDevices) {
-        for (auto& item : metaDevices) {
-            if (item.deviceName.find("CPU") == 0) {
-              return item;
+        for (auto & node : nGraphFunc->get_ordered_ops()) {
+            if (std::dynamic_pointer_cast<ngraph::opset1::Convolution>(node) ||
+                std::dynamic_pointer_cast<ngraph::opset1::GroupConvolution>(node) ||
+                std::dynamic_pointer_cast<ngraph::opset1::GroupConvolutionBackpropData>(node) ||
+                std::dynamic_pointer_cast<ngraph::opset1::ConvolutionBackpropData>(node) ||
+                std::dynamic_pointer_cast<ngraph::op::ConvolutionIE>(node) ||
+                std::dynamic_pointer_cast<ngraph::op::DeconvolutionIE>(node)) {
+                auto layerType = node->input(1).get_element_type().get_type_name();
+                if (layerType == "f32")
+                    return METRIC_VALUE(FP32);
+                if (layerType == "f16")
+                    return METRIC_VALUE(FP16);
             }
         }
-        IE_THROW(NotFound) << "No available device could be used";
+        return METRIC_VALUE(FP32);
     }
 }  // namespace
 
@@ -39,70 +51,17 @@ AutoInferencePlugin::AutoInferencePlugin() {
 
 IE::IExecutableNetworkInternal::Ptr AutoInferencePlugin::LoadNetwork(const std::string& fileName,
                                                                      const ConfigType&  config) {
-    if (GetCore() == nullptr) {
-        IE_THROW() << "Please, work with AUTO device via InferencEngine::Core object";
-    }
-
-    auto fullConfig = mergeConfigs(_config, config);
-    auto metaDevices = GetDeviceChoice(fullConfig);
-
-    // FIXME: always select CPU device now
-    DeviceInformation selectedDevice = SelectDevice(metaDevices);
-    IE::SoExecutableNetworkInternal executableNetwork;
-    try {
-        executableNetwork = GetCore()->LoadNetwork(fileName, selectedDevice.deviceName, selectedDevice.config);
-    } catch(const IE::Exception &iie) {
-        IE_THROW() << "Failed to load network to device named " << selectedDevice.deviceName
-                   << " with exception " << iie.what();
-    }
-
-    bool enablePerfCounters = false;
-    try {
-        enablePerfCounters =
-            executableNetwork->GetConfig(IE::PluginConfigParams::KEY_PERF_COUNT).as<std::string>() ==
-                IE::PluginConfigParams::YES;
-    } catch (...) {
-    }
-
-    return std::make_shared<AutoExecutableNetwork>(executableNetwork,
-                                                   selectedDevice,
-                                                   enablePerfCounters);
+    return LoadNetworkImpl(fileName, config);
 }
 
 IE::ExecutableNetworkInternal::Ptr AutoInferencePlugin::LoadExeNetworkImpl(const IE::CNNNetwork& network,
                                                                            const ConfigType&     config) {
-    if (GetCore() == nullptr) {
-        IE_THROW() << "Please, work with AUTO device via InferencEngine::Core object";
-    }
-
     if (network.getFunction() == nullptr) {
         IE_THROW() << "AUTO device supports just ngraph network representation";
     }
 
-    auto fullConfig = mergeConfigs(_config, config);
-    auto metaDevices = GetDeviceChoice(fullConfig);
-
-    // FIXME: always select CPU device now
-    DeviceInformation selectedDevice = SelectDevice(metaDevices);
-    IE::SoExecutableNetworkInternal executableNetwork;
-    try {
-        executableNetwork = GetCore()->LoadNetwork(network, selectedDevice.deviceName, selectedDevice.config);
-    } catch(const IE::Exception &iie) {
-        IE_THROW() << "Failed to load network to device named " << selectedDevice.deviceName
-                   << " with exception " << iie.what();
-    }
-
-    bool enablePerfCounters = false;
-    try {
-        enablePerfCounters =
-            executableNetwork->GetConfig(IE::PluginConfigParams::KEY_PERF_COUNT).as<std::string>() ==
-                IE::PluginConfigParams::YES;
-    } catch (...) {
-    }
-
-    return std::make_shared<AutoExecutableNetwork>(executableNetwork,
-                                                   selectedDevice,
-                                                   enablePerfCounters);
+    auto networkPrecision = GetNetworkPrecision(network);
+    return LoadNetworkImpl(network, config, networkPrecision);
 }
 
 IE::QueryNetworkResult AutoInferencePlugin::QueryNetwork(const IE::CNNNetwork& network, const ConfigType& config) const {
@@ -172,16 +131,24 @@ IE::Parameter AutoInferencePlugin::GetMetric(const std::string& name,
         std::vector<std::string> configKeys;
         IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
     } else if (name == METRIC_KEY(OPTIMIZATION_CAPABILITIES)) {
-        std::vector<std::string> capabilities = { "" };
+        std::vector<std::string> capabilities = GetOptimizationCapabilities();
         IE_SET_METRIC_RETURN(OPTIMIZATION_CAPABILITIES, capabilities);
     } else {
         IE_THROW() << "Unsupported metric key " << name;
     }
 }
 
+//////////////////////////////////// private & protected functions ///////////////////
 std::vector<AutoPlugin::DeviceInformation> AutoInferencePlugin::GetDeviceChoice(const ConfigType&  config) const {
     std::vector<DeviceInformation> metaDevices;
-    std::vector<std::string> availableDevices = GetCore()->GetAvailableDevices();
+    std::vector<std::string> availableDevices;
+
+    auto deviceListConfig = config.find(IE::AutoConfigParams::KEY_AUTO_DEVICE_LIST);
+    if (deviceListConfig == config.end()) {
+        availableDevices = GetCore()->GetAvailableDevices();
+    } else {
+        availableDevices = IE::DeviceIDParser::getHeteroDevices(deviceListConfig->second);
+    }
 
     auto getDeviceConfig = [&] (const DeviceName & deviceWithID) {
         IE::DeviceIDParser deviceParser(deviceWithID);
@@ -210,7 +177,28 @@ std::vector<AutoPlugin::DeviceInformation> AutoInferencePlugin::GetDeviceChoice(
     return metaDevices;
 }
 
-//////////////////////////////////// private & protected functions ///////////////////
+std::vector<std::string> AutoInferencePlugin::GetOptimizationCapabilities() const {
+    // FIXME: workaround to get devicelist.
+    std::unordered_set<std::string> capabilities;
+    std::vector<std::string> queryDeviceLists{"CPU", "GPU"};
+    for (auto &item : queryDeviceLists) {
+        try {
+            std::vector<std::string> device_cap =
+                GetCore()->GetMetric(item, METRIC_KEY(OPTIMIZATION_CAPABILITIES));
+            for (auto &cap : device_cap) {
+                // For CPU test SetBlobOfKindTest::CompareWithRefs which checks BATCHED_BLOB capability,
+                // and AUTO select CPU but not GPU (GPU has this capability).
+                if (cap == METRIC_VALUE(BATCHED_BLOB)) {
+                    continue;
+                }
+                capabilities.insert(cap);
+            }
+        } catch (...) {
+        }
+    }
+    return {capabilities.begin(), capabilities.end()};
+}
+
 ConfigType AutoInferencePlugin::GetSupportedConfig(const ConfigType&  config,
                                                    const std::string& deviceName) const {
     std::vector<std::string> supportedConfigKeys = GetCore()->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS));
@@ -224,6 +212,56 @@ ConfigType AutoInferencePlugin::GetSupportedConfig(const ConfigType&  config,
     return supportedConfig;
 }
 
+DeviceInformation AutoInferencePlugin::SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision) {
+    if (metaDevices.empty()) {
+        IE_THROW(NotFound) << "No available device to select in AUTO plugin";
+    }
+    if (metaDevices.size() == 1) {
+        return metaDevices.at(0);
+    }
+
+    std::vector<DeviceInformation> CPU;
+    std::vector<DeviceInformation> GPU;
+
+    for (auto& item : metaDevices) {
+        if (item.deviceName.find("CPU") == 0) {
+            CPU.push_back(item);
+            continue;
+        }
+        if (item.deviceName.find("GPU") == 0) {
+            GPU.push_back(item);
+            continue;
+        }
+    }
+
+    if (CPU.empty() && GPU.empty()) {
+        IE_THROW(NotFound) << "No available device found";
+    }
+
+    // Sort GPU by name: GPU.2 > GPU.1 > GPU.0 > GPU, so we always choose the GPU[0] as best device
+    std::sort(GPU.begin(), GPU.end(), [](const DeviceInformation& a, const DeviceInformation& b)->bool{return b.deviceName < a.deviceName;});
+
+    for (auto&& item : GPU) {
+        std::vector<std::string> capability = GetCore()->GetMetric(item.deviceName, METRIC_KEY(OPTIMIZATION_CAPABILITIES));
+        auto res = std::find(capability.begin(), capability.end(), networkPrecision);
+        if (res != capability.end()) {
+            return item;
+        }
+    }
+
+    if (CPU.empty()) {
+        IE_THROW() << "Cannot select any device";
+    }
+    return CPU[0];
+}
+
+ConfigType AutoInferencePlugin::mergeConfigs(ConfigType config, const ConfigType& local) {
+    for (auto && kvp : local) {
+        config[kvp.first] = kvp.second;
+    }
+    return config;
+}
+
 // define CreatePluginEngine to create plugin instance
 static const IE::Version version = {{2, 1}, CI_BUILD_NUMBER, "AutoPlugin"};
 IE_DEFINE_PLUGIN_CREATE_FUNCTION(AutoInferencePlugin, version)

inference-engine/src/auto_plugin/auto_plugin.hpp

@@ -30,9 +30,40 @@ class AutoInferencePlugin : public IE::InferencePluginInternal {
 
 private:
     std::vector<AutoPlugin::DeviceInformation> GetDeviceChoice(const ConfigType&  config) const;
-
-protected:
+    std::vector<std::string> GetOptimizationCapabilities() const;
+    DeviceInformation SelectDevice(const std::vector<DeviceInformation>& metaDevices, const std::string& networkPrecision = METRIC_VALUE(FP32));
     ConfigType GetSupportedConfig(const ConfigType& config, const AutoPlugin::DeviceName & deviceName) const;
+    static ConfigType mergeConfigs(ConfigType config, const ConfigType& local);
+
+    template <typename T>
+    std::shared_ptr<AutoExecutableNetwork> LoadNetworkImpl(const T &param, const ConfigType &config, const std::string &networkPrecision = METRIC_VALUE(FP32)) {
+        if (GetCore() == nullptr) {
+            IE_THROW() << "Please, work with AUTO device via InferencEngine::Core object";
+        }
+        auto fullConfig = mergeConfigs(_config, config);
+        auto metaDevices = GetDeviceChoice(fullConfig);
+        DeviceInformation selectedDevice;
+        IE::SoExecutableNetworkInternal executableNetwork;
+        while (!metaDevices.empty()) {
+            selectedDevice = SelectDevice(metaDevices, networkPrecision);
+            try {
+                executableNetwork = GetCore()->LoadNetwork(param, selectedDevice.deviceName, selectedDevice.config);
+                break;
+            } catch (...) {
+                auto eraseDevice = std::find_if(metaDevices.begin(), metaDevices.end(),
+                    [=](const DeviceInformation& d)->bool{return d.deviceName == selectedDevice.deviceName;});
+                if (eraseDevice == metaDevices.end()) {
+                    IE_THROW() << "Didn't find the selected device name";
+                }
+                metaDevices.erase(eraseDevice);
+                executableNetwork = {};
+            }
+        }
+        if (!executableNetwork) {
+            IE_THROW() << "Failed to load network by AUTO plugin";
+        }
+        return std::make_shared<AutoExecutableNetwork>(executableNetwork);
+    }
 };
 
 }  // namespace AutoPlugin