[IE CLDNN] Plugin-side kernels caching (#2871)

inference-engine/include/ie_plugin_config.hpp

@@ -362,5 +362,18 @@ DECLARE_CONFIG_KEY(DUMP_EXEC_GRAPH_AS_DOT);
  */
 DECLARE_CONFIG_KEY(ENFORCE_BF16);
 
+/**
+* @brief This key defines the directory which will be used to store any data cached by plugins.
+*
+* This key supports unicode symbols in path
+* The underlying cache structure is not defined and might differ between OpenVINO releases
+* Cached data might be platform/device specific and might be invalid after OpenVINO version change
+* If this key is not specified or value is empty string, then caching is disabled.
+* The key might enable caching for all plugin or some specific ones, e.g.:
+* ie.SetConfig({{CONFIG_KEY(CACHE_DIR), "cache/"}}) - enables cache for all plugins that might want to use it
+* ie.SetConfig({{CONFIG_KEY(CACHE_DIR), "cache/"}}, {"GPU"}) - enables cache only for GPU plugin
+*/
+DECLARE_CONFIG_KEY(CACHE_DIR);
+
 }  // namespace PluginConfigParams
 }  // namespace InferenceEngine

inference-engine/src/cldnn_engine/cldnn_config.cpp

@@ -8,16 +8,36 @@
 #include "cldnn_config.h"
 #include "cpp_interfaces/exception2status.hpp"
 #include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
+#include "ie_api.h"
+#include "file_utils.h"
 
 #ifdef _WIN32
 # include <direct.h>
+#ifdef ENABLE_UNICODE_PATH_SUPPORT
+# define mkdir(dir, mode) _wmkdir(dir)
+#else
 # define mkdir(dir, mode) _mkdir(dir)
-#endif
+#endif  // ENABLE_UNICODE_PATH_SUPPORT
+#endif  // _WIN32
 
 using namespace InferenceEngine;
 
 namespace CLDNNPlugin {
 
+static void createDirectory(std::string _path) {
+#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
+    std::wstring widepath = FileUtils::multiByteCharToWString(_path.c_str());
+    const wchar_t* path = widepath.c_str();
+#else
+    const char* path = _path.c_str();
+#endif
+
+    auto err = mkdir(path, 0755);
+    if (err != 0 && errno != EEXIST) {
+        THROW_IE_EXCEPTION << "Couldn't create directory! (err=" << err << "; errno=" << errno << ")";
+    }
+}
+
 void Config::UpdateFromMap(const std::map<std::string, std::string>& configMap) {
     for (auto& kvp : configMap) {
         std::string key = kvp.first;
@@ -129,16 +149,17 @@ void Config::UpdateFromMap(const std::map<std::string, std::string>& configMap)
         } else if (key.compare(CLDNNConfigParams::KEY_CLDNN_GRAPH_DUMPS_DIR) == 0) {
             if (!val.empty()) {
                 graph_dumps_dir = val;
-                if (mkdir(graph_dumps_dir.c_str(), 0755) != 0) {
-                    THROW_IE_EXCEPTION << "Couldn't create clDNN graph dump directory!";
-                }
+                createDirectory(graph_dumps_dir);
+            }
+        } else if (key.compare(PluginConfigParams::KEY_CACHE_DIR) == 0) {
+            if (!val.empty()) {
+                kernels_cache_dir = val;
+                createDirectory(kernels_cache_dir);
             }
         } else if (key.compare(CLDNNConfigParams::KEY_CLDNN_SOURCES_DUMPS_DIR) == 0) {
             if (!val.empty()) {
                 sources_dumps_dir = val;
-                if (mkdir(sources_dumps_dir.c_str(), 0755) != 0) {
-                    THROW_IE_EXCEPTION << "Couldn't create clDNN source dump directory!";
-                }
+                createDirectory(sources_dumps_dir);
             }
         } else if (key.compare(PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS) == 0) {
             if (val.compare(PluginConfigParams::YES) == 0) {
@@ -276,6 +297,7 @@ void Config::adjustKeyMapValues() {
 
     key_config_map[CLDNNConfigParams::KEY_CLDNN_GRAPH_DUMPS_DIR] = graph_dumps_dir;
     key_config_map[CLDNNConfigParams::KEY_CLDNN_SOURCES_DUMPS_DIR] = sources_dumps_dir;
+    key_config_map[PluginConfigParams::KEY_CACHE_DIR] = kernels_cache_dir;
 
     key_config_map[PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS] = std::to_string(throughput_streams);
     key_config_map[PluginConfigParams::KEY_DEVICE_ID] = device_id;

inference-engine/src/cldnn_engine/cldnn_config.h

@@ -35,7 +35,8 @@ struct Config {
                tuningConfig(),
                graph_dumps_dir(""),
                sources_dumps_dir(""),
-               device_id("") {
+               device_id(""),
+               kernels_cache_dir("") {
         adjustKeyMapValues();
     }
 
@@ -59,6 +60,7 @@ struct Config {
     std::string graph_dumps_dir;
     std::string sources_dumps_dir;
     std::string device_id;
+    std::string kernels_cache_dir;
 
     std::map<std::string, std::string> key_config_map;
 };

inference-engine/src/cldnn_engine/cldnn_engine.cpp

@@ -312,6 +312,7 @@ ExecutableNetworkInternal::Ptr clDNNEngine::LoadExeNetworkImpl(const InferenceEn
                context_config.sources_dumps_dir == current_config.sources_dumps_dir &&
                context_config.tuningConfig.mode == current_config.tuningConfig.mode &&
                context_config.tuningConfig.cache_file_path == current_config.tuningConfig.cache_file_path &&
+               context_config.kernels_cache_dir == current_config.kernels_cache_dir &&
                context_config.device_id == current_config.device_id;
     };
 

inference-engine/src/cldnn_engine/cldnn_remote_context.cpp

@@ -262,7 +262,8 @@ CLDNNExecutionContextImpl::CLDNNExecutionContextImpl(const std::shared_ptr<IInfe
             m_config.queuePriority,
             m_config.queueThrottle,
             m_config.memory_pool_on,
-            m_config.throughput_streams));
+            m_config.throughput_streams,
+            m_config.kernels_cache_dir));
 }
 
 ParamMap CLDNNExecutionContextImpl::getParams() const {

inference-engine/tests/functional/plugin/gpu/behavior/cache.cpp

@@ -0,0 +1,86 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "common_test_utils/test_common.hpp"
+#include "common_test_utils/file_utils.hpp"
+#include "common_test_utils/unicode_utils.hpp"
+#include "ngraph_functions/utils/ngraph_helpers.hpp"
+#include "functional_test_utils/plugin_cache.hpp"
+#include "ngraph_functions/subgraph_builders.hpp"
+#include <ie_core.hpp>
+#include <ie_plugin_config.hpp>
+
+class CompiledKernelsCacheTest : public CommonTestUtils::TestsCommon {
+protected:
+    std::string test_name = ::testing::UnitTest::GetInstance()->current_test_info()->name();
+    std::shared_ptr<ngraph::Function> function;
+    std::string cache_path;
+
+    void SetUp() override {
+        function = ngraph::builder::subgraph::makeConvPoolRelu();
+        cache_path = test_name + "_cache";
+    }
+};
+
+TEST_F(CompiledKernelsCacheTest, CanCreateCacheDirAndDumpBinaries) {
+    std::shared_ptr<InferenceEngine::Core> ie = PluginCache::get().ie();
+    // Create CNNNetwork from ngraph::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    std::map<std::string, std::string> config = {{ CONFIG_KEY(CACHE_DIR), cache_path }};
+    try {
+        // Load CNNNetwork to target plugins
+        auto execNet = ie->LoadNetwork(cnnNet, "GPU", config);
+
+        // Check that directory with cached kernels exists after loading network
+        ASSERT_TRUE(CommonTestUtils::directoryExists(cache_path)) << "Directory with cached kernels doesn't exist";
+        // Check that folder contains cache files and remove them
+        ASSERT_GT(CommonTestUtils::removeFilesWithExt(cache_path, "cl_cache"), 0);
+        // Remove directory and check that it doesn't exist anymore
+        ASSERT_EQ(CommonTestUtils::removeDir(cache_path), 0);
+        ASSERT_FALSE(CommonTestUtils::directoryExists(cache_path));
+    } catch (std::exception& ex) {
+        // Cleanup in case of any exception
+        if (CommonTestUtils::directoryExists(cache_path)) {
+            ASSERT_GE(CommonTestUtils::removeFilesWithExt(cache_path, "cl_cache"), 0);
+            ASSERT_EQ(CommonTestUtils::removeDir(cache_path), 0);
+        }
+        FAIL() << ex.what() << std::endl;
+    }
+}
+
+#ifdef ENABLE_UNICODE_PATH_SUPPORT
+
+TEST_F(CompiledKernelsCacheTest, CanCreateCacheDirAndDumpBinariesUnicodePath) {
+    std::shared_ptr<InferenceEngine::Core> ie = PluginCache::get().ie();
+    // Create CNNNetwork from ngraph::Function
+    InferenceEngine::CNNNetwork cnnNet(function);
+    for (std::size_t testIndex = 0; testIndex < CommonTestUtils::test_unicode_postfix_vector.size(); testIndex++) {
+        std::wstring postfix  = L"_" + CommonTestUtils::test_unicode_postfix_vector[testIndex];
+        std::wstring cache_path_w = CommonTestUtils::addUnicodePostfixToPath(cache_path, postfix);
+
+        try {
+            auto cache_path_mb = FileUtils::wStringtoMBCSstringChar(cache_path_w);
+            std::map<std::string, std::string> config = {{ CONFIG_KEY(CACHE_DIR), cache_path_mb }};
+            // Load CNNNetwork to target plugins
+            auto execNet = ie->LoadNetwork(cnnNet, "GPU", config);
+
+            // Check that directory with cached kernels exists after loading network
+            ASSERT_TRUE(CommonTestUtils::directoryExists(cache_path_w)) << "Directory with cached kernels doesn't exist";
+            // Check that folder contains cache files and remove them
+            ASSERT_GT(CommonTestUtils::removeFilesWithExt(cache_path_w, L"cl_cache"), 0);
+            // Remove directory and check that it doesn't exist anymore
+            ASSERT_EQ(CommonTestUtils::removeDir(cache_path_w), 0);
+            ASSERT_FALSE(CommonTestUtils::directoryExists(cache_path_w));
+        } catch (std::exception& ex) {
+            // Cleanup in case of any exception
+            if (CommonTestUtils::directoryExists(cache_path_w)) {
+                ASSERT_GE(CommonTestUtils::removeFilesWithExt(cache_path_w, L"cl_cache"), 0);
+                ASSERT_EQ(CommonTestUtils::removeDir(cache_path_w), 0);
+            }
+            FAIL() << ex.what() << std::endl;
+        }
+    }
+}
+
+#endif  // ENABLE_UNICODE_PATH_SUPPORT

inference-engine/tests/functional/plugin/gpu/shared_tests_instances/behavior/core_threading_tests.cpp

@@ -13,6 +13,7 @@ namespace {
 Params params[] = {
     std::tuple<Device, Config>{ CommonTestUtils::DEVICE_GPU, { { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) }}},
     std::tuple<Device, Config>{ CommonTestUtils::DEVICE_GPU, { { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(NO) }}},
+    std::tuple<Device, Config>{ CommonTestUtils::DEVICE_GPU, { { CONFIG_KEY(CACHE_DIR), "cache" }}},
 };
 
 }  // namespace

inference-engine/tests/ie_test_utils/common_test_utils/file_utils.hpp

@@ -6,8 +6,18 @@
 #include <fstream>
 #include <string>
 #include <vector>
+#include <sys/stat.h>
 
 #include "test_constants.hpp"
+#include "w_dirent.h"
+#include "common_utils.hpp"
+
+#ifdef _WIN32
+#include <direct.h>
+#define rmdir(dir) _rmdir(dir)
+#else  // _WIN32
+#include <unistd.h>
+#endif  // _WIN32
 
 namespace CommonTestUtils {
 
@@ -62,4 +72,47 @@ inline void removeIRFiles(const std::string &xmlFilePath, const std::string &bin
         std::remove(binFileName.c_str());
     }
 }
+
+// Removes all files with extension=ext from the given directory
+// Return value:
+// < 0 - error
+// >= 0 - count of removed files
+inline int removeFilesWithExt(std::string path, std::string ext) {
+    struct dirent *ent;
+    DIR *dir = opendir(path.c_str());
+    int ret = 0;
+    if (dir != nullptr) {
+        while ((ent = readdir(dir)) != NULL) {
+            auto file = makePath(path, std::string(ent->d_name));
+            struct stat stat_path;
+            stat(file.c_str(), &stat_path);
+            if (!S_ISDIR(stat_path.st_mode) && endsWith(file, "." + ext)) {
+                auto err = std::remove(file.c_str());
+                if (err != 0) {
+                    closedir(dir);
+                    return err;
+                }
+                ret++;
+            }
+        }
+        closedir(dir);
+    }
+
+    return ret;
+}
+
+inline int removeDir(const std::string &path) {
+    return rmdir(path.c_str());
+}
+
+inline bool directoryExists(const std::string &path) {
+    struct stat sb;
+
+    if (stat(path.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) {
+        return true;
+    }
+
+    return false;
+}
+
 }  // namespace CommonTestUtils

inference-engine/tests/ie_test_utils/common_test_utils/unicode_utils.hpp

@@ -10,6 +10,8 @@
 #include <algorithm>
 
 #include <file_utils.h>
+#include "common_utils.hpp"
+#include "w_dirent.h"
 
 #ifdef ENABLE_UNICODE_PATH_SUPPORT
 namespace CommonTestUtils {
@@ -71,6 +73,88 @@ static void removeFile(std::wstring path) {
     }
 }
 
+inline bool endsWith(const std::wstring& source, const std::wstring& expectedSuffix) {
+    return expectedSuffix.size() <= source.size() && source.compare(source.size() - expectedSuffix.size(), expectedSuffix.size(), expectedSuffix) == 0;
+}
+
+// Removes all files with extension=ext from the given directory
+// Return value:
+// < 0 - error
+// >= 0 - count of removed files
+inline int removeFilesWithExt(std::wstring path, std::wstring ext) {
+    int ret = 0;
+#ifdef _WIN32
+    struct _wdirent *ent;
+    _WDIR *dir = _wopendir(path.c_str());
+    if (dir != nullptr) {
+        while ((ent = _wreaddir(dir)) != NULL) {
+            auto file = ::FileUtils::makePath(path, std::wstring(ent->wd_name));
+            struct _stat64i32 stat_path;
+            _wstat(file.c_str(), &stat_path);
+            if (!S_ISDIR(stat_path.st_mode) && endsWith(file, L"." + ext)) {
+                auto err = _wremove(file.c_str());
+                if (err != 0) {
+                    _wclosedir(dir);
+                    return err;
+                }
+                ret++;
+            }
+        }
+        _wclosedir(dir);
+    }
+#else
+    struct dirent *ent;
+    auto path_mb = FileUtils::wStringtoMBCSstringChar(path);
+    auto ext_mb = FileUtils::wStringtoMBCSstringChar(ext);
+    DIR *dir = opendir(path_mb.c_str());
+    if (dir != nullptr) {
+        while ((ent = readdir(dir)) != NULL) {
+            std::string file = ::FileUtils::makePath(path_mb, std::string(ent->d_name));
+            struct stat stat_path;
+            stat(file.c_str(), &stat_path);
+            if (!S_ISDIR(stat_path.st_mode) && ::CommonTestUtils::endsWith(file, "." + ext_mb)) {
+                auto err = std::remove(file.c_str());
+                if (err != 0) {
+                    closedir(dir);
+                    return err;
+                }
+                ret++;
+            }
+        }
+        closedir(dir);
+    }
+#endif
+    return ret;
+}
+
+static int removeDir(std::wstring path) {
+    int result = 0;
+    if (!path.empty()) {
+#ifdef _WIN32
+        result = _wrmdir(path.c_str());
+#else
+        result = rmdir(FileUtils::wStringtoMBCSstringChar(path).c_str());
+#endif
+    }
+    return result;
+}
+
+inline bool directoryExists(const std::wstring &path) {
+#ifdef _WIN32
+    struct _stat64i32 sb;
+    if (_wstat(path.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) {
+        return true;
+    }
+#else
+    struct stat sb;
+    if (stat(FileUtils::wStringtoMBCSstringChar(path).c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) {
+        return true;
+    }
+#endif
+
+    return false;
+}
+
 static const std::vector<std::wstring> test_unicode_postfix_vector = {
         L"unicode_Яㅎあ",
         L"ひらがな日本語",
@@ -83,4 +167,4 @@ static const std::vector<std::wstring> test_unicode_postfix_vector = {
 };
 
 }  // namespace CommonTestUtils
-#endif  // ENABLE_UNICODE_PATH_SUPPORT
+#endif  // ENABLE_UNICODE_PATH_SUPPORT