Optimizations for precision conversion operations in nGraph reference…

… implementations (#3974)

* FP16->FP32 conversion vectorization

* int8->FP16 conversion vectorization in nGraph

.gitmodules

@@ -13,4 +13,8 @@
 [submodule "inference-engine/samples/thirdparty/gflags"]
 	path = inference-engine/samples/thirdparty/gflags
 	url = https://github.com/gflags/gflags.git
-	ignore = dirty
+	ignore = dirty
+[submodule "thirdparty/xbyak"]
+	path = thirdparty/xbyak
+	url = https://github.com/herumi/xbyak.git
+	ignore = dirty

CMakeLists.txt

@@ -108,6 +108,7 @@ function(build_ngraph)
     set(NGRAPH_COMPONENT_PREFIX "deployment_tools/ngraph/")
     add_subdirectory(ngraph)
     set(NGRAPH_LIBRARIES ngraph PARENT_SCOPE)
+    set(NGRAPH_REF_LIBRARIES ngraph_reference PARENT_SCOPE)
 endfunction()
 
 function(openvino_developer_export_targets)
@@ -156,6 +157,7 @@ add_subdirectory(docs)
 ie_shellcheck_process(DIRECTORY "${OpenVINO_MAIN_SOURCE_DIR}"
                       SKIP "${OpenVINO_MAIN_SOURCE_DIR}/bin"
                            "${OpenVINO_MAIN_SOURCE_DIR}/build"
+                           "${OpenVINO_MAIN_SOURCE_DIR}/thirdparty"
                            "${IE_MAIN_SOURCE_DIR}/tests/ie_test_utils/common_test_utils/gtest"
                            "${IE_MAIN_SOURCE_DIR}/samples/thirdparty"
                            "${IE_MAIN_SOURCE_DIR}/thirdparty"

inference-engine/src/inference_engine/CMakeLists.txt

@@ -109,7 +109,8 @@ target_compile_definitions(${TARGET_NAME}_obj PRIVATE IMPLEMENT_INFERENCE_ENGINE
                                                       $<TARGET_PROPERTY:ngraph::ngraph,INTERFACE_COMPILE_DEFINITIONS>)
 
 target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE $<TARGET_PROPERTY:ngraph::ngraph,INTERFACE_INCLUDE_DIRECTORIES>
-                                                             $<TARGET_PROPERTY:pugixml,INTERFACE_INCLUDE_DIRECTORIES>)
+                                                             $<TARGET_PROPERTY:pugixml,INTERFACE_INCLUDE_DIRECTORIES>
+                                                             $<TARGET_PROPERTY:xbyak,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(${TARGET_NAME}_obj PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}"
                                                       "${IE_MAIN_SOURCE_DIR}/src/readers/ir_reader" # for ie_ir_version.hpp
@@ -119,10 +120,6 @@ target_include_directories(${TARGET_NAME}_obj PRIVATE "${CMAKE_CURRENT_SOURCE_DI
 
 target_link_libraries(${TARGET_NAME}_obj PRIVATE ${TARGET_NAME}_reader_api)
 
-if(ENABLE_MKL_DNN)
-    target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE "${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/cpu/x64/xbyak")
-endif()
-
 set_ie_threading_interface_for(${TARGET_NAME}_obj)
 
 add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}_obj)
@@ -215,10 +212,6 @@ configure_file("${IE_MAIN_SOURCE_DIR}/cmake/templates/InferenceEngineConfig-vers
 
 # Export for developer package
 
-add_library(xbyak INTERFACE)
-target_include_directories(xbyak INTERFACE ${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/cpu/xbyak)
-
-openvino_developer_export_targets(COMPONENT openvino_common TARGETS xbyak)
 ie_developer_export_targets(${TARGET_NAME} ${TARGET_NAME}_plugin_api)
 
 # install TBB

inference-engine/src/inference_engine/ie_system_conf.cpp

@@ -9,89 +9,41 @@
 #include <iostream>
 #include <vector>
 
-#ifdef ENABLE_MKL_DNN
 # define XBYAK_NO_OP_NAMES
 # define XBYAK_UNDEF_JNL
-# include <xbyak_util.h>
-#endif
+# include <xbyak/xbyak_util.h>
 
 namespace InferenceEngine {
 
-#ifdef ENABLE_MKL_DNN
 static Xbyak::util::Cpu& get_cpu_info() {
     static Xbyak::util::Cpu cpu;
     return cpu;
 }
-#endif
 
 bool with_cpu_x86_sse42() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tSSE42);
-#else
-#if defined(HAVE_SSE)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX);
-#else
-#if defined(HAVE_AVX)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx2() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX2);
-#else
-#if defined(HAVE_AVX2)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx512f() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX512F);
-#else
-#if defined(HAVE_AVX512)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx512_core() {
-#ifdef ENABLE_MKL_DNN
        return get_cpu_info().has(Xbyak::util::Cpu::tAVX512F |
                                  Xbyak::util::Cpu::tAVX512DQ |
                                  Xbyak::util::Cpu::tAVX512BW);
-#else
-#if defined(HAVE_AVX512)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_bfloat16() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX512_BF16);
-#else
-    return false;
-#endif
 }
 
 bool checkOpenMpEnvVars(bool includeOMPNumThreads) {

inference-engine/src/transformations/CMakeLists.txt

@@ -28,7 +28,7 @@ ie_add_vs_version_file(NAME ${TARGET_NAME}
                        FILEDESCRIPTION "Inference Engine Transformations library")
 
 target_link_libraries(${TARGET_NAME} PUBLIC ${NGRAPH_LIBRARIES}
-                                     PRIVATE openvino::conditional_compilation openvino::itt ngraph::builder pugixml)
+                                     PRIVATE ${NGRAPH_REF_LIBRARIES} openvino::conditional_compilation openvino::itt ngraph::builder pugixml)
 
 target_include_directories(${TARGET_NAME} PUBLIC ${PUBLIC_HEADERS_DIR}
                                           PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")

inference-engine/src/transformations/src/transformations/convert_precision.cpp

@@ -14,6 +14,8 @@
 #include <ngraph/opsets/opset1.hpp>
 #include <ngraph_ops/type_relaxed.hpp>
 
+#include <ngraph/runtime/reference/convert.hpp>
+
 using namespace ngraph;
 
 bool fuse_type_to_constant(std::shared_ptr<ngraph::Node> & node, ngraph::element::Type to, const std::vector<ngraph::Input<ngraph::Node>> & consumers);
@@ -341,25 +343,44 @@ inline int32_t convert_value<uint32_t, int32_t>(uint32_t val) {
     return static_cast<int32_t>(val);
 }
 
-template <element::Type_t PREC_FROM, element::Type_t PREC_TO>
-static std::shared_ptr<Node> change_constant_precision(std::shared_ptr<opset4::Constant>& constant) {
-    using src_type = typename element_type_traits<PREC_FROM>::value_type;
-    using dst_type = typename element_type_traits<PREC_TO>::value_type;
+namespace {
+    template <element::Type_t PREC_FROM, element::Type_t PREC_TO>
+    std::shared_ptr<Node> change_constant_precision(std::shared_ptr<opset4::Constant>& constant) {
+        using src_type = typename element_type_traits<PREC_FROM>::value_type;
+        using dst_type = typename element_type_traits<PREC_TO>::value_type;
 
-    const auto * src_data = constant->get_data_ptr<src_type>();
-    const auto size = shape_size(constant->get_shape());
+        const auto * src_data = constant->get_data_ptr<src_type>();
+        const auto size = shape_size(constant->get_shape());
 
-    auto new_constant = std::make_shared<ngraph::opset4::Constant>(PREC_TO, constant->get_shape());
-    auto * dst_data = const_cast<dst_type *>(reinterpret_cast<const dst_type *>(new_constant->get_data_ptr()));
-    if (dst_data == nullptr)
-        throw ngraph_error("Can't get destination data pointer");
+        auto new_constant = std::make_shared<ngraph::opset4::Constant>(PREC_TO, constant->get_shape());
+        auto * dst_data = const_cast<dst_type *>(reinterpret_cast<const dst_type *>(new_constant->get_data_ptr()));
+        if (dst_data == nullptr)
+            throw ngraph_error("Can't get destination data pointer");
 
-    std::vector<dst_type> final_data;
-    for (size_t i = 0; i < size; ++i) {
-        dst_data[i] = convert_value<src_type, dst_type>(src_data[i]);
+        for (size_t i = 0; i < size; ++i) {
+            dst_data[i] = convert_value<src_type, dst_type>(src_data[i]);
+        }
+        return new_constant;
     }
-    return new_constant;
-}
+
+    template <>
+    std::shared_ptr<Node> change_constant_precision<element::Type_t::f16, element::Type_t::f32>(std::shared_ptr<opset4::Constant>& constant) {
+        using src_type = typename element_type_traits<element::Type_t::f16>::value_type;
+        using dst_type = typename element_type_traits<element::Type_t::f32>::value_type;
+
+        const auto * src_data = constant->get_data_ptr<src_type>();
+        const auto size = shape_size(constant->get_shape());
+
+        auto new_constant = std::make_shared<ngraph::opset4::Constant>(element::Type_t::f32, constant->get_shape());
+        auto * dst_data = const_cast<dst_type *>(reinterpret_cast<const dst_type *>(new_constant->get_data_ptr()));
+        if (dst_data == nullptr)
+            throw ngraph_error("Can't get destination data pointer");
+
+        ngraph::runtime::reference::convert<src_type, dst_type>(src_data, dst_data, size);
+
+        return new_constant;
+    }
+}   // namespace
 
 bool fuse_type_to_constant(std::shared_ptr<Node> & node, element::Type to, const std::vector<Input<Node>> & consumers) {
     if (auto constant = as_type_ptr<opset4::Constant>(node)) {

ngraph/core/reference/CMakeLists.txt

@@ -37,9 +37,13 @@ if(COMMAND ie_faster_build)
     )
 endif()
 
+target_compile_definitions(${TARGET_NAME} PRIVATE XBYAK_NO_OP_NAMES XBYAK64)
+
 # Defines macro in C++ to load backend plugin
 target_include_directories(${TARGET_NAME} PUBLIC ${REF_IMPL_INCLUDE_DIR} ${NGRAPH_INCLUDE_PATH})
 
+target_link_libraries(${TARGET_NAME} PRIVATE xbyak)
+
 # Add an alias so that library can be used inside the build tree, e.g. when testing
 add_library(ngraph::reference ALIAS ${TARGET_NAME})
 

ngraph/core/reference/include/ngraph/runtime/reference/convert.hpp

@@ -18,6 +18,8 @@
 
 #include <cstddef>
 
+#include "ngraph/type/float16.hpp"
+
 namespace ngraph
 {
     namespace runtime
@@ -34,6 +36,11 @@ namespace ngraph
                 }
             }
 
+            template <>
+            void convert<uint8_t, float16>(const uint8_t* arg, float16* out, size_t count);
+            template <>
+            void convert<float16, float>(const float16* arg, float* out, size_t count);
+
             template <typename TI, typename TO>
             typename std::enable_if<std::is_same<TO, char>::value>::type
                 convert(const TI* arg, TO* out, size_t count)