Added support of MUSL Linux (openvinotoolkit#24428)

### Details: - Added support of MUSL Linux ### Tickets: - Closes openvinotoolkit#24368 --------- Co-authored-by: Jyothish Kumar M S <[email protected]>
ynimmaga · May 9, 2024 · 9edef1b · 9edef1b
1 parent f270b69
commit 9edef1b
Show file tree

Hide file tree

Showing 12 changed files with 156 additions and 30 deletions.
diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag
@@ -1 +1 @@
-pr-23064
+pr-24428
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -86,8 +86,8 @@ endif()
 if(CMAKE_TOOLCHAIN_FILE)
     message (STATUS "CMAKE_TOOLCHAIN_FILE .................. " ${CMAKE_TOOLCHAIN_FILE})
 endif()
-if(NOT OV_GLIBC_VERSION VERSION_EQUAL 0.0)
-    message (STATUS "GLIBC_VERSION ......................... " ${OV_GLIBC_VERSION})
+if(NOT OV_LIBC_VERSION VERSION_EQUAL 0.0)
+    message (STATUS "LIBC_VERSION .......................... " ${OV_LIBC_VERSION})
 endif()
 
 # remove file with exported targets to force its regeneration

diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake
@@ -18,7 +18,7 @@ if(THREADING STREQUAL "OMP")
                 VERSION_REGEX ".*_([a-z]*_([a-z0-9]+\\.)*[0-9]+).*"
                 SHA256 "62c68646747fb10f19b53217cb04a1e10ff93606f992e6b35eb8c31187c68fbf"
                 USE_NEW_LOCATION TRUE)
-    elseif(LINUX AND X86_64)
+    elseif(LINUX AND X86_64 AND OPENVINO_GNU_LIBC)
         RESOLVE_DEPENDENCY(OMP
                 ARCHIVE_LIN "iomp.tgz"
                 TARGET_PATH "${TEMP}/omp"
@@ -101,7 +101,7 @@ function(ov_download_tbb)
                 ENVIRONMENT "TBBROOT"
                 SHA256 "f42d084224cc2d643314bd483ad180b081774608844000f132859fca3e9bf0ce"
                 USE_NEW_LOCATION TRUE)
-    elseif(LINUX AND X86_64 AND OV_GLIBC_VERSION VERSION_GREATER_EQUAL 2.17)
+    elseif(LINUX AND X86_64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17)
         # build oneTBB 2021.2.1 with gcc 4.8 (glibc 2.17)
         RESOLVE_DEPENDENCY(TBB
                 ARCHIVE_LIN "oneapi-tbb-2021.2.4-lin.tgz"
@@ -132,7 +132,7 @@ function(ov_download_tbb)
                 ENVIRONMENT "TBBROOT"
                 SHA256 "09fe7f5e7be589aa34ccd20fdfd7cad9e0afa89d1e74ecdb008a75d0af71d6e1"
                 USE_NEW_LOCATION TRUE)
-    elseif(LINUX AND AARCH64 AND OV_GLIBC_VERSION VERSION_GREATER_EQUAL 2.17)
+    elseif(LINUX AND AARCH64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17)
         # build oneTBB 2021.2.1 with gcc 4.8 (glibc 2.17)
         RESOLVE_DEPENDENCY(TBB
                 ARCHIVE_LIN "oneapi-tbb-2021.2.1-lin-arm64-20231012.tgz"
@@ -203,7 +203,7 @@ function(ov_download_tbbbind_2_5)
                 ENVIRONMENT "TBBBIND_2_5_ROOT"
                 SHA256 "49ae93b13a13953842ff9ae8d01681b269b5b0bc205daf18619ea9a828c44bee"
                 USE_NEW_LOCATION TRUE)
-    elseif(LINUX AND X86_64)
+    elseif(LINUX AND X86_64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17)
         RESOLVE_DEPENDENCY(TBBBIND_2_5
                 ARCHIVE_LIN "tbbbind_2_5_static_lin_v4.tgz"
                 TARGET_PATH "${TEMP}/tbbbind_2_5"

diff --git a/cmake/developer_package/target_flags.cmake b/cmake/developer_package/target_flags.cmake
@@ -117,6 +117,42 @@ endif()
 
 get_property(OV_GENERATOR_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 
+function(ov_detect_libc_type)
+    include(CheckCXXSourceCompiles)
+    check_cxx_source_compiles("
+# ifndef _GNU_SOURCE
+#   define _GNU_SOURCE
+#   include <features.h>
+#   ifndef __USE_GNU
+#     define CMAKE_OPENVINO_MUSL_LIBC
+#   endif
+#   undef _GNU_SOURCE /* don't contaminate other includes unnecessarily */
+# else
+#   include <features.h>
+#   ifndef __USE_GNU
+#     define CMAKE_OPENVINO_MUSL_LIBC
+#   endif
+# endif
+# ifndef CMAKE_OPENVINO_MUSL_LIBC
+#   error \"OpenVINO GNU LIBC\"
+# endif
+
+int main() {
+  return 0;
+}"
+    OPENVINO_MUSL_LIBC)
+
+    if(OPENVINO_MUSL_LIBC)
+        set(OPENVINO_MUSL_LIBC ON PARENT_SCOPE)
+    else()
+        set(OPENVINO_GNU_LIBC ON PARENT_SCOPE)
+    endif()
+endfunction()
+
+if(LINUX)
+  ov_detect_libc_type()
+endif()
+
 function(ov_get_compiler_definition definition var)
     if(NOT LINUX)
         message(FATAL_ERROR "Internal error: 'ov_get_definition' must be used only on Linux")
@@ -152,19 +188,26 @@ function(ov_get_compiler_definition definition var)
     endif()
 endfunction()
 
-function(ov_glibc_version)
+function(ov_libc_version)
     # cmake needs to look at glibc version only when we build for Linux on Linux
     if(LINUX)
-        ov_get_compiler_definition("__GLIBC__" _ov_glibc_major)
-        ov_get_compiler_definition("__GLIBC_MINOR__" _ov_glibc_minor)
-
-        set(OV_GLIBC_VERSION "${_ov_glibc_major}.${_ov_glibc_minor}" PARENT_SCOPE)
+        if(OPENVINO_GNU_LIBC)
+            ov_get_compiler_definition("__GLIBC__" _ov_glibc_major)
+            ov_get_compiler_definition("__GLIBC_MINOR__" _ov_glibc_minor)
+
+            set(OV_LIBC_VERSION "${_ov_glibc_major}.${_ov_glibc_minor}" PARENT_SCOPE)
+        elseif(OPENVINO_MUSL_LIBC)
+            # TODO: implement proper detection
+            set(OV_LIBC_VERSION "1.1" PARENT_SCOPE)
+        else()
+            message(FATAL_ERROR "Undefined libc type")
+        endif()
     else()
-        set(OV_GLIBC_VERSION "0.0" PARENT_SCOPE)
+        set(OV_LIBC_VERSION "0.0" PARENT_SCOPE)
     endif()
 endfunction()
 
-ov_glibc_version()
+ov_libc_version()
 
 #
 # Detects default value for _GLIBCXX_USE_CXX11_ABI for current compiler

diff --git a/install_build_dependencies.sh b/install_build_dependencies.sh
@@ -177,7 +177,7 @@ elif [ -f /etc/os-release ] && grep -q "raspbian" /etc/os-release; then
         scons \
         `# to find dependencies` \
         pkg-config \
-        `# to deternime product version via git` \
+        `# to determine product version via git` \
         git \
         `# to build and check pip packages` \
         patchelf \
@@ -192,6 +192,58 @@ elif [ -f /etc/os-release ] && grep -q "raspbian" /etc/os-release; then
         python3-venv \
         python3-setuptools \
         libpython3-dev
+elif [ -f /etc/os-release ] && grep -q "void" /etc/os-release; then
+    #Void Linux
+    xbps-install -Syu
+    xbps-install -y \
+        `# for python3-pip` \
+        `# ca-certificates (already included)` \
+        file \
+        `# build tools` \
+        base-devel \
+        ninja \
+        scons \
+        ccache \
+        cmake \
+        `# to find dependencies` \
+        pkgconf \
+        `# to determine product version via git` \
+        git \
+        `# to check bash scripts for correctness` \
+        shellcheck \
+        `# to build and check pip packages` \
+        patchelf \
+        fdupes \
+        `# main openvino dependencies` \
+        tbb-devel \
+        pugixml-devel \
+        `# OpenCL for GPU` \
+        ocl-icd-devel \
+        OpenCL-Headers \
+        OpenCL-CLHPP \
+        rapidjson \
+        `# GPU plugin dependency` \
+        libva-devel \
+        `# For TF FE saved models` \
+        snappy-devel \
+        `# For Python API` \
+        python3-pip \
+        python3-wheel \
+        python3-setuptools \
+        python3-devel \
+        python3-pybind11 \
+        libffi-devel \
+        `# Spell checking for MO sources` \
+        python3-enchant \
+        `# tools` \
+        wget \
+        git-lfs \
+        `# TF Lite Frontend` \
+        flatbuffers-devel \
+        `# for python3-enchant` \
+        enchant2-devel \
+        `# samples` \
+        json-c++
 else
     echo "Unknown OS, please install build dependencies manually"
 fi

diff --git a/src/bindings/python/wheel/CMakeLists.txt b/src/bindings/python/wheel/CMakeLists.txt
@@ -58,19 +58,27 @@ if(APPLE)
 elseif(LINUX)
     _ov_platform_arch()
 
-    string(REPLACE "." "_" _ov_glibc_version "${OV_GLIBC_VERSION}")
-    set(manylinux "manylinux_${_ov_glibc_version}")
+    if(OPENVINO_GNU_LIBC)
+        set(platform "manylinux")
+    elseif(OPENVINO_MUSL_LIBC)
+        set(platform "musllinux")
+    else()
+        message(FATAL_ERROR "Undefined libc type")
+    endif()
+
+    string(REPLACE "." "_" _ov_libc_version "${OV_LIBC_VERSION}")
+    set(platform "${platform}_${_ov_libc_version}")
 
     # convert to well-known formats according to PEP 600
-    if(manylinux STREQUAL "manylinux_2_5")
-        set(manylinux "manylinux1")
-    elseif(manylinux STREQUAL "manylinux_2_12")
-        set(manylinux "manylinux2010")
-    elseif(manylinux STREQUAL "manylinux_2_17")
-        set(manylinux "manylinux2014")
+    if(platform STREQUAL "manylinux_2_5")
+        set(platform "manylinux1")
+    elseif(platform STREQUAL "manylinux_2_12")
+        set(platform "manylinux2010")
+    elseif(platform STREQUAL "manylinux_2_17")
+        set(platform "manylinux2014")
     endif()
 
-    set(PLATFORM_TAG "${manylinux}_${_arch}")
+    set(PLATFORM_TAG "${platform}_${_arch}")
 endif()
 
 set(openvino_wheel_name "openvino-${WHEEL_VERSION}-${WHEEL_BUILD}-${PYTHON_TAG}-${ABI_TAG}-${PLATFORM_TAG}.whl")

diff --git a/src/cmake/install_tbb.cmake b/src/cmake/install_tbb.cmake
@@ -91,7 +91,7 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND
         set(tbb_custom ON)
     endif()
 
-    if(OV_GLIBC_VERSION VERSION_LESS_EQUAL 2.26)
+    if(OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_LESS_EQUAL 2.26)
         set(_ov_system_tbb_is_obsolete ON)
     endif()
 

diff --git a/src/common/util/include/openvino/util/pp.hpp b/src/common/util/include/openvino/util/pp.hpp
@@ -63,3 +63,23 @@ __cplusplus value compatibility issues.*/
 #else
 #    define OV_CAPTURE_CPY_AND_THIS =
 #endif /* C++20 */
+
+#ifdef __linux__
+#    ifndef _GNU_SOURCE
+#        define _GNU_SOURCE
+#        include <features.h>
+#        ifndef __USE_GNU
+#            define OPENVINO_MUSL_LIBC
+#        endif
+#        undef _GNU_SOURCE /* don't contaminate other includes unnecessarily */
+#    else
+#        include <features.h>
+#        ifndef __USE_GNU
+#            define OPENVINO_MUSL_LIBC
+#        endif
+#    endif
+
+#    ifndef OPENVINO_MUSL_LIBC
+#        define OPENVINO_GNU_LIBC
+#    endif
+#endif
diff --git a/src/common/util/src/file_util.cpp b/src/common/util/src/file_util.cpp
@@ -40,6 +40,7 @@
 #    include <dirent.h>
 #    include <dlfcn.h>
 #    include <ftw.h>
+#    include <limits.h>
 #    include <sys/file.h>
 #    include <sys/time.h>
 #    include <unistd.h>

diff --git a/src/plugins/intel_gpu/src/plugin/program_builder.cpp b/src/plugins/intel_gpu/src/plugin/program_builder.cpp
@@ -16,6 +16,7 @@
 #include "intel_gpu/primitives/data.hpp"
 #include "intel_gpu/op/fully_connected_compressed.hpp"
 #include "intel_gpu/op/placeholder.hpp"
+#include "openvino/util/pp.hpp"
 
 #ifdef __linux__
 # include <dlfcn.h>
@@ -132,14 +133,14 @@ void ProgramBuilder::prepare_build() {
 
 void ProgramBuilder::cleanup_build() {
     m_topology.reset();
-    #if defined(__unix__) && !defined(__ANDROID__)
+#if defined(OPENVINO_GNU_LIBC) && !defined(__ANDROID__)
     //  NOTE: In linux, without malloc_trim, an amount of the memory used by compilation is not being returned to system thought they are freed.
     //  (It is at least 500 MB when we perform parallel compilation)
     //  It is observed that freeing the memory manually with malloc_trim saves significant amount of the memory.
     //  Also, this is not happening in Windows.
     //  So, added malloc_trim for linux build until we figure out a better solution.
     malloc_trim(0);
-    #endif
+#endif
 }
 
 std::shared_ptr<cldnn::program> ProgramBuilder::build(const std::vector<std::shared_ptr<ov::Node>>& ops, bool partial_build, bool is_inner_program) {

diff --git a/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp b/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp
@@ -14,6 +14,7 @@
 #include "intel_gpu/runtime/debug_configuration.hpp"
 #include "intel_gpu/runtime/itt.hpp"
 #include "intel_gpu/runtime/file_util.hpp"
+#include "openvino/util/pp.hpp"
 
 #ifdef WIN32
 #include <sdkddkver.h>
@@ -453,7 +454,7 @@ void kernels_cache::build_all() {
         std::lock_guard<std::mutex> lock(_mutex);
         _kernels_code.clear();
         _pending_compilation = false;
-#if defined(__unix__) && !defined(__ANDROID__)
+#if defined(OPENVINO_GNU_LIBC) && !defined(__ANDROID__)
     //  NOTE: In linux, without malloc_trim, an amount of the memory used by compilation is not being returned to system thought they are freed.
     //  (It is at least 500 MB when we perform parallel compilation)
     //  It is observed that freeing the memory manually with malloc_trim saves significant amount of the memory.
@@ -616,7 +617,7 @@ kernels_cache::compiled_kernels kernels_cache::compile(const kernel_impl_params&
     OPENVINO_ASSERT(output_kernels.size() == 1, "Only the kernels of the single primitive should be compiled.");
 
     t_kernels_code.clear();
-#if defined(__unix__) && !defined(__ANDROID__)
+#if defined(OPENVINO_GNU_LIBC) && !defined(__ANDROID__)
     //  NOTE: In linux, without malloc_trim, an amount of the memory used by compilation is not being returned to system thought they are freed.
     //  (It is at least 500 MB when we perform parallel compilation)
     //  It is observed that freeing the memory manually with malloc_trim saves significant amount of the memory.

diff --git a/src/tests/test_utils/common_test_utils/include/common_test_utils/test_tools.hpp b/src/tests/test_utils/common_test_utils/include/common_test_utils/test_tools.hpp
@@ -37,7 +37,7 @@ std::vector<T> read_binary_file(const std::string& path) {
     return file_content;
 }
 
-template <typename T = int32_t>
+template <typename T = int>
 std::vector<T> gen_range(const size_t elements, const T start = T{0}) {
     std::vector<T> range;
     range.resize(elements);