Refactor cmake, added C++17 as required

CMakeLists.txt

@@ -14,25 +14,4 @@ find_package(OpenVINO REQUIRED COMPONENTS Runtime)
 set(CMAKE_PROJECT_VERSION_TWEAK 0)
 set(CMAKE_PROJECT_VERSION "${OpenVINO_VERSION}.${CMAKE_PROJECT_VERSION_TWEAK}")
 
-include(GNUInstallDirs)
-
-# setting RPATH / LC_RPATH depending on platform
-if(LINUX)
-  # to find libcore_tokenizer.so in the same folder
-  set(rpaths "$ORIGIN")
-elseif(APPLE)
-  # to find libcore_tokenizer.dylib in the same folder
-  set(rpaths "@loader_path")
-  if(DEFINED SKBUILD)
-    # in case we build pip package, we need to refer to libopenvino.dylib from 'openvino' package
-    list(APPEND rpaths "@loader_path/../../openvino/libs")
-  endif()
-endif()
-
-if(NOT CMAKE_CXX_STANDARD)
-  set(CMAKE_CXX_STANDARD 11)
-endif()
-
 add_subdirectory(src)
-
-

src/CMakeLists.txt

@@ -4,15 +4,24 @@
 
 set(TARGET_NAME "openvino_tokenizers")
 
-FILE(GLOB SRC *.cpp)
-add_library(${TARGET_NAME} SHARED ${SRC})
-target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API)
-target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime)
-
 if(POLICY CMP0135)
   cmake_policy(SET CMP0135 NEW)
 endif()
 
+function(ov_tokenizers_set_cxx_standard)
+  foreach(build_type "" "_DEBUG" "_MINSIZEREL" "_RELEASE" "_RELWITHDEBINFO")
+    set(flag_var "CMAKE_CXX_FLAGS${build_type}")
+    string(REPLACE "--std=c++11" "" ${flag_var} "${${flag_var}}")
+    set(${flag_var} "${${flag_var}}" PARENT_SCOPE)
+  endforeach()
+
+  set(CMAKE_CXX_STANDARD 17 PARENT_SCOPE)
+  set(CMAKE_CXX_EXTENSIONS OFF PARENT_SCOPE)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON PARENT_SCOPE)
+endfunction()
+
+ov_tokenizers_set_cxx_standard()
+
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 option(BUILD_FAST_TOKENIZERS OFF)
@@ -28,7 +37,7 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
     # C4244: 'argument' : conversion from 'type1' to 'type2', possible loss of data
     # C4267: 'var' : conversion from 'size_t' to 'type', possible loss of data
     # C4700: uninitialized local variable 'var' used
-    set(c_cxx_flags "/wd4146 /wd4244 /wd4267 /wd4700")
+    set(c_cxx_flags "/wd4146 /wd4244 /wd4267 /wd4700 /wd4703")
 endif()
 
 include(CheckCXXCompilerFlag)
@@ -37,11 +46,6 @@ if(SUGGEST_OVERRIDE_SUPPORTED)
     set(cxx_flags "${cxx_flags} -Wno-suggest-override")
 endif()
 
-if(WIN32 AND X86_64)
-  # disable compiler warning C4703
-  set(cxx_flags "${cxx_flags} /wd4703")
-endif()
-
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${cxx_flags} ${c_cxx_flags}")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${c_cxx_flags}")
 
@@ -148,6 +152,8 @@ else()
   set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER)
   include("${fast_tokenizer_SOURCE_DIR}/FastTokenizer.cmake")
   set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ${_old_CMAKE_FIND_ROOT_PATH_MODE_LIBRARY})
+  # since FastTokenizers.cmake overrides C++ standard, let's override it once again to required one
+  ov_tokenizers_set_cxx_standard()
 
   if(WIN32 AND X86_64)
       # we use re2 library in regex_normalization operation, so have to add to this list
@@ -156,6 +162,16 @@ else()
   endif()
 endif()
 
+#
+# Build library
+#
+
+file(GLOB SRC *.cpp)
+add_library(${TARGET_NAME} SHARED ${SRC})
+
+target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API)
+target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime)
+
 #
 # Target include dirs, link libraries and other properties
 #
@@ -177,21 +193,14 @@ endif()
 
 target_link_libraries(${TARGET_NAME} PRIVATE ${FAST_TOKENIZER_LIBS} sentencepiece-static)
 
-# string_view is used from cxx17
-set_target_properties(${TARGET_NAME} PROPERTIES CXX_STANDARD 17)
-
 string(REPLACE " " ";" extra_flags "${c_cxx_flags} ${cxx_flags}")
 set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_OPTIONS "${extra_flags}")
 
 #
 # Post build steps to copy core_tokenizers dependencies
 #
 
-if(BUILD_FAST_TOKENIZERS)
-  install(TARGETS core_tokenizers
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
-else()
+if(NOT BUILD_FAST_TOKENIZERS)
   if(WIN32 AND X86_64)
     set(extra_libs "${fast_tokenizer_SOURCE_DIR}/lib/core_tokenizers.dll"
                    "${fast_tokenizer_SOURCE_DIR}/third_party/lib/icudt70.dll"
@@ -201,32 +210,61 @@ else()
   elseif(APPLE)
     set(extra_libs "${fast_tokenizer_SOURCE_DIR}/lib/libcore_tokenizers.dylib")
   endif()
-endif()
 
-# post build steps
-if(extra_libs)
   # post build steps
   add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy ${extra_libs} $<TARGET_FILE_DIR:${TARGET_NAME}>)
+endif()
 
-  if(WIN32)
-    set(extra_libs_location ${CMAKE_INSTALL_BINDIR})
-  else()
-    set(extra_libs_location ${CMAKE_INSTALL_LIBDIR})
+#
+# Set install RPATH
+#
+
+# setting RPATH / LC_RPATH depending on platform
+if(LINUX)
+  # to find libcore_tokenizer.so in the same folder
+  set(rpaths "$ORIGIN")
+elseif(APPLE)
+  # to find libcore_tokenizer.dylib in the same folder
+  set(rpaths "@loader_path")
+  if(DEFINED SKBUILD)
+    # in case we build pip package, we need to refer to libopenvino.dylib from 'openvino' package
+    list(APPEND rpaths "@loader_path/../../openvino/libs")
   endif()
-  install(FILES ${extra_libs} DESTINATION ${extra_libs_location})
 endif()
 
 if(rpaths)
   set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}")
 endif()
 
+#
+# Installation
+#
+
+include(GNUInstallDirs)
+
 # Installing the extension module to the root of the package
 install(TARGETS ${TARGET_NAME}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 
-# cpack configuration
+if(BUILD_FAST_TOKENIZERS)
+  install(TARGETS core_tokenizers
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+else()
+  if(WIN32)
+    set(extra_libs_location ${CMAKE_INSTALL_BINDIR})
+  else()
+    set(extra_libs_location ${CMAKE_INSTALL_LIBDIR})
+  endif()
+  install(FILES ${extra_libs} DESTINATION ${extra_libs_location})
+endif()
+
+#
+# Cpack configuration
+#
+
 set(CPACK_PACKAGE_NAME ${TARGET_NAME})
 set(CPACK_PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}")
 set(CPACK_SOURCE_GENERATOR "") # not used