Merge branch 'master' into del-tf-translators

.github/workflows/linux.yml

@@ -289,11 +289,6 @@ jobs:
             python3 -m pip install $wheel_name[dev]
           popd
 
-      - name: Tokenizers Bandit tests
-        run: |
-          bandit -c pyproject.toml -r python
-        working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}
-
       - name: Tokenizers regression tests
         run: |
           python3 -m pytest tokenizers_test.py

.github/workflows/mac.yml

@@ -14,6 +14,7 @@ concurrency:
 
 env:
   PYTHON_VERSION: '3.11'
+  MACOSX_DEPLOYMENT_TARGET: '10.12'
 
 jobs:
   openvino_build:

.github/workflows/sdl.yml

@@ -0,0 +1,55 @@
+name: OpenVINO tokenizers sdl tests
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - master
+      - 'releases/**'
+
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+
+concurrency:
+  # github.ref is not unique in post-commit
+  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-sdl
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  sdl_tests:
+    name: SDL tests
+    timeout-minutes: 5
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone Openvino tokenizers sources and tests
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - name: Install Python tests dependencies
+        run: |
+          python3 -m pip install bandit
+
+      - name: Tokenizers Bandit tests
+        run: |
+          bandit -c pyproject.toml -r python
+
+      - name: Run Trivy vulnerability scanner in fs mode
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+

.github/workflows/windows.yml

@@ -32,6 +32,9 @@ jobs:
       BUILD_DIR: ${{ github.workspace }}\\openvino\\build
 
     steps:
+      - name: git configuration
+        run: git config --system core.longpaths true
+
       - name: Clone OpenVINO
         uses: actions/checkout@v4
         with:

CMakeLists.txt

@@ -8,12 +8,7 @@ project(openvino_tokenizers)
 
 include(cmake/platforms.cmake)
 
-# Find OpenVINODeveloperPackage first to compile with SDL flags
-find_package(OpenVINODeveloperPackage QUIET
-             PATHS "${OpenVINO_DIR}")
-if(NOT OpenVINODeveloperPackage_FOUND)
-    find_package(OpenVINO REQUIRED COMPONENTS Runtime)
-endif()
+find_package(OpenVINO REQUIRED COMPONENTS Runtime)
 
 # set the project version using OpenVINO major, minor and patch versions
 set(CMAKE_PROJECT_VERSION_TWEAK 0)

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -2,6 +2,7 @@
 # Copyright (C) 2018-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import annotations
 import logging
 import weakref
 from dataclasses import dataclass, field

src/CMakeLists.txt

@@ -9,20 +9,6 @@ add_library(${TARGET_NAME} SHARED ${SRC})
 target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API)
 target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime)
 
-if(rpaths)
-  set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}")
-endif()
-
-# Installing the extension module to the root of the package
-install(TARGETS ${TARGET_NAME}
-        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
-
-# cpack configuration
-set(CPACK_PACKAGE_NAME ${TARGET_NAME})
-set(CPACK_PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}")
-set(CPACK_SOURCE_GENERATOR "") # not used
-
 if(POLICY CMP0135)
   cmake_policy(SET CMP0135 NEW)
 endif()
@@ -74,7 +60,12 @@ FetchContent_Declare(
   URL      https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.tar.gz
   URL_HASH SHA256=63617eaf56c7a3857597dcd8780461f57dd21381b56a27716ef7d7e02e14ced4
 )
-FetchContent_MakeAvailable(sentencepiece)
+FetchContent_GetProperties(sentencepiece)
+if(NOT sentencepiece_POPULATED)
+  FetchContent_Populate(sentencepiece)
+  add_subdirectory(${sentencepiece_SOURCE_DIR} ${sentencepiece_BINARY_DIR} EXCLUDE_FROM_ALL)
+endif()
+
 
 if(BUILD_FAST_TOKENIZERS)
   FetchContent_Declare(
@@ -115,8 +106,11 @@ else()
         URL      https://github.com/google/re2/archive/refs/tags/2022-04-01.tar.gz
         URL_HASH SHA256=1ae8ccfdb1066a731bba6ee0881baad5efd2cd661acd9569b689f2586e1a50e9
       )
-      FetchContent_MakeAvailable(re2)
-      set_property(DIRECTORY ${re2_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON)
+      FetchContent_GetProperties(re2)
+      if(NOT re2_POPULATED)
+        FetchContent_Populate(re2)
+        add_subdirectory(${re2_SOURCE_DIR} ${re2_BINARY_DIR} EXCLUDE_FROM_ALL)
+      endif()
   elseif(LINUX AND X86_64)
       FetchContent_Declare(
         fast_tokenizer
@@ -148,6 +142,7 @@ else()
   endif()
 
   FetchContent_MakeAvailable(fast_tokenizer)
+
   # to allow find_library to work with conda-forge env
   set(_old_CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ${CMAKE_FIND_ROOT_PATH_MODE_LIBRARY})
   set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER)
@@ -192,8 +187,6 @@ set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_OPTIONS "${extra_flags}"
 # Post build steps to copy core_tokenizers dependencies
 #
 
-set_property(DIRECTORY ${sentencepiece_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON)
-
 if(BUILD_FAST_TOKENIZERS)
   install(TARGETS core_tokenizers
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -224,4 +217,18 @@ if(extra_libs)
   install(FILES ${extra_libs} DESTINATION ${extra_libs_location})
 endif()
 
+if(rpaths)
+  set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}")
+endif()
+
+# Installing the extension module to the root of the package
+install(TARGETS ${TARGET_NAME}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+# cpack configuration
+set(CPACK_PACKAGE_NAME ${TARGET_NAME})
+set(CPACK_PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}")
+set(CPACK_SOURCE_GENERATOR "") # not used
+
 include (CPack)