Fix merge conflicts resolution (#685)

openvinotoolkit · Jul 25, 2024 · f42e63d · f42e63d
1 parent 9759520
commit f42e63d
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 18 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -57,33 +57,17 @@ if(ENABLE_PYTHON)
     endif()
 endif()
 
-if(ENABLE_PYTHON)
-    # the following two calls are required for cross-compilation
-    if(OpenVINODeveloperPackage_DIR)
-        ov_find_python3(REQUIRED)
-        ov_detect_python_module_extension()
-    else()
-        if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-            find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
-        else()
-            find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
-        endif()
-    endif()
-endif()
-
 add_subdirectory(thirdparty)
 add_subdirectory(src)
 add_subdirectory(samples)
 add_subdirectory(tests/cpp)
 
-install(FILES LICENSE DESTINATION docs/licensing COMPONENT licensing_genai RENAME LICENSE-GENAI)
-install(FILES third-party-programs.txt DESTINATION docs/licensing COMPONENT licensing_genai RENAME third-party-programs-genai.txt)
 install(FILES LICENSE DESTINATION docs/licensing COMPONENT licensing_genai RENAME LICENSE-GENAI)
 install(FILES third-party-programs.txt DESTINATION docs/licensing COMPONENT licensing_genai RENAME third-party-programs-genai.txt)
 set(CPACK_ARCHIVE_COMPONENT_INSTALL ON)
 set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF)
 # Workaround https://gitlab.kitware.com/cmake/cmake/-/issues/2614
-set(CPACK_COMPONENTS_ALL core_genai core_genai_dev cpp_samples_genai licensing_genai openvino_tokenizers openvino_tokenizers_licenses)
+set(CPACK_COMPONENTS_ALL core_genai core_genai_dev cpp_samples_genai licensing_genai openvino_tokenizers openvino_tokenizers_docs)
 if(ENABLE_PYTHON)
     list(APPEND CPACK_COMPONENTS_ALL pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR})
 endif() 

diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers
+125 −12		.github/workflows/windows.yml
+1 −0		.gitignore
+80 −19		README.md
+17 −6		python/openvino_tokenizers/cli.py
+2 −0		python/openvino_tokenizers/convert_tokenizer.py
+47 −11		python/openvino_tokenizers/hf_parser.py
+6 −2		python/openvino_tokenizers/tiktoken_parser.py
+22 −4		python/openvino_tokenizers/tokenizer_pipeline.py
+34 −5		src/CMakeLists.txt
+8 −0		src/icu_filter_en.json
+71 −27		src/regex_normalization.cpp
+5 −1		src/regex_normalization.hpp
+93 −21		src/sentence_piece.cpp
+15 −2		src/sentence_piece.hpp
+71 −0		src/utils.cpp
+10 −1		src/utils.hpp
+4 −2		src/vocab_decoder.cpp
+0 −10		tests/conftest.py
+1 −1		tests/pass_rates.json
+2,298 −2,154		tests/stats.json
+16 −12		tests/tokenizers_test.py
+96 −0		third-party-programs.txt