Use TensorFlow Conversion extension for TF FE translators (#46)

* Use TensorFlow Conversion extension for TF FE translators Signed-off-by: Kazantsev, Roman <[email protected]> * Add TensorFlow component to build Signed-off-by: Kazantsev, Roman <[email protected]> * Update src/ov_extension.cpp * Allow build without TensorFlow FE --------- Signed-off-by: Kazantsev, Roman <[email protected]> Co-authored-by: Ilya Lavrenov <[email protected]>
openvinotoolkit · Feb 29, 2024 · a1afb67 · a1afb67
1 parent cf41db3
commit a1afb67
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 10 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -8,7 +8,7 @@ project(openvino_tokenizers)
 
 include(cmake/platforms.cmake)
 
-find_package(OpenVINO REQUIRED COMPONENTS Runtime)
+find_package(OpenVINO REQUIRED COMPONENTS Runtime OPTIONAL_COMPONENTS TensorFlow)
 
 # set the project version using OpenVINO major, minor and patch versions
 set(CMAKE_PROJECT_VERSION_TWEAK 0)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -167,11 +167,20 @@ endif()
 #
 
 file(GLOB SRC *.cpp)
+if(NOT OpenVINO_Frontend_TensorFlow_FOUND)
+  list(REMOVE_ITEM SRC ${CMAKE_CURRENT_SOURCE_DIR}/tensorflow_translators.cpp)
+endif()
+
 add_library(${TARGET_NAME} SHARED ${SRC})
 
 target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API)
 target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime)
 
+if(OpenVINO_Frontend_TensorFlow_FOUND)
+  target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::tensorflow)
+  target_compile_definitions(${TARGET_NAME} PRIVATE OpenVINO_Frontend_TensorFlow_FOUND)
+endif()
+
 #
 # Target include dirs, link libraries and other properties
 #

diff --git a/src/ov_extension.cpp b/src/ov_extension.cpp
@@ -8,6 +8,21 @@
 
 #include "tokenizer.hpp"
 
+#ifdef OpenVINO_Frontend_TensorFlow_FOUND
+#include <openvino/frontend/tensorflow/extension/conversion.hpp>
+#define OPENVINO_TOKENIZERS_TENSORFLOW_CONVERSION_EXTENSIONS                                                                                     \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("WordpieceTokenizeWithOffsets", translate_wordpiece_tokenize_with_offsets),  \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("RegexSplitWithOffsets", translate_regex_split_with_offsets),                \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("NormalizeUTF8", translate_normalize_utf8),                                  \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("CaseFoldUTF8", translate_case_fold_utf8),                                   \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("SentencepieceOp", translate_sentencepiece_op),                              \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("RaggedTensorToSparse", translate_sentencepiece_tokenizer),                  \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("StringLower", translate_string_lower),                                      \
+    std::make_shared<ov::frontend::tensorflow::ConversionExtension>("StaticRegexReplace", translate_static_regex_replace),
+#else
+#define OPENVINO_TOKENIZERS_TENSORFLOW_CONVERSION_EXTENSIONS
+#endif
+
 // clang-format off
 //! [ov_extension:entry_point]
 OPENVINO_CREATE_EXTENSIONS(
@@ -16,15 +31,10 @@ OPENVINO_CREATE_EXTENSIONS(
             std::make_shared<ov::OpExtension<RaggedTensorPack>>(),
             std::make_shared<ov::OpExtension<StringTensorUnpack>>(),
             std::make_shared<ov::OpExtension<CaseFold>>(),
-            std::make_shared<ov::frontend::ConversionExtension>("CaseFoldUTF8", translate_case_fold_utf8),
             std::make_shared<ov::OpExtension<NormalizeUnicode>>(),
-            std::make_shared<ov::frontend::ConversionExtension>("NormalizeUTF8", translate_normalize_utf8),
             std::make_shared<ov::OpExtension<RegexNormalization>>(),
-            std::make_shared<ov::frontend::ConversionExtension>("StaticRegexReplace", translate_static_regex_replace),
             std::make_shared<ov::OpExtension<RegexSplit>>(),
-            std::make_shared<ov::frontend::ConversionExtension>("RegexSplitWithOffsets", translate_regex_split_with_offsets),
             std::make_shared<ov::OpExtension<WordpieceTokenizer>>(),
-            std::make_shared<ov::frontend::ConversionExtension>("WordpieceTokenizeWithOffsets", translate_wordpiece_tokenize_with_offsets),
             std::make_shared<ov::OpExtension<BPETokenizer>>(),
             std::make_shared<ov::OpExtension<BytesToChars>>(),
             std::make_shared<ov::OpExtension<CombineSegments>>(),
@@ -34,9 +44,7 @@ OPENVINO_CREATE_EXTENSIONS(
             std::make_shared<ov::OpExtension<TemplateExtension::SentencepieceTokenizer>>(),
             std::make_shared<ov::OpExtension<TemplateExtension::SentencepieceDetokenizer>>(),
             std::make_shared<ov::OpExtension<TemplateExtension::SentencepieceStreamDetokenizer>>(),
-            std::make_shared<ov::frontend::ConversionExtension>("SentencepieceOp", translate_sentencepiece_op),
-            std::make_shared<ov::frontend::ConversionExtension>("RaggedTensorToSparse", translate_sentencepiece_tokenizer),
-            std::make_shared<ov::frontend::ConversionExtension>("StringLower", translate_string_lower),
-    }));
+            OPENVINO_TOKENIZERS_TENSORFLOW_CONVERSION_EXTENSIONS
+}));
 //! [ov_extension:entry_point]
 // clang-format on