Skip to content

Commit

Permalink
Flatbuffers integration
Browse files Browse the repository at this point in the history
  • Loading branch information
ilya-lavrenov committed Dec 8, 2022
1 parent a767aa9 commit cabed4e
Show file tree
Hide file tree
Showing 9 changed files with 183 additions and 89 deletions.
1 change: 1 addition & 0 deletions cmake/developer_package/IEDevScriptsConfig.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,7 @@ function(ov_mark_target_as_cc)
endfunction()

include(python_requirements)
include(native_compile)

# Code style utils

Expand Down
23 changes: 11 additions & 12 deletions cmake/developer_package/frontends/frontends.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,6 @@ macro(ov_add_frontend)

# Generate protobuf file on build time for each '.proto' file in src/proto
file(GLOB proto_files ${frontend_root_dir}/src/proto/*.proto)

foreach(INFILE IN LISTS proto_files)
get_filename_component(FILE_DIR ${INFILE} DIRECTORY)
get_filename_component(FILE_WE ${INFILE} NAME_WE)
Expand All @@ -155,28 +154,27 @@ macro(ov_add_frontend)
list(APPEND PROTO_HDRS "${OUTPUT_PB_HEADER}")
endforeach()

file(GLOB schema_files ${frontend_root_dir}/schema/schema.fbs)
if (schema_files)
set(INFILE ${frontend_root_dir}/schema/schema.fbs)
set(FLATC_EXECUTABLE /home/evgenya/repos/openvino/bin/intel64/Debug/flatc)
file(GLOB flatbuffers_schema_files ${frontend_root_dir}/schema/*.fbs)
foreach(INFILE IN LISTS flatbuffers_schema_files)
get_filename_component(FILE_WE ${INFILE} NAME_WE)
set(OUTPUT_FC_HEADER ${CMAKE_CURRENT_BINARY_DIR}/${FILE_WE}_generated.h)
set(GENERATED_PROTO ${INFILE})
add_custom_command(
OUTPUT "${OUTPUT_FC_HEADER}"
COMMAND ${FLATC_EXECUTABLE} ARGS -c --gen-mutable -o ${CMAKE_CURRENT_BINARY_DIR} ${INFILE}
DEPENDS ${FLATC_EXECUTABLE} ${GENERATED_PROTO}
COMMENT "Running C++ flatbuffers compiler (${FLATC_EXECUTABLE}) on ${GENERATED_PROTO}"
COMMAND ${flatbuffers_COMPILER} ARGS -c --gen-mutable -o ${CMAKE_CURRENT_BINARY_DIR} ${INFILE}
DEPENDS ${flatbuffers_DEPENDENCY} ${GENERATED_PROTO}
COMMENT "Running C++ flatbuffers compiler (${flatbuffers_COMPILER}) on ${GENERATED_PROTO}"
VERBATIM
COMMAND_EXPAND_LISTS)
list(APPEND PROTO_HDRS "${OUTPUT_FC_HEADER}")
endif()
endforeach()

# Disable all warnings for generated code
set_source_files_properties(${PROTO_SRCS} ${PROTO_HDRS} PROPERTIES COMPILE_OPTIONS -w GENERATED TRUE)

# Create library
add_library(${TARGET_NAME} ${LIBRARY_SRC} ${LIBRARY_HEADERS} ${LIBRARY_PUBLIC_HEADERS} ${PROTO_SRCS} ${PROTO_HDRS})
add_library(${TARGET_NAME} ${LIBRARY_SRC} ${LIBRARY_HEADERS} ${LIBRARY_PUBLIC_HEADERS}
${PROTO_SRCS} ${PROTO_HDRS} ${flatbuffers_schema_files} ${proto_files})

if(OV_FRONTEND_LINKABLE_FRONTEND)
# create beautiful alias
Expand Down Expand Up @@ -251,8 +249,9 @@ macro(ov_add_frontend)
endif()
endif()

if (schema_files)

if(flatbuffers_schema_files)
ov_install_static_lib(${flatbuffers_LIBRARY} ${OV_CPACK_COMP_CORE})
link_system_libraries(${TARGET_NAME} PRIVATE ${flatbuffers_LIBRARY})
endif()

add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}
Expand Down
91 changes: 91 additions & 0 deletions cmake/developer_package/native_compile.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Copyright (C) 2018-2022 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#

include(ExternalProject)

#
# ov_native_compile_external_project(
# TARGET_NAME <name>
# NATIVE_SOURCE_DIR <source dir>
# NATIVE_BUILD_DIR <source dir>
# NATIVE_INSTALL_DIR <source dir>
# NATIVE_PREFIX_DIR <source dir>
# CMAKE_OPTION <option1 option2 ...>
# )
#
function(ov_native_compile_external_project)
set(oneValueRequiredArgs NATIVE_SOURCE_DIR NATIVE_BUILD_DIR
NATIVE_INSTALL_DIR NATIVE_PREFIX_DIR
TARGET_NAME)
set(multiValueArgs CMAKE_OPTION)
cmake_parse_arguments(ARG "" "${oneValueRequiredArgs};${oneValueOptionalArgs}" "${multiValueArgs}" ${ARGN})

if(YOCTO_AARCH64)
# need to unset several variables which can set env to cross-environment
foreach(var SDKTARGETSYSROOT CONFIG_SITE OECORE_NATIVE_SYSROOT OECORE_TARGET_SYSROOT
OECORE_ACLOCAL_OPTS OECORE_BASELIB OECORE_TARGET_ARCH OECORE_TARGET_OS CC CXX
CPP AS LD GDB STRIP RANLIB OBJCOPY OBJDUMP READELF AR NM M4 TARGET_PREFIX
CONFIGURE_FLAGS CFLAGS CXXFLAGS LDFLAGS CPPFLAGS KCFLAGS OECORE_DISTRO_VERSION
OECORE_SDK_VERSION ARCH CROSS_COMPILE OE_CMAKE_TOOLCHAIN_FILE OPENSSL_CONF
OE_CMAKE_FIND_LIBRARY_CUSTOM_LIB_SUFFIX PKG_CONFIG_SYSROOT_DIR PKG_CONFIG_PATH)
if(DEFINED ENV{${var}})
list(APPEND cmake_env --unset=${var})
endif()
endforeach()

# filter out PATH from yocto locations
string(REPLACE ":" ";" custom_path "$ENV{PATH}")
foreach(path IN LISTS custom_path)
if(NOT path MATCHES "^$ENV{OECORE_NATIVE_SYSROOT}")
list(APPEND clean_path "${path}")
endif()
endforeach()

find_host_program(NATIVE_CMAKE_COMMAND
NAMES cmake
PATHS ${clean_path}
DOC "Host cmake"
REQUIRED
NO_DEFAULT_PATH)
else()
set(NATIVE_CMAKE_COMMAND "${CMAKE_COMMAND}")
endif()

# if env has CMAKE_TOOLCHAIN_FILE, we need to skip it
if(DEFINED ENV{CMAKE_TOOLCHAIN_FILE})
list(APPEND cmake_env --unset=CMAKE_TOOLCHAIN_FILE)
endif()

# compile flags
if(CMAKE_COMPILER_IS_GNUCXX)
set(compile_flags "-Wno-undef -Wno-error")
endif()

ExternalProject_Add(${ARG_TARGET_NAME}
SOURCE_DIR "${ARG_NATIVE_SOURCE_DIR}"
CONFIGURE_COMMAND
"${CMAKE_COMMAND}" -E env ${cmake_env}
"${NATIVE_CMAKE_COMMAND}"
# explicitly skip compiler and generator
# "-DCMAKE_GENERATOR=${CMAKE_GENERATOR}"
"-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}"
"-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}"
"-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}"
"-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}"
"-DCMAKE_CXX_FLAGS=${compile_flags}"
"-DCMAKE_C_FLAGS=${compile_flags}"
"-DCMAKE_POLICY_DEFAULT_CMP0069=NEW"
"-DCMAKE_INSTALL_PREFIX=${ARG_NATIVE_INSTALL_DIR}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
"-DTHREADS_PREFER_PTHREAD_FLAG=${THREADS_PREFER_PTHREAD_FLAG}"
# project specific
${ARG_CMAKE_OPTIONS}
# source and build directories
"-S${ARG_NATIVE_SOURCE_DIR}"
"-B${ARG_NATIVE_BUILD_DIR}"
BINARY_DIR "${ARG_NATIVE_BUILD_DIR}"
INSTALL_DIR "${ARG_NATIVE_INSTALL_DIR}"
PREFIX "${ARG_NATIVE_PREFIX_DIR}"
EXCLUDE_FROM_ALL ON)
endfunction()
2 changes: 2 additions & 0 deletions cmake/features.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ ie_option(ENABLE_OV_TF_FRONTEND "Enable TensorFlow FrontEnd" ON)
ie_option(ENABLE_OV_TF_LITE_FRONTEND "Enable TensorFlow Lite FrontEnd" ON)
ie_dependent_option(ENABLE_SYSTEM_PROTOBUF "Use system protobuf" OFF
"ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_TF_FRONTEND;BUILD_SHARED_LIBS" OFF)
ie_dependent_option(ENABLE_SYSTEM_FLATBUFFERS "Use system flatbuffers" OFF
"ENABLE_OV_TF_LITE_FRONTEND;BUILD_SHARED_LIBS" OFF)

ie_dependent_option(ENABLE_OV_CORE_UNIT_TESTS "Enables OpenVINO core unit tests" ON "ENABLE_TESTS" OFF)
ie_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF)
Expand Down
3 changes: 2 additions & 1 deletion src/frontends/tensorflow_lite/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
ov_add_frontend(NAME tensorflow_lite
LINKABLE_FRONTEND
FILEDESCRIPTION "FrontEnd to load and convert TensorFlow Lite file format"
LINK_LIBRARIES openvino::util openvino::runtime::dev openvino::frontend::tensorflow openvino_tensorflow_frontend_common_logic)
LINK_LIBRARIES openvino::util openvino::runtime::dev openvino::frontend::tensorflow
openvino_tensorflow_frontend_common_logic)
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

#include <fstream>

//#include "schema_generated.h"
#include "schema_generated.h"
//#include "decoder_proto.hpp"
//#include "graph.pb.h"
//#include "node_def.pb.h"
Expand Down
24 changes: 21 additions & 3 deletions thirdparty/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ if(ENABLE_OV_PADDLE_FRONTEND OR ENABLE_OV_ONNX_FRONTEND OR ENABLE_OV_TF_FRONTEND
set(Protobuf_LIBRARIES protobuf::libprotobuf)
set(PROTOC_EXECUTABLE protobuf::protoc)

foreach(target ${PROTOC_EXECUTABLE} ${Protobuf_LIBRARIES} ${Protobuf_LITE_LIBRARIES})
foreach(target IN LISTS PROTOC_EXECUTABLE Protobuf_LIBRARIES Protobuf_LITE_LIBRARIES)
set_property(TARGET ${target} PROPERTY IMPORTED_GLOBAL ON)
endforeach()
else()
Expand Down Expand Up @@ -287,8 +287,26 @@ endif()
#
# Flat Buffers
#
if (ENABLE_OV_TF_LITE_FRONTEND)
add_subdirectory(flatbuffers)

if(ENABLE_OV_TF_LITE_FRONTEND)
if(ENABLE_SYSTEM_FLATBUFFERS)
find_package(flatbuffers REQUIRED)

set(flatbuffers_LIBRARY flatbuffers::flatbuffers)
set(flatbuffers_COMPILER flatbuffers::flatc)

foreach(target IN LISTS flatbuffers_LIBRARY flatbuffers_COMPILER)
set_property(TARGET ${target} PROPERTY IMPORTED_GLOBAL ON)
endforeach()
endif()

if(NOT flatbuffers_FOUND)
add_subdirectory(flatbuffers EXCLUDE_FROM_ALL)
endif()

set(flatbuffers_LIBRARY ${flatbuffers_LIBRARY} PARENT_SCOPE)
set(flatbuffers_COMPILER ${flatbuffers_COMPILER} PARENT_SCOPE)
set(flatbuffers_DEPENDENCY ${flatbuffers_DEPENDENCY} PARENT_SCOPE)
endif()

#
Expand Down
45 changes: 44 additions & 1 deletion thirdparty/flatbuffers/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,47 @@

set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "" FORCE)
add_subdirectory(flatbuffers)
set(FLATBUFFERS_INSTALL OFF CACHE BOOL "" FORCE)

# note: HOST_AARCH64 AND X86_64 are not handled for Apple explicitly, becuase it can work via Rosetta
if(CMAKE_CROSSCOMPILING OR (APPLE AND (HOST_X86_64 AND AARCH64)) )
set(FLATBUFFERS_BUILD_FLATC OFF CACHE BOOL "" FORCE)
else()
set(FLATBUFFERS_BUILD_FLATC ON CACHE BOOL "" FORCE)
endif()

# build library at least
add_subdirectory(flatbuffers EXCLUDE_FROM_ALL)

# build flatc separatelly for host system processor

if(NOT FLATBUFFERS_BUILD_FLATC)
set(HOST_FLATC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/flatbuffers")
set(HOST_FLATC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_flatc_build")
set(HOST_FLATC_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_flatc_install")
set(HOST_FLATC_PREFIX_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_flatc_root")

ov_native_compile_external_project(
TARGET_NAME host_flatc
NATIVE_SOURCE_DIR "${HOST_FLATC_SOURCE_DIR}"
NATIVE_BUILD_DIR "${HOST_FLATC_BUILD_DIR}"
NATIVE_INSTALL_DIR "${HOST_FLATC_INSTALL_DIR}"
NATIVE_PREFIX_DIR "${HOST_FLATC_PREFIX_DIR}"
CMAKE_OPTIONS "-DFLATBUFFERS_BUILD_TESTS=${FLATBUFFERS_BUILD_TESTS}"
"-DFLATBUFFERS_BUILD_FLATHASH=${FLATBUFFERS_BUILD_FLATHASH}")

add_executable(flatbuffers::flatc IMPORTED GLOBAL)
set_target_properties(flatbuffers::flatc PROPERTIES
IMPORTED_LOCATION_RELEASE "${HOST_FLATC_INSTALL_DIR}/bin/flatc")
add_dependencies(flatbuffers::flatc host_flatc)

set(flatbuffers_DEPENDENCY host_flatc PARENT_SCOPE)
set(flatbuffers_COMPILER "${HOST_FLATC_INSTALL_DIR}/bin/flatc" PARENT_SCOPE)
else()
set(flatbuffers_COMPILER $<TARGET_FILE:flatc> PARENT_SCOPE)
set(flatbuffers_DEPENDENCY flatc PARENT_SCOPE)
endif()

# set parent scope

set(flatbuffers_LIBRARY flatbuffers PARENT_SCOPE)
81 changes: 10 additions & 71 deletions thirdparty/protobuf/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -66,84 +66,23 @@ if(protobuf_VERSION VERSION_LESS "3.9")
message(FATAL_ERROR "Minimum supported version of protobuf-lite library is 3.9.0 (provided ${protobuf_VERSION})")
endif()

# build protoc separatelly for host system processor
# build protoc separatelly for build system processor

if(NOT protobuf_BUILD_PROTOC_BINARIES)
include(ExternalProject)

set(HOST_PROTOC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/protobuf/cmake")
set(HOST_PROTOC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_build")
set(HOST_PROTOC_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_install")
set(HOST_PROTOC_PREFIX_DIR "${CMAKE_CURRENT_BINARY_DIR}/host_protoc_root")

if(YOCTO_AARCH64)
# need to unset several variables which can set env to cross-environment
foreach(var SDKTARGETSYSROOT CONFIG_SITE OECORE_NATIVE_SYSROOT OECORE_TARGET_SYSROOT
OECORE_ACLOCAL_OPTS OECORE_BASELIB OECORE_TARGET_ARCH OECORE_TARGET_OS CC CXX
CPP AS LD GDB STRIP RANLIB OBJCOPY OBJDUMP READELF AR NM M4 TARGET_PREFIX
CONFIGURE_FLAGS CFLAGS CXXFLAGS LDFLAGS CPPFLAGS KCFLAGS OECORE_DISTRO_VERSION
OECORE_SDK_VERSION ARCH CROSS_COMPILE OE_CMAKE_TOOLCHAIN_FILE OPENSSL_CONF
OE_CMAKE_FIND_LIBRARY_CUSTOM_LIB_SUFFIX PKG_CONFIG_SYSROOT_DIR PKG_CONFIG_PATH)
if(DEFINED ENV{${var}})
list(APPEND cmake_env --unset=${var})
endif()
endforeach()

# filter out PATH from yocto locations
string(REPLACE ":" ";" custom_path "$ENV{PATH}")
foreach(path IN LISTS custom_path)
if(NOT path MATCHES "^$ENV{OECORE_NATIVE_SYSROOT}")
list(APPEND clean_path "${path}")
endif()
endforeach()

find_host_program(HOST_CMAKE_COMMAND
NAMES cmake
PATHS ${clean_path}
DOC "Host cmake"
REQUIRED
NO_DEFAULT_PATH)
else()
set(HOST_CMAKE_COMMAND "${CMAKE_COMMAND}")
endif()

# if env has CMAKE_TOOLCHAIN_FILE, we need to skip it
if(DEFINED ENV{CMAKE_TOOLCHAIN_FILE})
list(APPEND cmake_env --unset=CMAKE_TOOLCHAIN_FILE)
endif()

# compile flags
if(CMAKE_COMPILER_IS_GNUCXX)
set(compile_flags "-Wno-undef -Wno-error")
endif()

ExternalProject_Add(host_protoc
SOURCE_DIR "${HOST_PROTOC_SOURCE_DIR}"
CONFIGURE_COMMAND
"${CMAKE_COMMAND}" -E env ${cmake_env}
"${HOST_CMAKE_COMMAND}"
# "-DCMAKE_GENERATOR=${CMAKE_GENERATOR}"
"-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}"
"-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}"
"-DCMAKE_CXX_LINKER_LAUNCHER=${CMAKE_CXX_LINKER_LAUNCHER}"
"-DCMAKE_C_LINKER_LAUNCHER=${CMAKE_C_LINKER_LAUNCHER}"
"-DCMAKE_CXX_FLAGS=${compile_flags}"
"-DCMAKE_C_FLAGS=${compile_flags}"
"-DCMAKE_POLICY_DEFAULT_CMP0069=NEW"
"-DCMAKE_INSTALL_PREFIX=${HOST_PROTOC_INSTALL_DIR}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
# protoc specific
"-DTHREADS_PREFER_PTHREAD_FLAG=${THREADS_PREFER_PTHREAD_FLAG}"
"-Dprotobuf_VERBOSE=${protobuf_VERBOSE}"
"-Dprotobuf_BUILD_TESTS=${protobuf_BUILD_TESTS}"
"-Dprotobuf_WITH_ZLIB=${protobuf_WITH_ZLIB}"
"-S${HOST_PROTOC_SOURCE_DIR}"
"-B${HOST_PROTOC_BUILD_DIR}"
BINARY_DIR "${HOST_PROTOC_BUILD_DIR}"
INSTALL_DIR "${HOST_PROTOC_INSTALL_DIR}"
PREFIX "${HOST_PROTOC_PREFIX_DIR}"
EXCLUDE_FROM_ALL ON
)
ov_native_compile_external_project(
TARGET_NAME host_protoc
NATIVE_SOURCE_DIR "${HOST_PROTOC_SOURCE_DIR}"
NATIVE_BUILD_DIR "${HOST_PROTOC_BUILD_DIR}"
NATIVE_INSTALL_DIR "${HOST_PROTOC_INSTALL_DIR}"
NATIVE_PREFIX_DIR "${HOST_PROTOC_PREFIX_DIR}"
CMAKE_OPTIONS "-Dprotobuf_VERBOSE=${protobuf_VERBOSE}"
"-Dprotobuf_BUILD_TESTS=${protobuf_BUILD_TESTS}"
"-Dprotobuf_WITH_ZLIB=${protobuf_WITH_ZLIB}")

add_executable(protobuf::protoc IMPORTED GLOBAL)
set_target_properties(protobuf::protoc PROPERTIES
Expand Down

0 comments on commit cabed4e

Please sign in to comment.