Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: use the system provided packages if found #8930

Merged
9 changes: 7 additions & 2 deletions .github/workflows/pr-compile-check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,13 @@ jobs:
- name: Setup environment
run: |
sudo apt-get update
sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev
sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \
libnghttp2-dev libjemalloc-dev
sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
mkdir -p /tmp/libbacktrace/build && \
curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \
tar --strip-components=1 -xzC /tmp/libbacktrace/ && \
pushd /tmp/libbacktrace/build && ../configure && make && sudo make install && popd

- name: Checkout Fluent Bit code
uses: actions/checkout@v4
Expand All @@ -67,7 +72,7 @@ jobs:
CC: ${{ matrix.compiler }}
CXX: ${{ matrix.compiler }}
FLB_OPT: ${{ matrix.flb_option }}
GLOBAL_OPTS: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
GLOBAL_OPTS: "-DFLB_JEMALLOC=On -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"

- name: Display dependencies w/ ldd
run: |
Expand Down
166 changes: 98 additions & 68 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,12 @@ option(FLB_RUN_LDCONFIG "Enable execution of ldconfig after installation" No)

# Prefer system libraries if available
option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" No)
option(FLB_PREFER_SYSTEM_LIB_BACKTRACE "Prefer the libbacktrace system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_CARES "Prefer the libcares system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_JEMALLOC "Prefer the libjemalloc system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_KAFKA "Prefer the libkafka system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS})
option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS})

# Enable all features
if(FLB_ALL)
Expand Down Expand Up @@ -484,25 +489,28 @@ add_subdirectory(${FLB_PATH_LIB_CMETRICS} EXCLUDE_FROM_ALL)
add_subdirectory(${FLB_PATH_LIB_CTRACES} EXCLUDE_FROM_ALL)

# Nghttp2 options
FLB_OPTION(ENABLE_LIB_ONLY ON)
FLB_OPTION(ENABLE_STATIC_LIB ON)
FLB_OPTION(ENABLE_SHARED_LIB OFF)

FLB_DEFINITION(NGHTTP2_STATICLIB)

add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL)
if(FLB_PREFER_SYSTEM_LIB_NGHTTP2)
find_package(PkgConfig)
pkg_check_modules(NGHTTP2 libnghttp2>=1.0.0)
endif()
if(NGHTTP2_FOUND)
include_directories(${NGHTTP2_INCLUDE_DIRS})
link_directories(${NGHTTP2_LIBRARY_DIRS})
else()
include(cmake/nghttp2.cmake)
endif()

# C-Ares (DNS library)
FLB_OPTION(CARES_STATIC ON)
FLB_OPTION(CARES_SHARED OFF)
FLB_OPTION(CARES_INSTALL OFF)
FLB_OPTION(CARES_BUILD_TESTS OFF)
FLB_OPTION(CARES_BUILD_TOOLS OFF)
if (FLB_SYSTEM_MACOS)
# macOS SDK always has <arpa/nameser.h>.
FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H)
if(FLB_PREFER_SYSTEM_LIB_CARES)
find_package(PkgConfig)
pkg_check_modules(LIBCARES libcares>=1.18.0)
endif()
if(LIBCARES_FOUND)
include_directories(${LIBCARES_INCLUDE_DIRS})
link_directories(${LIBCARES_LIBRARY_DIRS})
else()
include(cmake/cares.cmake)
endif()
add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL)

# Chunk I/O
FLB_OPTION(CIO_LIB_STATIC ON)
Expand Down Expand Up @@ -598,6 +606,7 @@ if(FLB_WASM)
_Atomic int a;
return 0;
}" FLB_HAVE_STDATOMIC_H)
include(CheckIncludeFiles)
check_include_files(stdatomic.h FLB_HAVE_STDATOMIC_H)
if (FLB_HAVE_STDATOMIC_H)
enable_language (ASM)
Expand Down Expand Up @@ -902,72 +911,93 @@ endif()
# Memory Allocator
# ================
if(FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux")
FLB_DEFINITION(FLB_HAVE_JEMALLOC)
FLB_DEFINITION(JEMALLOC_MANGLE)
if(FLB_PREFER_SYSTEM_LIB_JEMALLOC)
find_package(PkgConfig)
pkg_check_modules(JEMALLOC jemalloc>=5.0.0)
endif()
if(JEMALLOC_FOUND)
include_directories(${JEMALLOC_INCLUDE_DIRS})
link_directories(${JEMALLOC_LIBRARY_DIRS})
else()
FLB_DEFINITION(JEMALLOC_MANGLE)

# Add support for options like page size, if empty we default it
if(NOT DEFINED FLB_JEMALLOC_OPTIONS OR "${FLB_JEMALLOC_OPTIONS}" STREQUAL "")
set(FLB_JEMALLOC_OPTIONS "--with-lg-quantum=3")
# Add support for options like page size, if empty we default it
if(NOT DEFINED FLB_JEMALLOC_OPTIONS OR "${FLB_JEMALLOC_OPTIONS}" STREQUAL "")
set(FLB_JEMALLOC_OPTIONS "--with-lg-quantum=3")
endif()
# Split into a list so CMake handles it correctly when passing to configure command
separate_arguments(FLB_JEMALLOC_OPTIONS_LIST UNIX_COMMAND ${FLB_JEMALLOC_OPTIONS})
message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}")

# Link to Jemalloc as an external dependency
set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a")
ExternalProject_Add(jemalloc
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix=<INSTALL_DIR>
CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH}
INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include)
add_library(libjemalloc STATIC IMPORTED GLOBAL)
set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH})
add_dependencies(libjemalloc jemalloc)
include_directories("${CMAKE_BINARY_DIR}/include/")
set(JEMALLOC_LIBRARIES "libjemalloc")
endif()
# Split into a list so CMake handles it correctly when passing to configure command
separate_arguments(FLB_JEMALLOC_OPTIONS_LIST UNIX_COMMAND ${FLB_JEMALLOC_OPTIONS})
message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}")

# Link to Jemalloc as an external dependency
set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a")
ExternalProject_Add(jemalloc
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix=<INSTALL_DIR>
CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH}
INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include)
add_library(libjemalloc STATIC IMPORTED GLOBAL)
set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH})
add_dependencies(libjemalloc jemalloc)
include_directories("${CMAKE_BINARY_DIR}/include/")
FLB_DEFINITION(FLB_HAVE_JEMALLOC)
else()
FLB_OPTION(FLB_JEMALLOC OFF)
endif()

# LibBacktrace (friendly stacktrace support)
# =========================================
if(FLB_BACKTRACE)
FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE)
if (CMAKE_OSX_SYSROOT)
# From macOS Mojave, /usr/include does not store C SDK headers.
# For libbacktrace building on macOS, we have to tell C headers where they are located.
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}")
if(FLB_PREFER_SYSTEM_LIB_BACKTRACE)
find_path(LIBBACKTRACE_INCLUDE_DIRS NAMES backtrace.h)
find_library(LIBBACKTRACE_LIBRARIES NAMES backtrace)
endif()
if(LIBBACKTRACE_INCLUDE_DIRS AND LIBBACKTRACE_LIBRARIES)
message(STATUS "libbacktrace found (${LIBBACKTRACE_LIBRARIES})")
include_directories(${LIBBACKTRACE_INCLUDE_DIRS})
link_directories(${LIBBACKTRACE_LIBRARY_DIRS})
else()
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
message(STATUS "libbacktrace not found, building ourselves")
if (CMAKE_OSX_SYSROOT)
# From macOS Mojave, /usr/include does not store C SDK headers.
# For libbacktrace building on macOS, we have to tell C headers where they are located.
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}")
else()
set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
endif()
set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a")
ExternalProject_Add(backtrace
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix=<INSTALL_DIR> --enable-shared=no --enable-static=yes
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH}
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install
)
add_library(libbacktrace STATIC IMPORTED GLOBAL)
set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH})
add_dependencies(libbacktrace backtrace)
include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/")
set(LIBBACKTRACE_LIBRARIES "libbacktrace")
endif()
set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a")
ExternalProject_Add(backtrace
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix=<INSTALL_DIR> --enable-shared=no --enable-static=yes
BUILD_COMMAND ${EXTERNAL_BUILD_TOOL}
BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH}
INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install
)
add_library(libbacktrace STATIC IMPORTED GLOBAL)
set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH})
add_dependencies(libbacktrace backtrace)
include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/")
FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE)
endif()

if(FLB_IN_KAFKA OR FLB_OUT_KAFKA)
FLB_OPTION(RDKAFKA_BUILD_STATIC On)
FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off)
FLB_OPTION(RDKAFKA_BUILD_TESTS Off)
FLB_OPTION(ENABLE_LZ4_EXT Off)

# disable Curl
if (FLB_SYSTEM_MACOS)
FLB_OPTION(WITH_CURL Off)
endif()

add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL)
if(FLB_PREFER_SYSTEM_LIB_KAFKA)
find_package(PkgConfig)
pkg_check_modules(KAFKA rdkafka>=2.3.0)
endif()
if(KAFKA_FOUND)
include_directories(${KAFKA_INCLUDE_DIRS})
link_directories(${KAFKA_LIBRARY_DIRS})
else()
include(cmake/kafka.cmake)
endif()
endif()

# Onigmo (Regex Engine) options
Expand Down
20 changes: 20 additions & 0 deletions cmake/cares.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# cares cmake
FLB_OPTION(CARES_STATIC ON)
FLB_OPTION(CARES_SHARED OFF)
FLB_OPTION(CARES_INSTALL OFF)
FLB_OPTION(CARES_BUILD_TESTS OFF)
FLB_OPTION(CARES_BUILD_TOOLS OFF)

if (FLB_SYSTEM_MACOS)
# macOS SDK always has <arpa/nameser.h>.
FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H)
endif()

include_directories(
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include
${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_CARES}
)

add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL)

set(LIBCARES_LIBRARIES "c-ares")
10 changes: 0 additions & 10 deletions cmake/headers.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,16 @@ include_directories(
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_SNAPPY}
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CMETRICS}/include
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CTRACES}/include
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RING_BUFFER}/lwrb/src/include

${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_CARES}
${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include
${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics
${CMAKE_CURRENT_BINARY_DIR}/include

${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/
${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/monkey/

${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2
${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/
)

if(FLB_IN_KAFKA OR FLB_OUT_KAFKA)
include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src/)
endif()

if(FLB_UTF8_ENCODER)
include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_TUTF8E}/include)
endif()
Expand Down
16 changes: 16 additions & 0 deletions cmake/kafka.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# kafka cmake
FLB_OPTION(RDKAFKA_BUILD_STATIC On)
FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off)
FLB_OPTION(RDKAFKA_BUILD_TESTS Off)
FLB_OPTION(ENABLE_LZ4_EXT Off)

# disable Curl
if (FLB_SYSTEM_MACOS)
FLB_OPTION(WITH_CURL Off)
endif()

include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src/)

add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL)

set(KAFKA_LIBRARIES "rdkafka")
12 changes: 12 additions & 0 deletions cmake/nghttp2.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# nghttp2 cmake
FLB_OPTION(ENABLE_LIB_ONLY ON)
FLB_OPTION(ENABLE_STATIC_LIB ON)
FLB_OPTION(ENABLE_SHARED_LIB OFF)
FLB_DEFINITION(NGHTTP2_STATICLIB)
include_directories(
${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2
${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/
${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/
)
add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL)
set(NGHTTP2_LIBRARIES "nghttp2_static")
3 changes: 2 additions & 1 deletion plugins/in_kafka/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ set(src
in_kafka.c
)

FLB_PLUGIN(in_kafka "${src}" "rdkafka")
FLB_PLUGIN(in_kafka "${src}" ${KAFKA_LIBRARIES})
target_include_directories(flb-plugin-in_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka)
target_link_libraries(flb-plugin-in_kafka -lpthread)
3 changes: 2 additions & 1 deletion plugins/out_kafka/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ set(src
kafka_topic.c
kafka.c)

FLB_PLUGIN(out_kafka "${src}" "rdkafka")
FLB_PLUGIN(out_kafka "${src}" ${KAFKA_LIBRARIES})
target_include_directories(flb-plugin-out_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka)
target_link_libraries(flb-plugin-out_kafka -lpthread)
2 changes: 1 addition & 1 deletion plugins/processor_sql/parser/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ add_library(processor-sql-parser STATIC

add_flex_bison_dependency(lexer parser)

if (FLB_JEMALLOC)
if (FLB_JEMALLOC AND (NOT JEMALLOC_FOUND))
add_dependencies(processor-sql-parser libjemalloc)
include_directories("${CMAKE_BINARY_DIR}/include/")
endif ()
12 changes: 6 additions & 6 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ set(extra_libs
if(FLB_JEMALLOC)
set(extra_libs
${extra_libs}
"libjemalloc")
${JEMALLOC_LIBRARIES})
endif()

if(FLB_REGEX)
Expand Down Expand Up @@ -378,10 +378,10 @@ set(FLB_DEPS
${FLB_PLUGINS}
${FLB_PROXY_PLUGINS}
${extra_libs}
c-ares
${LIBCARES_LIBRARIES}
snappy-c
lwrb
nghttp2_static
${NGHTTP2_LIBRARIES}
)

if(OPENSSL_FOUND)
Expand Down Expand Up @@ -478,7 +478,7 @@ else()
endif(MSVC)

if(FLB_JEMALLOC)
target_link_libraries(fluent-bit-static libjemalloc)
target_link_libraries(fluent-bit-static ${JEMALLOC_LIBRARIES})
endif()

# Binary / Executable
Expand All @@ -501,12 +501,12 @@ if(FLB_BINARY)
endif()

if(FLB_JEMALLOC)
target_link_libraries(fluent-bit-bin libjemalloc)
target_link_libraries(fluent-bit-bin ${JEMALLOC_LIBRARIES})
endif()

if(FLB_BACKTRACE)
add_definitions(-DFLB_DUMP_STACKTRACE=1)
target_link_libraries(fluent-bit-bin libbacktrace)
target_link_libraries(fluent-bit-bin ${LIBBACKTRACE_LIBRARIES})
endif()

target_link_libraries(fluent-bit-bin fluent-bit-static ${CMAKE_THREAD_LIBS_INIT})
Expand Down
2 changes: 1 addition & 1 deletion src/aws/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,5 @@ add_library(flb-aws STATIC ${src})
target_link_libraries(flb-aws flb-aws-compress)

if(FLB_JEMALLOC)
target_link_libraries(flb-aws libjemalloc)
target_link_libraries(flb-aws ${JEMALLOC_LIBRARIES})
endif()
Loading
Loading