From 88bc70cfb7618513eb74e5c05def9b2e62e4eceb Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Mon, 10 Jun 2024 20:12:27 +0200 Subject: [PATCH 1/8] build: use the system provided libnghttp2 if found e.g. buildroot has logic to build libnghttp2, so if pkg_check_modules can find a suitable version, then use that one if -DFLB_PREFER_SYSTEM_LIB_NGHTTP2=Yes. Signed-off-by: Thomas Devoogdt --- CMakeLists.txt | 18 +++++++++++------- cmake/headers.cmake | 4 ---- cmake/nghttp2.cmake | 12 ++++++++++++ src/CMakeLists.txt | 2 +- 4 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 cmake/nghttp2.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e2e9fbf3b04..65b4ae2511b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,6 +199,7 @@ option(FLB_RUN_LDCONFIG "Enable execution of ldconfig after installation" No) # Prefer system libraries if available option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" No) option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS}) +option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS}) # Enable all features if(FLB_ALL) @@ -484,13 +485,16 @@ add_subdirectory(${FLB_PATH_LIB_CMETRICS} EXCLUDE_FROM_ALL) add_subdirectory(${FLB_PATH_LIB_CTRACES} EXCLUDE_FROM_ALL) # Nghttp2 options -FLB_OPTION(ENABLE_LIB_ONLY ON) -FLB_OPTION(ENABLE_STATIC_LIB ON) -FLB_OPTION(ENABLE_SHARED_LIB OFF) - -FLB_DEFINITION(NGHTTP2_STATICLIB) - -add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL) +if(FLB_PREFER_SYSTEM_LIB_NGHTTP2) + find_package(PkgConfig) + pkg_check_modules(NGHTTP2 libnghttp2>=1.0.0) +endif() +if(NGHTTP2_FOUND) + include_directories(${NGHTTP2_INCLUDE_DIRS}) + link_directories(${NGHTTP2_LIBRARY_DIRS}) +else() + include(cmake/nghttp2.cmake) +endif() # C-Ares (DNS library) FLB_OPTION(CARES_STATIC ON) diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 500c4c3b947..9f9927cd8b0 100755 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -42,10 +42,6 @@ include_directories( ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/ ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/monkey/ - - ${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2 - ${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/ - ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/ ) if(FLB_IN_KAFKA OR FLB_OUT_KAFKA) diff --git a/cmake/nghttp2.cmake b/cmake/nghttp2.cmake new file mode 100644 index 00000000000..cc291736c0a --- /dev/null +++ b/cmake/nghttp2.cmake @@ -0,0 +1,12 @@ +# nghttp2 cmake +FLB_OPTION(ENABLE_LIB_ONLY ON) +FLB_OPTION(ENABLE_STATIC_LIB ON) +FLB_OPTION(ENABLE_SHARED_LIB OFF) +FLB_DEFINITION(NGHTTP2_STATICLIB) +include_directories( + ${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2 + ${CMAKE_CURRENT_BINARY_DIR}/lib/nghttp2/lib/includes/ + ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_NGHTTP2}/lib/includes/ +) +add_subdirectory(${FLB_PATH_LIB_NGHTTP2} EXCLUDE_FROM_ALL) +set(NGHTTP2_LIBRARIES "nghttp2_static") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 084f2c57d52..c5291eaa94a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -381,7 +381,7 @@ set(FLB_DEPS c-ares snappy-c lwrb - nghttp2_static + ${NGHTTP2_LIBRARIES} ) if(OPENSSL_FOUND) From 44e6d9fce0a83fe9bc07e24cf051fc26398b1636 Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sat, 8 Jun 2024 15:51:53 +0200 Subject: [PATCH 2/8] build: use the system provided backtrace if found e.g. buildroot has logic to build backtrace, so if find_path/find_library can find a suitable version, then use that one if -DFLB_PREFER_SYSTEM_LIB_BACKTRACE=Yes. Signed-off-by: Thomas Devoogdt --- CMakeLists.txt | 49 +++++++++++++++++++++++++++++----------------- src/CMakeLists.txt | 2 +- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 65b4ae2511b..16da3ebb48c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,7 @@ option(FLB_RUN_LDCONFIG "Enable execution of ldconfig after installation" No) # Prefer system libraries if available option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" No) +option(FLB_PREFER_SYSTEM_LIB_BACKTRACE "Prefer the libbacktrace system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS}) @@ -938,26 +939,38 @@ endif() # LibBacktrace (friendly stacktrace support) # ========================================= if(FLB_BACKTRACE) - FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE) - if (CMAKE_OSX_SYSROOT) - # From macOS Mojave, /usr/include does not store C SDK headers. - # For libbacktrace building on macOS, we have to tell C headers where they are located. - set(DEPS_C_COMPILER "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}") + if(FLB_PREFER_SYSTEM_LIB_BACKTRACE) + find_path(LIBBACKTRACE_INCLUDE_DIRS NAMES backtrace.h) + find_library(LIBBACKTRACE_LIBRARIES NAMES backtrace) + endif() + if(LIBBACKTRACE_INCLUDE_DIRS AND LIBBACKTRACE_LIBRARIES) + message(STATUS "libbacktrace found (${LIBBACKTRACE_LIBRARIES})") + include_directories(${LIBBACKTRACE_INCLUDE_DIRS}) + link_directories(${LIBBACKTRACE_LIBRARY_DIRS}) else() - set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}") + message(STATUS "libbacktrace not found, building ourselves") + if (CMAKE_OSX_SYSROOT) + # From macOS Mojave, /usr/include does not store C SDK headers. + # For libbacktrace building on macOS, we have to tell C headers where they are located. + set(DEPS_C_COMPILER "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}") + else() + set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}") + endif() + set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a") + ExternalProject_Add(backtrace + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/ + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix= --enable-shared=no --enable-static=yes + BUILD_COMMAND ${EXTERNAL_BUILD_TOOL} + BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH} + INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install + ) + add_library(libbacktrace STATIC IMPORTED GLOBAL) + set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH}) + add_dependencies(libbacktrace backtrace) + include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/") + set(LIBBACKTRACE_LIBRARIES "libbacktrace") endif() - set(FLB_LIBBACKTRACE_PATH "${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/lib/libbacktrace.a") - ExternalProject_Add(backtrace - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/ - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/libbacktrace-8602fda/configure ${AUTOCONF_HOST_OPT} --prefix= --enable-shared=no --enable-static=yes - BUILD_COMMAND ${EXTERNAL_BUILD_TOOL} - BUILD_BYPRODUCTS ${FLB_LIBBACKTRACE_PATH} - INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} DESTDIR= install - ) - add_library(libbacktrace STATIC IMPORTED GLOBAL) - set_target_properties(libbacktrace PROPERTIES IMPORTED_LOCATION ${FLB_LIBBACKTRACE_PATH}) - add_dependencies(libbacktrace backtrace) - include_directories("${CMAKE_CURRENT_BINARY_DIR}/backtrace-prefix/include/") + FLB_DEFINITION(FLB_HAVE_LIBBACKTRACE) endif() if(FLB_IN_KAFKA OR FLB_OUT_KAFKA) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c5291eaa94a..b01123b8249 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -506,7 +506,7 @@ if(FLB_BINARY) if(FLB_BACKTRACE) add_definitions(-DFLB_DUMP_STACKTRACE=1) - target_link_libraries(fluent-bit-bin libbacktrace) + target_link_libraries(fluent-bit-bin ${LIBBACKTRACE_LIBRARIES}) endif() target_link_libraries(fluent-bit-bin fluent-bit-static ${CMAKE_THREAD_LIBS_INIT}) From b1433eb2aef8bff1225d58729abe6a17d18d36d8 Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sat, 8 Jun 2024 15:51:22 +0200 Subject: [PATCH 3/8] build: use the system provided jemalloc if found e.g. buildroot has logic to build jemalloc, so if pkg_check_modules can find a suitable version, then use that one if -DFLB_PREFER_SYSTEM_LIB_JEMALLOC=Yes. Signed-off-by: Thomas Devoogdt --- CMakeLists.txt | 57 ++++++++++++--------- plugins/processor_sql/parser/CMakeLists.txt | 2 +- src/CMakeLists.txt | 6 +-- src/aws/CMakeLists.txt | 2 +- src/proxy/go/CMakeLists.txt | 2 +- src/record_accessor/CMakeLists.txt | 2 +- src/stream_processor/parser/CMakeLists.txt | 2 +- src/wasm/CMakeLists.txt | 4 +- tests/internal/CMakeLists.txt | 2 +- tests/internal/fuzzers/CMakeLists.txt | 2 +- 10 files changed, 46 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16da3ebb48c..2667a86ab32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,6 +199,7 @@ option(FLB_RUN_LDCONFIG "Enable execution of ldconfig after installation" No) # Prefer system libraries if available option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" No) option(FLB_PREFER_SYSTEM_LIB_BACKTRACE "Prefer the libbacktrace system library" ${FLB_PREFER_SYSTEM_LIBS}) +option(FLB_PREFER_SYSTEM_LIB_JEMALLOC "Prefer the libjemalloc system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS}) @@ -907,31 +908,41 @@ endif() # Memory Allocator # ================ if(FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux") - FLB_DEFINITION(FLB_HAVE_JEMALLOC) - FLB_DEFINITION(JEMALLOC_MANGLE) + if(FLB_PREFER_SYSTEM_LIB_JEMALLOC) + find_package(PkgConfig) + pkg_check_modules(JEMALLOC jemalloc>=5.0.0) + endif() + if(JEMALLOC_FOUND) + include_directories(${JEMALLOC_INCLUDE_DIRS}) + link_directories(${JEMALLOC_LIBRARY_DIRS}) + else() + FLB_DEFINITION(JEMALLOC_MANGLE) - # Add support for options like page size, if empty we default it - if(NOT DEFINED FLB_JEMALLOC_OPTIONS OR "${FLB_JEMALLOC_OPTIONS}" STREQUAL "") - set(FLB_JEMALLOC_OPTIONS "--with-lg-quantum=3") + # Add support for options like page size, if empty we default it + if(NOT DEFINED FLB_JEMALLOC_OPTIONS OR "${FLB_JEMALLOC_OPTIONS}" STREQUAL "") + set(FLB_JEMALLOC_OPTIONS "--with-lg-quantum=3") + endif() + # Split into a list so CMake handles it correctly when passing to configure command + separate_arguments(FLB_JEMALLOC_OPTIONS_LIST UNIX_COMMAND ${FLB_JEMALLOC_OPTIONS}) + message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}") + + # Link to Jemalloc as an external dependency + set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a") + ExternalProject_Add(jemalloc + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0 + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix= + CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops + BUILD_COMMAND ${EXTERNAL_BUILD_TOOL} + BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH} + INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/ + INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include) + add_library(libjemalloc STATIC IMPORTED GLOBAL) + set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH}) + add_dependencies(libjemalloc jemalloc) + include_directories("${CMAKE_BINARY_DIR}/include/") + set(JEMALLOC_LIBRARIES "libjemalloc") endif() - # Split into a list so CMake handles it correctly when passing to configure command - separate_arguments(FLB_JEMALLOC_OPTIONS_LIST UNIX_COMMAND ${FLB_JEMALLOC_OPTIONS}) - message(STATUS "jemalloc configure: ${FLB_JEMALLOC_OPTIONS_LIST}") - - # Link to Jemalloc as an external dependency - set(FLB_LIBJEMALLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/lib/libjemalloc_pic.a") - ExternalProject_Add(jemalloc - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0 - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/jemalloc-5.3.0/configure ${AUTOCONF_HOST_OPT} "${FLB_JEMALLOC_OPTIONS_LIST}" --prefix= - CFLAGS=-std=gnu99\ -Wall\ -pipe\ -g3\ -O3\ -funroll-loops - BUILD_COMMAND ${EXTERNAL_BUILD_TOOL} - BUILD_BYPRODUCTS ${FLB_LIBJEMALLOC_PATH} - INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/ - INSTALL_COMMAND ${EXTERNAL_BUILD_TOOL} install_lib_static install_include) - add_library(libjemalloc STATIC IMPORTED GLOBAL) - set_target_properties(libjemalloc PROPERTIES IMPORTED_LOCATION ${FLB_LIBJEMALLOC_PATH}) - add_dependencies(libjemalloc jemalloc) - include_directories("${CMAKE_BINARY_DIR}/include/") + FLB_DEFINITION(FLB_HAVE_JEMALLOC) else() FLB_OPTION(FLB_JEMALLOC OFF) endif() diff --git a/plugins/processor_sql/parser/CMakeLists.txt b/plugins/processor_sql/parser/CMakeLists.txt index 7e5feca944e..e78a5cba1ef 100644 --- a/plugins/processor_sql/parser/CMakeLists.txt +++ b/plugins/processor_sql/parser/CMakeLists.txt @@ -26,7 +26,7 @@ add_library(processor-sql-parser STATIC add_flex_bison_dependency(lexer parser) -if (FLB_JEMALLOC) +if (FLB_JEMALLOC AND (NOT JEMALLOC_FOUND)) add_dependencies(processor-sql-parser libjemalloc) include_directories("${CMAKE_BINARY_DIR}/include/") endif () \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b01123b8249..acf3027eb3b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -207,7 +207,7 @@ set(extra_libs if(FLB_JEMALLOC) set(extra_libs ${extra_libs} - "libjemalloc") + ${JEMALLOC_LIBRARIES}) endif() if(FLB_REGEX) @@ -478,7 +478,7 @@ else() endif(MSVC) if(FLB_JEMALLOC) - target_link_libraries(fluent-bit-static libjemalloc) + target_link_libraries(fluent-bit-static ${JEMALLOC_LIBRARIES}) endif() # Binary / Executable @@ -501,7 +501,7 @@ if(FLB_BINARY) endif() if(FLB_JEMALLOC) - target_link_libraries(fluent-bit-bin libjemalloc) + target_link_libraries(fluent-bit-bin ${JEMALLOC_LIBRARIES}) endif() if(FLB_BACKTRACE) diff --git a/src/aws/CMakeLists.txt b/src/aws/CMakeLists.txt index a6580e7a511..0b02fdb5127 100644 --- a/src/aws/CMakeLists.txt +++ b/src/aws/CMakeLists.txt @@ -28,5 +28,5 @@ add_library(flb-aws STATIC ${src}) target_link_libraries(flb-aws flb-aws-compress) if(FLB_JEMALLOC) - target_link_libraries(flb-aws libjemalloc) + target_link_libraries(flb-aws ${JEMALLOC_LIBRARIES}) endif() diff --git a/src/proxy/go/CMakeLists.txt b/src/proxy/go/CMakeLists.txt index 93e5003537a..69ef4204dc5 100644 --- a/src/proxy/go/CMakeLists.txt +++ b/src/proxy/go/CMakeLists.txt @@ -3,7 +3,7 @@ set(src add_library(flb-plugin-proxy-go STATIC ${src}) if(FLB_JEMALLOC) - target_link_libraries(flb-plugin-proxy-go libjemalloc) + target_link_libraries(flb-plugin-proxy-go ${JEMALLOC_LIBRARIES}) endif() if(FLB_REGEX) target_link_libraries(flb-plugin-proxy-go onigmo-static) diff --git a/src/record_accessor/CMakeLists.txt b/src/record_accessor/CMakeLists.txt index 9eb3825d9cb..406f9013845 100644 --- a/src/record_accessor/CMakeLists.txt +++ b/src/record_accessor/CMakeLists.txt @@ -27,5 +27,5 @@ add_flex_bison_dependency(lexer parser) add_dependencies(flb-ra-parser onigmo-static) if(FLB_JEMALLOC) - target_link_libraries(flb-ra-parser libjemalloc) + target_link_libraries(flb-ra-parser ${JEMALLOC_LIBRARIES}) endif() diff --git a/src/stream_processor/parser/CMakeLists.txt b/src/stream_processor/parser/CMakeLists.txt index d14b02bbd3d..05649ec4ff1 100644 --- a/src/stream_processor/parser/CMakeLists.txt +++ b/src/stream_processor/parser/CMakeLists.txt @@ -27,5 +27,5 @@ add_flex_bison_dependency(lexer parser) add_dependencies(flb-sp-parser onigmo-static) if(FLB_JEMALLOC) - target_link_libraries(flb-sp-parser libjemalloc) + target_link_libraries(flb-sp-parser ${JEMALLOC_LIBRARIES}) endif() diff --git a/src/wasm/CMakeLists.txt b/src/wasm/CMakeLists.txt index 4ea7eba65a7..2855e837b08 100644 --- a/src/wasm/CMakeLists.txt +++ b/src/wasm/CMakeLists.txt @@ -131,7 +131,7 @@ set(src add_library(flb-wasm-static STATIC ${src}) -if (FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux") +if (FLB_JEMALLOC AND ${CMAKE_SYSTEM_NAME} MATCHES "Linux" AND (NOT JEMALLOC_FOUND)) set(${JEMALLOC_LIBS} libjemalloc) add_dependencies(flb-wasm-static libjemalloc) include_directories("${CMAKE_BINARY_DIR}/include/") @@ -140,5 +140,5 @@ endif () if (WAMR_BUILD_LIBC_UVWASI) target_link_libraries(flb-wasm-static vmlib-static ${UV_A_LIBS}) else () - target_link_libraries(flb-wasm-static vmlib-static ${JEMALLOC_LIBS}) + target_link_libraries(flb-wasm-static vmlib-static ${JEMALLOC_LIBRARIES}) endif() diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 7fe08ea0d8f..6ef5df4bd4a 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -184,7 +184,7 @@ function(prepare_unit_tests TEST_PREFIX SOURCEFILES) add_sanitizers(${source_file_we}) if(FLB_JEMALLOC) - target_link_libraries(${source_file_we} libjemalloc ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${source_file_we} ${JEMALLOC_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) else() target_link_libraries(${source_file_we} ${CMAKE_THREAD_LIBS_INIT}) endif() diff --git a/tests/internal/fuzzers/CMakeLists.txt b/tests/internal/fuzzers/CMakeLists.txt index e76be548a20..aae2b19ba85 100644 --- a/tests/internal/fuzzers/CMakeLists.txt +++ b/tests/internal/fuzzers/CMakeLists.txt @@ -40,7 +40,7 @@ foreach(source_file ${UNIT_TESTS_FILES}) ) if(FLB_JEMALLOC) - target_link_libraries(${source_file_we} libjemalloc ${CMAKE_THREAD_LIBS_INIT}) + target_link_libraries(${source_file_we} ${JEMALLOC_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) else() target_link_libraries(${source_file_we} ${CMAKE_THREAD_LIBS_INIT}) endif() From c2c9a3eef7637cc586e1a2221bb72a0cac8f16ea Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sun, 9 Jun 2024 16:04:32 +0200 Subject: [PATCH 4/8] build: use the system provided c-ares if found e.g. buildroot has logic to build c-ares, so if pkg_check_modules can find a suitable version, then use that one if -DFLB_PREFER_SYSTEM_LIB_CARES=Yes. Note that the CheckIncludeFiles include for wasm is added, since that was done by c-ares otherwise. Signed-off-by: Thomas Devoogdt --- CMakeLists.txt | 20 +++++++++++--------- cmake/cares.cmake | 20 ++++++++++++++++++++ cmake/headers.cmake | 2 -- src/CMakeLists.txt | 2 +- 4 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 cmake/cares.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 2667a86ab32..55d2260b423 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,6 +199,7 @@ option(FLB_RUN_LDCONFIG "Enable execution of ldconfig after installation" No) # Prefer system libraries if available option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" No) option(FLB_PREFER_SYSTEM_LIB_BACKTRACE "Prefer the libbacktrace system library" ${FLB_PREFER_SYSTEM_LIBS}) +option(FLB_PREFER_SYSTEM_LIB_CARES "Prefer the libcares system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_JEMALLOC "Prefer the libjemalloc system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS}) @@ -499,16 +500,16 @@ else() endif() # C-Ares (DNS library) -FLB_OPTION(CARES_STATIC ON) -FLB_OPTION(CARES_SHARED OFF) -FLB_OPTION(CARES_INSTALL OFF) -FLB_OPTION(CARES_BUILD_TESTS OFF) -FLB_OPTION(CARES_BUILD_TOOLS OFF) -if (FLB_SYSTEM_MACOS) - # macOS SDK always has . - FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H) +if(FLB_PREFER_SYSTEM_LIB_CARES) + find_package(PkgConfig) + pkg_check_modules(LIBCARES libcares>=1.18.0) +endif() +if(LIBCARES_FOUND) + include_directories(${LIBCARES_INCLUDE_DIRS}) + link_directories(${LIBCARES_LIBRARY_DIRS}) +else() + include(cmake/cares.cmake) endif() -add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL) # Chunk I/O FLB_OPTION(CIO_LIB_STATIC ON) @@ -604,6 +605,7 @@ if(FLB_WASM) _Atomic int a; return 0; }" FLB_HAVE_STDATOMIC_H) + include(CheckIncludeFiles) check_include_files(stdatomic.h FLB_HAVE_STDATOMIC_H) if (FLB_HAVE_STDATOMIC_H) enable_language (ASM) diff --git a/cmake/cares.cmake b/cmake/cares.cmake new file mode 100644 index 00000000000..212d3f1c2f6 --- /dev/null +++ b/cmake/cares.cmake @@ -0,0 +1,20 @@ +# cares cmake +FLB_OPTION(CARES_STATIC ON) +FLB_OPTION(CARES_SHARED OFF) +FLB_OPTION(CARES_INSTALL OFF) +FLB_OPTION(CARES_BUILD_TESTS OFF) +FLB_OPTION(CARES_BUILD_TOOLS OFF) + +if (FLB_SYSTEM_MACOS) + # macOS SDK always has . + FLB_DEFINITION(CARES_HAVE_ARPA_NAMESER_H) +endif() + +include_directories( + ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include + ${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_CARES} +) + +add_subdirectory(${FLB_PATH_LIB_CARES})# EXCLUDE_FROM_ALL) + +set(LIBCARES_LIBRARIES "c-ares") diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 9f9927cd8b0..1b0347b2c79 100755 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -32,10 +32,8 @@ include_directories( ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_SNAPPY} ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CMETRICS}/include ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CTRACES}/include - ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CARES}/include ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RING_BUFFER}/lwrb/src/include - ${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_CARES} ${CMAKE_CURRENT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include ${CMAKE_CURRENT_BINARY_DIR}/lib/cmetrics ${CMAKE_CURRENT_BINARY_DIR}/include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index acf3027eb3b..223ea93ca29 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -378,7 +378,7 @@ set(FLB_DEPS ${FLB_PLUGINS} ${FLB_PROXY_PLUGINS} ${extra_libs} - c-ares + ${LIBCARES_LIBRARIES} snappy-c lwrb ${NGHTTP2_LIBRARIES} From 243f9c8da59ef4feac7e02e5f7c5b66659afc476 Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sat, 15 Jun 2024 17:47:31 +0200 Subject: [PATCH 5/8] build: use the system provided librdkafka if found e.g. buildroot has logic to build librdkafka, so if pkg_check_modules can find a suitable version, then use that one if -DFLB_PREFER_SYSTEM_LIB_KAFKA=Yes. Signed-off-by: Thomas Devoogdt --- CMakeLists.txt | 22 +++++++++++----------- cmake/headers.cmake | 4 ---- cmake/kafka.cmake | 16 ++++++++++++++++ plugins/in_kafka/CMakeLists.txt | 3 ++- plugins/out_kafka/CMakeLists.txt | 3 ++- 5 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 cmake/kafka.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 55d2260b423..5ebc197a313 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,6 +201,7 @@ option(FLB_PREFER_SYSTEM_LIBS "Prefer system libraries" option(FLB_PREFER_SYSTEM_LIB_BACKTRACE "Prefer the libbacktrace system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_CARES "Prefer the libcares system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_JEMALLOC "Prefer the libjemalloc system library" ${FLB_PREFER_SYSTEM_LIBS}) +option(FLB_PREFER_SYSTEM_LIB_KAFKA "Prefer the libkafka system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_LUAJIT "Prefer the libluajit system library" ${FLB_PREFER_SYSTEM_LIBS}) option(FLB_PREFER_SYSTEM_LIB_NGHTTP2 "Prefer the libnghttp2 system library" ${FLB_PREFER_SYSTEM_LIBS}) @@ -987,17 +988,16 @@ if(FLB_BACKTRACE) endif() if(FLB_IN_KAFKA OR FLB_OUT_KAFKA) - FLB_OPTION(RDKAFKA_BUILD_STATIC On) - FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off) - FLB_OPTION(RDKAFKA_BUILD_TESTS Off) - FLB_OPTION(ENABLE_LZ4_EXT Off) - - # disable Curl - if (FLB_SYSTEM_MACOS) - FLB_OPTION(WITH_CURL Off) - endif() - - add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL) + if(FLB_PREFER_SYSTEM_LIB_KAFKA) + find_package(PkgConfig) + pkg_check_modules(KAFKA rdkafka>=2.3.0) + endif() + if(KAFKA_FOUND) + include_directories(${KAFKA_INCLUDE_DIRS}) + link_directories(${KAFKA_LIBRARY_DIRS}) + else() + include(cmake/kafka.cmake) + endif() endif() # Onigmo (Regex Engine) options diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 1b0347b2c79..5280d8e8bbe 100755 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -42,10 +42,6 @@ include_directories( ${CMAKE_CURRENT_BINARY_DIR}/lib/monkey/include/monkey/ ) -if(FLB_IN_KAFKA OR FLB_OUT_KAFKA) - include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src/) -endif() - if(FLB_UTF8_ENCODER) include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_TUTF8E}/include) endif() diff --git a/cmake/kafka.cmake b/cmake/kafka.cmake new file mode 100644 index 00000000000..3a0ebfed23c --- /dev/null +++ b/cmake/kafka.cmake @@ -0,0 +1,16 @@ +# kafka cmake +FLB_OPTION(RDKAFKA_BUILD_STATIC On) +FLB_OPTION(RDKAFKA_BUILD_EXAMPLES Off) +FLB_OPTION(RDKAFKA_BUILD_TESTS Off) +FLB_OPTION(ENABLE_LZ4_EXT Off) + +# disable Curl +if (FLB_SYSTEM_MACOS) + FLB_OPTION(WITH_CURL Off) +endif() + +include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src/) + +add_subdirectory(${FLB_PATH_LIB_RDKAFKA} EXCLUDE_FROM_ALL) + +set(KAFKA_LIBRARIES "rdkafka") diff --git a/plugins/in_kafka/CMakeLists.txt b/plugins/in_kafka/CMakeLists.txt index 696e263fab7..ede0831f9f8 100644 --- a/plugins/in_kafka/CMakeLists.txt +++ b/plugins/in_kafka/CMakeLists.txt @@ -2,5 +2,6 @@ set(src in_kafka.c ) -FLB_PLUGIN(in_kafka "${src}" "rdkafka") +FLB_PLUGIN(in_kafka "${src}" ${KAFKA_LIBRARIES}) +target_include_directories(flb-plugin-in_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka) target_link_libraries(flb-plugin-in_kafka -lpthread) diff --git a/plugins/out_kafka/CMakeLists.txt b/plugins/out_kafka/CMakeLists.txt index 526910d49a0..01c96529b1c 100644 --- a/plugins/out_kafka/CMakeLists.txt +++ b/plugins/out_kafka/CMakeLists.txt @@ -4,5 +4,6 @@ set(src kafka_topic.c kafka.c) -FLB_PLUGIN(out_kafka "${src}" "rdkafka") +FLB_PLUGIN(out_kafka "${src}" ${KAFKA_LIBRARIES}) +target_include_directories(flb-plugin-out_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka) target_link_libraries(flb-plugin-out_kafka -lpthread) From 0375a2f51a8119bf4df769101a5bd33d910cc98c Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sat, 6 Jul 2024 21:04:19 +0000 Subject: [PATCH 6/8] workflows: pr-compile-check.yaml: add libbacktrace system library test Signed-off-by: Thomas Devoogdt --- .github/workflows/pr-compile-check.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-compile-check.yaml b/.github/workflows/pr-compile-check.yaml index dd982fab26e..0b47a2027aa 100644 --- a/.github/workflows/pr-compile-check.yaml +++ b/.github/workflows/pr-compile-check.yaml @@ -47,8 +47,12 @@ jobs: - name: Setup environment run: | sudo apt-get update - sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev + sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true + mkdir -p /tmp/libbacktrace/build && \ + curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \ + tar --strip-components=1 -xzC /tmp/libbacktrace/ && \ + pushd /tmp/libbacktrace/build && ../configure && make && sudo make install && popd - name: Checkout Fluent Bit code uses: actions/checkout@v4 @@ -67,7 +71,7 @@ jobs: CC: ${{ matrix.compiler }} CXX: ${{ matrix.compiler }} FLB_OPT: ${{ matrix.flb_option }} - GLOBAL_OPTS: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" + GLOBAL_OPTS: "-DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" - name: Display dependencies w/ ldd run: | From 881256c0aa679a96a3cb9add1afc9846cb4b21f5 Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sat, 6 Jul 2024 21:04:49 +0000 Subject: [PATCH 7/8] workflows: pr-compile-check.yaml: add libnghttp2 system library test Signed-off-by: Thomas Devoogdt --- .github/workflows/pr-compile-check.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-compile-check.yaml b/.github/workflows/pr-compile-check.yaml index 0b47a2027aa..c55aa24da94 100644 --- a/.github/workflows/pr-compile-check.yaml +++ b/.github/workflows/pr-compile-check.yaml @@ -47,7 +47,8 @@ jobs: - name: Setup environment run: | sudo apt-get update - sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev + sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \ + libnghttp2-dev sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true mkdir -p /tmp/libbacktrace/build && \ curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \ From a24eac79429203ca6bebf1c919162bb3d60c9084 Mon Sep 17 00:00:00 2001 From: Thomas Devoogdt Date: Sat, 6 Jul 2024 21:05:12 +0000 Subject: [PATCH 8/8] workflows: pr-compile-check.yaml: add libjemalloc system library test Signed-off-by: Thomas Devoogdt --- .github/workflows/pr-compile-check.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-compile-check.yaml b/.github/workflows/pr-compile-check.yaml index c55aa24da94..42381a8a1d1 100644 --- a/.github/workflows/pr-compile-check.yaml +++ b/.github/workflows/pr-compile-check.yaml @@ -48,7 +48,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y curl gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libluajit-5.1-dev \ - libnghttp2-dev + libnghttp2-dev libjemalloc-dev sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true mkdir -p /tmp/libbacktrace/build && \ curl -L https://github.com/ianlancetaylor/libbacktrace/archive/8602fda.tar.gz | \ @@ -72,7 +72,7 @@ jobs: CC: ${{ matrix.compiler }} CXX: ${{ matrix.compiler }} FLB_OPT: ${{ matrix.flb_option }} - GLOBAL_OPTS: "-DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" + GLOBAL_OPTS: "-DFLB_JEMALLOC=On -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off" - name: Display dependencies w/ ldd run: |