[Enhancement] upgrade some thirdparty

upgrade brpc to 1.9.0 upgrade glog to 0.7.1 brpc has support thrift extends https://github.com/apache/brpc/blob/master/docs/en/thrift.md we could use brpc async interface instead of using thread pool simulate an async interface glog has support change vlog level dynamic in google/glog#650 Signed-off-by: stdpain <[email protected]>
kevincai · Jul 25, 2024 · 518c130 · 518c130
1 parent 5833705
commit 518c130
Show file tree

Hide file tree

Showing 11 changed files with 390 additions and 34 deletions.
diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp
@@ -136,7 +136,7 @@ static void dontdump_unused_pages() {
     start_dump = true;
 }
 
-static void failure_writer(const char* data, int size) {
+static void failure_writer(const char* data, size_t size) {
     if (config::enable_core_file_size_optimization) {
         dontdump_unused_pages();
     }

diff --git a/be/src/service/service_be/starrocks_be.cpp b/be/src/service/service_be/starrocks_be.cpp
@@ -217,9 +217,7 @@ void start_be(const std::vector<StorePath>& paths, bool as_cn) {
     brpc::FLAGS_max_body_size = config::brpc_max_body_size;
 
     // Configure keepalive.
-#ifdef WITH_BRPC_KEEPALIVE
     brpc::FLAGS_socket_keepalive = config::brpc_socket_keepalive;
-#endif
 
     brpc::FLAGS_socket_max_unwritten_bytes = config::brpc_socket_max_unwritten_bytes;
     auto brpc_server = std::make_unique<brpc::Server>();

diff --git a/be/src/util/filesystem_util.cc b/be/src/util/filesystem_util.cc
@@ -37,6 +37,7 @@
 #include <fcntl.h>
 #include <sys/resource.h>
 #include <sys/stat.h>
+#include <unistd.h>
 
 #include <cstring>
 #include <filesystem>

diff --git a/be/src/util/stack_util.cpp b/be/src/util/stack_util.cpp
@@ -22,31 +22,28 @@
 #include <fmt/format.h>
 #include <sys/syscall.h>
 
-#include <exception>
-
 #include "common/config.h"
 #include "gutil/strings/join.h"
 #include "gutil/strings/split.h"
 #include "gutil/strings/substitute.h"
 #include "runtime/current_thread.h"
 #include "util/time.h"
 
-namespace google::glog_internal_namespace_ {
-void DumpStackTraceToString(std::string* stacktrace);
-} // namespace google::glog_internal_namespace_
+namespace google {
+std::string GetStackTrace();
+}
 
 // import hidden stack trace functions from glog
-namespace google {
+namespace google::glog_internal_namespace_ {
+enum class SymbolizeOptions { kNone = 0, kNoLineNumbers = 1 };
 int GetStackTrace(void** result, int max_depth, int skip_count);
-bool Symbolize(void* pc, char* out, int out_size);
-} // namespace google
+bool Symbolize(void* pc, char* out, unsigned long out_size, SymbolizeOptions options = SymbolizeOptions::kNone);
+} // namespace google::glog_internal_namespace_
 
 namespace starrocks {
 
 std::string get_stack_trace() {
-    std::string s;
-    google::glog_internal_namespace_::DumpStackTraceToString(&s);
-    return s;
+    return google::GetStackTrace();
 }
 
 struct StackTraceTask {
@@ -59,7 +56,7 @@ struct StackTraceTask {
         for (int i = 0; i < depth; ++i) {
             char line[2048];
             char buf[1024];
-            if (google::Symbolize(addrs[i], buf, sizeof(buf))) {
+            if (google::glog_internal_namespace_::Symbolize(addrs[i], buf, sizeof(buf))) {
                 snprintf(line, 2048, "  %16p  %s\n", addrs[i], buf);
             } else {
                 snprintf(line, 2048, "  %16p  (unknown)\n", addrs[i]);
@@ -93,7 +90,7 @@ struct StackTraceTaskHash {
 
 void get_stack_trace_sighandler(int signum, siginfo_t* siginfo, void* ucontext) {
     auto task = reinterpret_cast<StackTraceTask*>(siginfo->si_value.sival_ptr);
-    task->depth = google::GetStackTrace(task->addrs, StackTraceTask::kMaxStackDepth, 2);
+    task->depth = google::glog_internal_namespace_::GetStackTrace(task->addrs, StackTraceTask::kMaxStackDepth, 2);
     task->done = true;
 }
 

diff --git a/build.sh b/build.sh
@@ -153,7 +153,6 @@ WITH_GCOV=OFF
 WITH_BENCH=OFF
 WITH_CLANG_TIDY=OFF
 WITH_STARCACHE=ON
-WITH_BRPC_KEEPALIVE=OFF
 USE_STAROS=OFF
 BUILD_JAVA_EXT=ON
 OUTPUT_COMPILE_TIME=OFF
@@ -244,7 +243,6 @@ else
             --with-clang-tidy) WITH_CLANG_TIDY=ON; shift ;;
             --without-java-ext) BUILD_JAVA_EXT=OFF; shift ;;
             --without-starcache) WITH_STARCACHE=OFF; shift ;;
-            --with-brpc-keepalive) WITH_BRPC_KEEPALIVE=ON; shift ;;
             --output-compile-time) OUTPUT_COMPILE_TIME=ON; shift ;;
             --with-tenann) WITH_TENANN=ON; shift ;;
             -h) HELP=1; shift ;;
@@ -383,7 +381,6 @@ if [ ${BUILD_BE} -eq 1 ] ; then
                   -DWITH_CLANG_TIDY=${WITH_CLANG_TIDY}                  \
                   -DWITH_COMPRESS=${WITH_COMPRESS}                      \
                   -DWITH_STARCACHE=${WITH_STARCACHE}                    \
-                  -DWITH_BRPC_KEEPALIVE=${WITH_BRPC_KEEPALIVE}          \
                   -DUSE_STAROS=${USE_STAROS}                            \
                   -DENABLE_FAULT_INJECTION=${ENABLE_FAULT_INJECTION}    \
                   -DWITH_TENANN=${WITH_TENANN}                          \

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
@@ -368,13 +368,8 @@ build_glog() {
     check_if_source_exist $GLOG_SOURCE
     cd $TP_SOURCE_DIR/$GLOG_SOURCE
 
-    # to generate config.guess and config.sub to support aarch64
-    rm -rf config.*
-    autoreconf -i
+    $CMAKE_CMD -G "${CMAKE_GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_LIBDIR=lib
 
-    LDFLAGS="-L${TP_LIB_DIR}" \
-    CPPFLAGS="-I${TP_INCLUDE_DIR}" \
-    ./configure --prefix=$TP_INSTALL_DIR --enable-frame-pointers --disable-shared --enable-static
     make -j$PARALLEL
     make install
 }
@@ -560,7 +555,7 @@ build_brpc() {
     cd $TP_SOURCE_DIR/$BRPC_SOURCE
     CMAKE_GENERATOR="Unix Makefiles"
     BUILD_SYSTEM='make'
-    ./config_brpc.sh --headers="$TP_INSTALL_DIR/include /usr/include" --libs="$TP_INSTALL_DIR/bin $TP_INSTALL_DIR/lib /usr/lib" --with-glog
+    PATH=$PATH:$TP_INSTALL_DIR/bin/ ./config_brpc.sh --headers="$TP_INSTALL_DIR/include" --libs="$TP_INSTALL_DIR/bin $TP_INSTALL_DIR/lib" --with-glog --with-thrift    
     make -j$PARALLEL
     cp -rf output/* ${TP_INSTALL_DIR}/
     if [ -f $TP_INSTALL_DIR/lib/libbrpc.a ]; then
@@ -747,6 +742,7 @@ build_s2() {
     $CMAKE_CMD -G "${CMAKE_GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
     -DBUILD_SHARED_LIBS=OFF \
+    -DCMAKE_CXX_STANDARD="17" \
     -DGFLAGS_ROOT_DIR="$TP_INSTALL_DIR/include" \
     -DWITH_GFLAGS=ON \
     -DGLOG_ROOT_DIR="$TP_INSTALL_DIR/include" \

diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
@@ -250,6 +250,10 @@ if [ ! -f $PATCHED_MARK ] && [ $GLOG_SOURCE == "glog-0.4.0" ]; then
     patch -p1 < $TP_PATCH_DIR/glog-0.4.0-remove-unwind-dependency.patch 
     touch $PATCHED_MARK
 fi
+if [ ! -f $PATCHED_MARK ] && [ $GLOG_SOURCE == "glog-0.7.1" ]; then
+    patch -p1 < $TP_PATCH_DIR/glog-0.7.1.patch
+    touch $PATCHED_MARK
+fi
 cd -
 echo "Finished patching $GLOG_SOURCE"
 
@@ -305,6 +309,10 @@ if [ ! -f $PATCHED_MARK ] && [ $BRPC_SOURCE == "brpc-1.3.0" ]; then
     patch -p1 < $TP_PATCH_DIR/brpc-1.3.0-2479.patch
     touch $PATCHED_MARK
 fi
+if [ ! -f $PATCHED_MARK ] && [ $BRPC_SOURCE == "brpc-1.9.0" ]; then
+    patch < $TP_PATCH_DIR/brpc-1.9.0.patch
+    touch $PATCHED_MARK
+fi
 cd -
 echo "Finished patching $BRPC_SOURCE"
 
@@ -314,6 +322,7 @@ if [ ! -f $PATCHED_MARK ]; then
     patch -p1 < $TP_PATCH_DIR/s2geometry-0.9.0.patch
     # replace uint64 with uint64_t to make compiler happy
     patch -p0 < $TP_PATCH_DIR/s2geometry-0.9.0-uint64.patch
+    patch -p1 < $TP_PATCH_DIR/s2geometry-0.9.0-cxx17.patch
     touch $PATCHED_MARK
 fi
 cd -

diff --git a/thirdparty/patches/brpc-1.9.0.patch b/thirdparty/patches/brpc-1.9.0.patch
@@ -0,0 +1,13 @@
+diff --git a/config_brpc.sh b/config_brpc.sh
+index ddaa6daa..2269587a 100755
+--- a/config_brpc.sh
++++ b/config_brpc.sh
+@@ -341,7 +341,7 @@ if [ "$PROTOBUF_VERSION" -ge 4022000 ]; then
+     done
+     CXXFLAGS="-std=c++17"
+ else
+-    CXXFLAGS="-std=c++0x"
++    CXXFLAGS="-std=c++17"
+ fi
+
+ LEVELDB_HDR=$(find_dir_of_header_or_die leveldb/db.h)