feat: Distinguish log file names and limit log file count

CMakeLists.txt

@@ -64,4 +64,7 @@ if(USE_JEMALLOC)
     set(JEMALLOC_LIB_TYPE "SHARED")
 endif()
 
+option(SEPARATE_SERVERS "Whether to build pegasus_collector，pegasus_meta_server and pegasus_replica_server binaries separately, otherwise a combined pegasus_server binary will be built" OFF)
+message(STATUS "SEPARATE_SERVERS = ${SEPARATE_SERVERS}")
+
 add_subdirectory(src)

collector/go.mod

@@ -94,7 +94,7 @@ require (
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/protobuf v1.29.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

collector/go.sum

@@ -543,8 +543,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
-google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

run.sh

@@ -18,14 +18,23 @@
 
 set -e
 
-LOCAL_HOSTNAME=`hostname -f`
+LOCAL_HOSTNAME=$(hostname -f)
 PID=$$
 ROOT="$(cd "$(dirname "$0")" && pwd)"
 export BUILD_ROOT_DIR=${ROOT}/build
 export BUILD_LATEST_DIR=${BUILD_ROOT_DIR}/latest
 export REPORT_DIR="$ROOT/test_report"
 export THIRDPARTY_ROOT=$ROOT/thirdparty
-export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:${ROOT}/lib:${BUILD_LATEST_DIR}/output/lib:${THIRDPARTY_ROOT}/output/lib:${LD_LIBRARY_PATH}
+ARCH_TYPE=''
+arch_output=$(arch)
+if [ "$arch_output"x == "x86_64"x ]; then
+    ARCH_TYPE="amd64"
+elif [ "$arch_output"x == "aarch64"x ]; then
+    ARCH_TYPE="aarch64"
+else
+    echo "WARNING: unsupported CPU architecture '$arch_output', use 'x86_64' as default"
+fi
+export LD_LIBRARY_PATH=${JAVA_HOME}/jre/lib/${ARCH_TYPE}:${JAVA_HOME}/jre/lib/${ARCH_TYPE}/server:${BUILD_LATEST_DIR}/output/lib:${THIRDPARTY_ROOT}/output/lib:${LD_LIBRARY_PATH}
 # Disable AddressSanitizerOneDefinitionRuleViolation, see https://github.com/google/sanitizers/issues/1017 for details.
 # Add parameters in order to be able to generate coredump file when run ASAN tests
 export ASAN_OPTIONS=detect_odr_violation=0:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
@@ -97,6 +106,7 @@ function usage_build()
     echo "   --disable_gperf       build without gperftools, this flag is mainly used"
     echo "                         to enable valgrind memcheck, default no"
     echo "   --use_jemalloc        build with jemalloc"
+    echo "   --separate_servers    whether to build pegasus_collector，pegasus_meta_server and pegasus_replica_server binaries separately, otherwise a combined pegasus_server binary will be built"
     echo "   --sanitizer <type>    build with sanitizer to check potential problems,
                                    type: address|leak|thread|undefined"
     echo "   --skip_thirdparty     whether to skip building thirdparties, default no"
@@ -130,6 +140,7 @@ function run_build()
     SANITIZER=""
     ROCKSDB_PORTABLE=0
     USE_JEMALLOC=OFF
+    SEPARATE_SERVERS=OFF
     BUILD_TEST=OFF
     IWYU=""
     BUILD_MODULES=""
@@ -198,6 +209,9 @@ function run_build()
                 ENABLE_GPERF=OFF
                 USE_JEMALLOC=ON
                 ;;
+            --separate_servers)
+                SEPARATE_SERVERS=ON
+                ;;
             --test)
                 BUILD_TEST=ON
                 ;;
@@ -230,7 +244,8 @@ function run_build()
 
     CMAKE_OPTIONS="-DCMAKE_C_COMPILER=${C_COMPILER}
                    -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-                   -DUSE_JEMALLOC=${USE_JEMALLOC}"
+                   -DUSE_JEMALLOC=${USE_JEMALLOC}
+                   -DSEPARATE_SERVERS=${SEPARATE_SERVERS}"
 
     echo "BUILD_TYPE=$BUILD_TYPE"
     if [ "$BUILD_TYPE" == "debug" ]

src/base/pegasus_utils.cpp

@@ -19,35 +19,14 @@
 
 #include "pegasus_utils.h"
 
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <netinet/in.h>
 #include <stdio.h>
-#include <sys/socket.h>
 #include <cctype>
 
-#include "runtime/rpc/rpc_address.h"
 #include "utils/fmt_logging.h"
 
 namespace pegasus {
 namespace utils {
 
-void addr2host(const ::dsn::rpc_address &addr, char *str, int len /* = 100*/)
-{
-    struct sockaddr_in addr2;
-    addr2.sin_addr.s_addr = htonl(addr.ip());
-    addr2.sin_family = AF_INET;
-    if (getnameinfo((struct sockaddr *)&addr2,
-                    sizeof(sockaddr),
-                    str,
-                    sizeof(char *) * len,
-                    nullptr,
-                    0,
-                    NI_NAMEREQD)) {
-        inet_ntop(AF_INET, &(addr2.sin_addr), str, 100);
-    }
-}
-
 size_t
 c_escape_string(const char *src, size_t src_len, char *dest, size_t dest_len, bool always_escape)
 {

src/base/pegasus_utils.h

@@ -28,15 +28,11 @@
 #include <string>
 #include <vector>
 
-#include "utils/flags.h"
 #include "absl/strings/string_view.h"
+#include "utils/flags.h"
 
 DSN_DECLARE_bool(encrypt_data_at_rest);
 
-namespace dsn {
-class rpc_address;
-} // namespace dsn
-
 namespace pegasus {
 namespace utils {
 
@@ -45,9 +41,6 @@ const uint32_t epoch_begin = 1451606400;
 inline uint32_t epoch_now() { return time(nullptr) - epoch_begin; }
 const static std::string kRedactedString = "<redacted>";
 
-// extract "host" from rpc_address
-void addr2host(const ::dsn::rpc_address &addr, char *str, int len);
-
 template <typename elem_type, typename compare = std::less<elem_type>>
 class top_n
 {

src/base/test/CMakeLists.txt

@@ -28,10 +28,14 @@ set(MY_PROJ_SRC "")
 set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_PROJ_LIBS
-    dsn_runtime
-    dsn_utils
-    pegasus_base
-    gtest)
+        dsn_runtime
+        dsn_utils
+        pegasus_base
+        rocksdb
+        lz4
+        zstd
+        snappy
+        gtest)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 

src/client/replication_ddl_client.h

@@ -162,7 +162,7 @@ class replication_ddl_client
                                int32_t old_app_id,
                                const std::string &new_app_name,
                                bool skip_bad_partition,
-                               const std::string &restore_path = "");
+                               const std::string &restore_path);
 
     dsn::error_code query_restore(int32_t restore_app_id, bool detailed);
 

src/common/fs_manager.cpp

@@ -298,6 +298,7 @@ dir_node *fs_manager::find_best_dir_for_new_replica(const gpid &pid) const
         for (const auto &dn : _dir_nodes) {
             // Do not allocate new replica on dir_node which is not NORMAL.
             if (dn->status != disk_status::NORMAL) {
+                LOG_INFO("skip the {} state dir_node({})", enum_to_string(dn->status), dn->tag);
                 continue;
             }
             CHECK(!dn->has(pid), "gpid({}) already exists in dir_node({})", pid, dn->tag);
@@ -440,6 +441,7 @@ dir_node *fs_manager::find_replica_dir(absl::string_view app_type, gpid pid)
         for (const auto &dn : _dir_nodes) {
             // Skip IO error dir_node.
             if (dn->status == disk_status::IO_ERROR) {
+                LOG_INFO("skip the {} state dir_node({})", enum_to_string(dn->status), dn->tag);
                 continue;
             }
             const auto dir = dn->replica_dir(app_type, pid);
@@ -498,6 +500,7 @@ dir_node *fs_manager::create_child_replica_dir(absl::string_view app_type,
         for (const auto &dn : _dir_nodes) {
             // Skip non-available dir_node.
             if (dn->status != disk_status::NORMAL) {
+                LOG_INFO("skip the {} state dir_node({})", enum_to_string(dn->status), dn->tag);
                 continue;
             }
             child_dir = dn->replica_dir(app_type, child_pid);

src/failure_detector/test/run.sh

@@ -40,9 +40,9 @@ while read -r -a line; do
         echo "run dsn.failure_detector.tests $test_case failed"
         echo "---- ls ----"
         ls -l
-        if find . -name log.1.txt; then
-            echo "---- tail -n 100 log.1.txt ----"
-            tail -n 100 `find . -name log.1.txt`
+        if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+            echo "---- tail -n 100 pegasus.log.* ----"
+            tail -n 100 `find . -name pegasus.log.*`
         fi
         if [ -f core ]; then
             echo "---- gdb ./dsn.failure_detector.tests core ----"

src/geo/test/CMakeLists.txt

@@ -35,6 +35,10 @@ set(MY_PROJ_LIBS
         s2
         pegasus_client_static
         dsn_utils
+        rocksdb
+        lz4
+        zstd
+        snappy
         gtest)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)

src/meta/test/meta_state/run.sh

@@ -40,9 +40,9 @@ while read -r -a line; do
         echo "run dsn_meta_state_tests $test_case failed"
         echo "---- ls ----"
         ls -l
-        if find . -name log.1.txt; then
-            echo "---- tail -n 100 log.1.txt ----"
-            tail -n 100 `find . -name log.1.txt`
+        if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+            echo "---- tail -n 100 pegasus.log.* ----"
+            tail -n 100 `find . -name pegasus.log.*`
         fi
         if [ -f core ]; then
             echo "---- gdb ./dsn_meta_state_tests core ----"

src/meta/test/misc/misc.cpp

@@ -27,6 +27,7 @@
 #include "misc.h"
 
 #include <boost/lexical_cast.hpp>
+#include <fmt/core.h>
 // IWYU pragma: no_include <ext/alloc_traits.h>
 #include <stdio.h>
 #include <atomic>
@@ -50,6 +51,7 @@
 #include "runtime/rpc/dns_resolver.h" // IWYU pragma: keep
 #include "runtime/rpc/rpc_address.h"
 #include "runtime/rpc/rpc_host_port.h"
+#include "utils/filesystem.h"
 #include "utils/fmt_logging.h"
 #include "utils/rand.h"
 
@@ -195,15 +197,13 @@ void generate_node_fs_manager(const app_mapper &apps,
                               int total_disks)
 {
     nfm.clear();
-    const char *prefix = "/home/work/";
-    char pid_dir[256];
+    std::string prefix;
+    CHECK(dsn::utils::filesystem::get_current_directory(prefix), "");
     std::vector<std::string> data_dirs(total_disks);
     std::vector<std::string> tags(total_disks);
     for (int i = 0; i < data_dirs.size(); ++i) {
-        snprintf(pid_dir, 256, "%sdisk%d", prefix, i + 1);
-        data_dirs[i] = pid_dir;
-        snprintf(pid_dir, 256, "disk%d", i + 1);
-        tags[i] = pid_dir;
+        data_dirs[i] = fmt::format("{}disk{}", prefix, i + 1);
+        tags[i] = fmt::format("disk{}", i + 1);
     }
 
     for (const auto &kv : nodes) {
@@ -215,15 +215,13 @@ void generate_node_fs_manager(const app_mapper &apps,
         manager.initialize(data_dirs, tags);
         ns.for_each_partition([&](const dsn::gpid &pid) {
             const config_context &cc = *get_config_context(apps, pid);
-            snprintf(pid_dir,
-                     256,
-                     "%s%s/%d.%d.test",
-                     prefix,
-                     cc.find_from_serving(ns.host_port())->disk_tag.c_str(),
-                     pid.get_app_id(),
-                     pid.get_partition_index());
-            LOG_DEBUG("concat pid_dir({}) of node({})", pid_dir, ns.host_port());
-            manager.add_replica(pid, pid_dir);
+            const auto dir = fmt::format("{}{}/{}.{}.test",
+                                         prefix,
+                                         cc.find_from_serving(ns.host_port())->disk_tag,
+                                         pid.get_app_id(),
+                                         pid.get_partition_index());
+            LOG_DEBUG("concat {} of node({})", dir, ns.host_port());
+            manager.add_replica(pid, dir);
             return true;
         });
     }

src/meta/test/run.sh

@@ -54,9 +54,9 @@ if [ $? -ne 0 ]; then
     echo "run dsn.meta.test failed"
     echo "---- ls ----"
     ls -l
-    if find . -name log.1.txt; then
-        echo "---- tail -n 100 log.1.txt ----"
-        tail -n 100 `find . -name log.1.txt`
+    if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+            echo "---- tail -n 100 pegasus.log.* ----"
+            tail -n 100 `find . -name pegasus.log.*`
     fi
     if [ -f core ]; then
         echo "---- gdb ./dsn.meta.test core ----"

src/redis_protocol/proxy/CMakeLists.txt

@@ -26,14 +26,18 @@ set(MY_PROJ_SRC "")
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS pegasus.rproxylib
-                 absl::flat_hash_set
-                 absl::strings
-                 pegasus_geo_lib
-                 event
-                 s2
-                 pegasus_client_static
-                 )
+set(MY_PROJ_LIBS
+        pegasus.rproxylib
+        absl::flat_hash_set
+        absl::strings
+        pegasus_geo_lib
+        event
+        s2
+        pegasus_client_static
+        rocksdb
+        lz4
+        zstd
+        snappy)
 
 set(MY_BINPLACES "config.ini")
 

src/redis_protocol/proxy_ut/CMakeLists.txt

@@ -29,15 +29,19 @@ set(MY_SRC_SEARCH_MODE "GLOB")
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem)
 
-set(MY_PROJ_LIBS pegasus.rproxylib
-                pegasus_base
-                absl::flat_hash_set
-                absl::strings
-                pegasus_geo_lib
-                s2
-                pegasus_client_static
-                gtest
-                )
+set(MY_PROJ_LIBS
+        pegasus.rproxylib
+        pegasus_base
+        absl::flat_hash_set
+        absl::strings
+        pegasus_geo_lib
+        s2
+        pegasus_client_static
+        rocksdb
+        lz4
+        zstd
+        snappy
+        gtest)
 
 set(MY_BINPLACES "config.ini" "run.sh")
 

src/replica/backup/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_backup_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_backup_test core -ex "bt"
     fi

src/replica/bulk_load/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_bulk_load_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_bulk_load_test core -ex "bt"
     fi

src/replica/duplication/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_dup_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_dup_test core -ex "bt"
     fi