feat: Distinguish log file names and limit log file count

src/failure_detector/test/run.sh

@@ -40,9 +40,9 @@ while read -r -a line; do
         echo "run dsn.failure_detector.tests $test_case failed"
         echo "---- ls ----"
         ls -l
-        if find . -name log.1.txt; then
-            echo "---- tail -n 100 log.1.txt ----"
-            tail -n 100 `find . -name log.1.txt`
+        if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+            echo "---- tail -n 100 pegasus.log.* ----"
+            tail -n 100 `find . -name pegasus.log.*`
         fi
         if [ -f core ]; then
             echo "---- gdb ./dsn.failure_detector.tests core ----"

src/meta/test/meta_state/run.sh

@@ -40,9 +40,9 @@ while read -r -a line; do
         echo "run dsn_meta_state_tests $test_case failed"
         echo "---- ls ----"
         ls -l
-        if find . -name log.1.txt; then
-            echo "---- tail -n 100 log.1.txt ----"
-            tail -n 100 `find . -name log.1.txt`
+        if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+            echo "---- tail -n 100 pegasus.log.* ----"
+            tail -n 100 `find . -name pegasus.log.*`
         fi
         if [ -f core ]; then
             echo "---- gdb ./dsn_meta_state_tests core ----"

src/meta/test/run.sh

@@ -54,9 +54,9 @@ if [ $? -ne 0 ]; then
     echo "run dsn.meta.test failed"
     echo "---- ls ----"
     ls -l
-    if find . -name log.1.txt; then
-        echo "---- tail -n 100 log.1.txt ----"
-        tail -n 100 `find . -name log.1.txt`
+    if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+            echo "---- tail -n 100 pegasus.log.* ----"
+            tail -n 100 `find . -name pegasus.log.*`
     fi
     if [ -f core ]; then
         echo "---- gdb ./dsn.meta.test core ----"

src/replica/backup/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_backup_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_backup_test core -ex "bt"
     fi

src/replica/bulk_load/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_bulk_load_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_bulk_load_test core -ex "bt"
     fi

src/replica/duplication/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_dup_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_dup_test core -ex "bt"
     fi

src/replica/split/test/run.sh

@@ -45,7 +45,7 @@ fi
 ./dsn_replica_split_test
 
 if [ $? -ne 0 ]; then
-    tail -n 100 data/log/log.1.txt
+    tail -n 100 `find . -name pegasus.log.*`
     if [ -f core ]; then
         gdb ./dsn_replica_split_test core -ex "bt"
     fi

src/replica/storage/simple_kv/run.sh

@@ -62,9 +62,9 @@ if [ -f core ] || ! grep ERR_OK out > /dev/null ; then
     ls -l
     echo "---- head -n 100 out ----"
     head -n 100 out
-    if [ -f data/logs/log.1.txt ]; then
-        echo "---- tail -n 100 log.1.txt ----"
-        tail -n 100 data/logs/log.1.txt
+    if [ `find data/logs -name pegasus.log.* | wc -l` -ne 0 ]; then
+        echo "---- tail -n 100 pegasus.log.* ----"
+        tail -n 100 `find data/logs -name pegasus.log.*`
     fi
     if [ -f core ]; then
         echo "---- gdb ./dsn.replication.simple_kv core ----"

src/replica/storage/simple_kv/test/run.sh

@@ -52,8 +52,8 @@ function run_single()
     echo "${bin} ${prefix}.ini ${prefix}.act"
     ${bin} ${prefix}.ini ${prefix}.act
     ret=$?
-    if find . -name log.1.txt &>/dev/null; then
-        log=`find . -name log.1.txt`
+    if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then
+        log=`find . -name pegasus.log.*`
         cat ${log} | grep -v FAILURE_DETECT | grep -v BEACON | grep -v beacon | grep -v THREAD_POOL_FD >${prefix}.log
         rm ${log}
     fi

src/runtime/global_config.h

@@ -169,8 +169,7 @@ struct service_spec
     std::vector<threadpool_spec> threadpool_specs;
     std::vector<service_app_spec> app_specs;
 
-    // auto-set
-    std::string dir_log;
+    std::string log_dir;
 
     service_spec() {}
     bool init();
@@ -186,6 +185,11 @@ CONFIG_FLD_STRING_LIST(toollets,
 CONFIG_FLD_STRING(data_dir,
                   "./data",
                   "The default directory to place the all the data, logs, coredump files, and etc.")
+CONFIG_FLD_STRING_BY_KEY(
+    log_dir,
+    "log_dir",
+    "",
+    "The directory to place the logs especially. 'data_dir' will be used if it's empty.")
 CONFIG_FLD(
     bool,
     bool,

src/runtime/service_api_c.cpp

@@ -443,40 +443,66 @@ bool run(const char *config_file,
 
     ::dsn::utils::coredump::init();
 
-    // setup log dir
-    spec.dir_log = ::dsn::utils::filesystem::path_combine(cdir, "log");
-    dsn::utils::filesystem::create_directory(spec.dir_log);
+    // Setup log directory.
+    // If log_dir is not set, use data_dir/log instead.
+    if (spec.log_dir.empty()) {
+        spec.log_dir = ::dsn::utils::filesystem::path_combine(spec.data_dir, "log");
+        fmt::print(stdout, "log_dir is not set, use '{}' instead\n", spec.log_dir);
+    }
+    // Validate log_dir.
+    if (!dsn::utils::filesystem::is_absolute_path(spec.log_dir)) {
+        fmt::print(stderr, "log_dir({}) should be set with an absolute path\n", spec.log_dir);
+        return false;
+    }
+    dsn::utils::filesystem::create_directory(spec.log_dir);
 
-    // init tools
+    // Initialize tools.
     dsn_all.tool.reset(::dsn::utils::factory_store<::dsn::tools::tool_app>::create(
         spec.tool.c_str(), ::dsn::PROVIDER_TYPE_MAIN, spec.tool.c_str()));
     dsn_all.tool->install(spec);
 
-    // init app specs
+    // Initialize app specs.
     if (!spec.init_app_specs()) {
-        printf("error in config file %s, exit ...\n", config_file);
+        fmt::print(stderr, "error in config file {}, exit ...\n", config_file);
         return false;
     }
 
 #ifdef DSN_ENABLE_GPERF
     ::MallocExtension::instance()->SetMemoryReleaseRate(FLAGS_tcmalloc_release_rate);
 #endif
 
-    // init logging
-    dsn_log_init(spec.logging_factory_name, spec.dir_log, dsn_log_prefixed_message_func);
+    // Extract app_names.
+    std::list<std::string> app_names_and_indexes;
+    ::dsn::utils::split_args(app_list.c_str(), app_names_and_indexes, ';');
+    std::vector<std::string> app_names;
+    for (const auto &app_name_and_index : app_names_and_indexes) {
+        std::vector<std::string> name_and_index;
+        ::dsn::utils::split_args(app_name_and_index.c_str(), name_and_index, '@');
+        if (name_and_index.empty()) {
+            fmt::print(stderr, "app_name should be specified in '{}'", app_name_and_index);
+            return false;
+        }
+        app_names.push_back(name_and_index[0]);
+    }
+
+    // Initialize logging.
+    dsn_log_init(spec.logging_factory_name,
+                 spec.log_dir,
+                 fmt::format("{}", fmt::join(app_names, ".")),
+                 dsn_log_prefixed_message_func);
 
-    // prepare minimum necessary
+    // Prepare the minimum necessary.
     ::dsn::service_engine::instance().init_before_toollets(spec);
 
     LOG_INFO("process({}) start: {}, date: {}",
              getpid(),
              dsn::utils::process_start_millis(),
              dsn::utils::process_start_date_time_mills());
 
-    // init toollets
-    for (auto it = spec.toollets.begin(); it != spec.toollets.end(); ++it) {
-        auto tlet =
-            dsn::tools::internal_use_only::get_toollet(it->c_str(), ::dsn::PROVIDER_TYPE_MAIN);
+    // Initialize toollets.
+    for (const auto &toollet_name : spec.toollets) {
+        auto tlet = dsn::tools::internal_use_only::get_toollet(toollet_name.c_str(),
+                                                               ::dsn::PROVIDER_TYPE_MAIN);
         CHECK_NOTNULL(tlet, "toolet not found");
         tlet->install(spec);
     }
@@ -507,29 +533,34 @@ bool run(const char *config_file,
         }
     }
 
-    // split app_name and app_index
-    std::list<std::string> applistkvs;
-    ::dsn::utils::split_args(app_list.c_str(), applistkvs, ';');
-
     // init apps
     for (auto &sp : spec.app_specs) {
-        if (!sp.run)
+        if (!sp.run) {
             continue;
+        }
 
         bool create_it = false;
-
         // create all apps
-        if (app_list == "") {
+        if (app_list.empty()) {
             create_it = true;
         } else {
-            for (auto &kv : applistkvs) {
-                std::list<std::string> argskvs;
-                ::dsn::utils::split_args(kv.c_str(), argskvs, '@');
-                if (std::string("apps.") + argskvs.front() == sp.config_section) {
-                    if (argskvs.size() < 2)
+            for (const auto &app_name_and_index : app_names_and_indexes) {
+                std::vector<std::string> name_and_index;
+                ::dsn::utils::split_args(app_name_and_index.c_str(), name_and_index, '@');
+                CHECK(!name_and_index.empty(),
+                      "app_name should be specified in '{}'",
+                      app_name_and_index);
+                if (std::string("apps.") + name_and_index.front() == sp.config_section) {
+                    if (name_and_index.size() < 2) {
                         create_it = true;
-                    else
-                        create_it = (std::stoi(argskvs.back()) == sp.index);
+                    } else {
+                        int32_t index;
+                        const auto index_str = name_and_index.back();
+                        CHECK(dsn::buf2int32(index_str, index),
+                              "'{}' is not an valid index",
+                              index_str);
+                        create_it = (index == sp.index);
+                    }
                     break;
                 }
             }
@@ -540,10 +571,11 @@ bool run(const char *config_file,
         }
     }
 
-    if (dsn::service_engine::instance().get_all_nodes().size() == 0) {
-        printf("no app are created, usually because \n"
-               "app_name is not specified correctly, should be 'xxx' in [apps.xxx]\n"
-               "or app_index (1-based) is greater than specified count in config file\n");
+    if (dsn::service_engine::instance().get_all_nodes().empty()) {
+        fmt::print(stderr,
+                   "no app are created, usually because \n"
+                   "app_name is not specified correctly, should be 'xxx' in [apps.xxx]\n"
+                   "or app_index (1-based) is greater than specified count in config file\n");
         exit(1);
     }
 

src/utils/config_helper.h

@@ -49,9 +49,12 @@
         section, #fld, default_value ? default_value->fld : default_fld_value, dsptr);
 
 #define CONFIG_FLD_STRING(fld, default_fld_value, dsptr)                                           \
+    CONFIG_FLD_STRING_BY_KEY(fld, #fld, default_fld_value, dsptr)
+
+#define CONFIG_FLD_STRING_BY_KEY(fld, key, default_fld_value, dsptr)                               \
     val.fld = dsn_config_get_value_string(                                                         \
         section,                                                                                   \
-        #fld,                                                                                      \
+        key,                                                                                       \
         (val.fld.length() > 0 && val.fld != std::string(default_fld_value))                        \
             ? val.fld.c_str()                                                                      \
             : (default_value ? default_value->fld.c_str() : default_fld_value),                    \

src/utils/filesystem.cpp

@@ -29,6 +29,7 @@
 #include <errno.h>
 #include <fmt/core.h>
 #include <ftw.h>
+#include <glob.h>
 #include <limits.h>
 #include <openssl/md5.h>
 #include <rocksdb/env.h>
@@ -49,6 +50,7 @@
 #include "utils/fmt_logging.h"
 #include "utils/ports.h"
 #include "utils/safe_strerror_posix.h"
+#include "errors.h"
 
 #define getcwd_ getcwd
 #define rmdir_ rmdir
@@ -492,6 +494,12 @@ bool create_file(const std::string &path)
     return true;
 }
 
+bool is_absolute_path(const std::string &path)
+{
+    boost::filesystem::path p(path);
+    return p.is_absolute();
+}
+
 bool get_absolute_path(const std::string &path1, std::string &path2)
 {
     bool succ;
@@ -901,6 +909,36 @@ bool check_dir_rw(const std::string &path, std::string &err_msg)
     return true;
 }
 
+error_s glob(const std::string &path_pattern, std::vector<std::string> &path_list)
+{
+    glob_t result;
+    auto cleanup = dsn::defer([&] { ::globfree(&result); });
+
+    errno = 0;
+    int ret = ::glob(path_pattern.c_str(), GLOB_TILDE | GLOB_ERR, NULL, &result);
+    switch (ret) {
+    case 0:
+        break;
+
+    case GLOB_NOMATCH:
+        return error_s::ok();
+
+    case GLOB_NOSPACE:
+        return error_s::make(ERR_FS_INTERNAL, "glob out of memory");
+
+    default:
+        std::string error(errno == 0 ? "unknown error" : safe_strerror(errno));
+        return error_s::make(ERR_FS_INTERNAL,
+                             fmt::format("glob failed for '{}': {}", path_pattern, error));
+    }
+
+    for (size_t i = 0; i < result.gl_pathc; ++i) {
+        path_list.emplace_back(result.gl_pathv[i]);
+    }
+
+    return error_s::ok();
+}
+
 } // namespace filesystem
 } // namespace utils
 } // namespace dsn

src/utils/filesystem.h

@@ -32,6 +32,7 @@
 #include <utility>
 #include <vector>
 
+#include "utils/errors.h"
 #include "utils/error_code.h"
 
 #ifndef _XOPEN_SOURCE
@@ -69,6 +70,8 @@ namespace filesystem {
 
 void get_normalized_path(const std::string &path, std::string &npath);
 
+bool is_absolute_path(const std::string &path);
+
 bool get_absolute_path(const std::string &path1, std::string &path2);
 
 std::string remove_file_name(const std::string &path);
@@ -165,6 +168,13 @@ bool create_directory(const std::string &path,
 // call `create_directory` before to make `path` exist
 bool check_dir_rw(const std::string &path, /*out*/ std::string &err_msg);
 
+// Finds paths on the filesystem matching a pattern.
+//
+// The found pathnames are added to the 'paths' vector. If no pathnames are
+// found matching the pattern, no paths are added to the vector and an OK
+// status is returned.
+error_s glob(const std::string &path_pattern, std::vector<std::string> &path_list);
+
 } // namespace filesystem
 } // namespace utils
 } // namespace dsn

src/utils/logging.cpp

@@ -72,7 +72,8 @@ static void log_on_sys_exit(::dsn::sys_exit_type)
 }
 
 void dsn_log_init(const std::string &logging_factory_name,
-                  const std::string &dir_log,
+                  const std::string &log_dir,
+                  const std::string &role_name,
                   std::function<std::string()> dsn_log_prefixed_message_func)
 {
     log_start_level = enum_from_string(FLAGS_logging_start_level, LOG_LEVEL_INVALID);
@@ -86,7 +87,7 @@ void dsn_log_init(const std::string &logging_factory_name,
     }
 
     dsn::logging_provider *logger = dsn::utils::factory_store<dsn::logging_provider>::create(
-        logging_factory_name.c_str(), dsn::PROVIDER_TYPE_MAIN, dir_log.c_str());
+        logging_factory_name.c_str(), dsn::PROVIDER_TYPE_MAIN, log_dir.c_str(), role_name.c_str());
     dsn::logging_provider::set_logger(logger);
 
     if (dsn_log_prefixed_message_func != nullptr) {