From 67b5b121bd422a35f5a6d40ba09ea7c4443d15cf Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 21 May 2024 09:53:28 +0800 Subject: [PATCH] feat: Distinguish log file names --- src/failure_detector/test/run.sh | 6 +++--- src/meta/test/meta_state/run.sh | 6 +++--- src/meta/test/run.sh | 6 +++--- src/replica/backup/test/run.sh | 2 +- src/replica/bulk_load/test/run.sh | 2 +- src/replica/duplication/test/run.sh | 2 +- src/replica/split/test/run.sh | 2 +- src/replica/storage/simple_kv/run.sh | 6 +++--- src/replica/storage/simple_kv/test/run.sh | 4 ++-- src/runtime/service_api_c.cpp | 25 ++++++++++++++++++----- src/runtime/test/run.sh | 6 +++--- src/runtime/tracer.cpp | 2 +- src/utils/logging.cpp | 8 +++++--- src/utils/logging_provider.h | 9 ++++---- src/utils/simple_logger.cpp | 7 +++++-- src/utils/simple_logger.h | 5 ++++- src/utils/test/clear.sh | 2 +- src/utils/test/logger.cpp | 4 ++-- src/utils/test/main.cpp | 2 +- src/utils/test/run.sh | 6 +++--- src/zookeeper/test/run.sh | 6 +++--- 21 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/failure_detector/test/run.sh b/src/failure_detector/test/run.sh index be8861ea2b..067eee1d12 100755 --- a/src/failure_detector/test/run.sh +++ b/src/failure_detector/test/run.sh @@ -40,9 +40,9 @@ while read -r -a line; do echo "run dsn.failure_detector.tests $test_case failed" echo "---- ls ----" ls -l - if find . -name log.1.txt; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 `find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find . -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn.failure_detector.tests core ----" diff --git a/src/meta/test/meta_state/run.sh b/src/meta/test/meta_state/run.sh index 62db6b8d26..cb8ad502d0 100755 --- a/src/meta/test/meta_state/run.sh +++ b/src/meta/test/meta_state/run.sh @@ -40,9 +40,9 @@ while read -r -a line; do echo "run dsn_meta_state_tests $test_case failed" echo "---- ls ----" ls -l - if find . -name log.1.txt; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 `find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find . -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn_meta_state_tests core ----" diff --git a/src/meta/test/run.sh b/src/meta/test/run.sh index fe9a116961..a27f485717 100755 --- a/src/meta/test/run.sh +++ b/src/meta/test/run.sh @@ -54,9 +54,9 @@ if [ $? -ne 0 ]; then echo "run dsn.meta.test failed" echo "---- ls ----" ls -l - if find . -name log.1.txt; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 `find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find . -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn.meta.test core ----" diff --git a/src/replica/backup/test/run.sh b/src/replica/backup/test/run.sh index eb88404940..87d08b7b52 100755 --- a/src/replica/backup/test/run.sh +++ b/src/replica/backup/test/run.sh @@ -45,7 +45,7 @@ fi ./dsn_replica_backup_test if [ $? -ne 0 ]; then - tail -n 100 data/log/log.1.txt + tail -n 100 `find . -name pegasus.log.*` if [ -f core ]; then gdb ./dsn_replica_backup_test core -ex "bt" fi diff --git a/src/replica/bulk_load/test/run.sh b/src/replica/bulk_load/test/run.sh index 7689036b02..fff86cd0e0 100755 --- a/src/replica/bulk_load/test/run.sh +++ b/src/replica/bulk_load/test/run.sh @@ -45,7 +45,7 @@ fi ./dsn_replica_bulk_load_test if [ $? -ne 0 ]; then - tail -n 100 data/log/log.1.txt + tail -n 100 `find . -name pegasus.log.*` if [ -f core ]; then gdb ./dsn_replica_bulk_load_test core -ex "bt" fi diff --git a/src/replica/duplication/test/run.sh b/src/replica/duplication/test/run.sh index 90dee780db..bab5b57caa 100755 --- a/src/replica/duplication/test/run.sh +++ b/src/replica/duplication/test/run.sh @@ -45,7 +45,7 @@ fi ./dsn_replica_dup_test if [ $? -ne 0 ]; then - tail -n 100 data/log/log.1.txt + tail -n 100 `find . -name pegasus.log.*` if [ -f core ]; then gdb ./dsn_replica_dup_test core -ex "bt" fi diff --git a/src/replica/split/test/run.sh b/src/replica/split/test/run.sh index bd75b96198..66d4340166 100755 --- a/src/replica/split/test/run.sh +++ b/src/replica/split/test/run.sh @@ -45,7 +45,7 @@ fi ./dsn_replica_split_test if [ $? -ne 0 ]; then - tail -n 100 data/log/log.1.txt + tail -n 100 `find . -name pegasus.log.*` if [ -f core ]; then gdb ./dsn_replica_split_test core -ex "bt" fi diff --git a/src/replica/storage/simple_kv/run.sh b/src/replica/storage/simple_kv/run.sh index cf7c0380c2..debaf5ba88 100755 --- a/src/replica/storage/simple_kv/run.sh +++ b/src/replica/storage/simple_kv/run.sh @@ -62,9 +62,9 @@ if [ -f core ] || ! grep ERR_OK out > /dev/null ; then ls -l echo "---- head -n 100 out ----" head -n 100 out - if [ -f data/logs/log.1.txt ]; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 data/logs/log.1.txt + if [ `find data/logs -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find data/logs -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn.replication.simple_kv core ----" diff --git a/src/replica/storage/simple_kv/test/run.sh b/src/replica/storage/simple_kv/test/run.sh index 9c8d07328b..d09bf716d0 100755 --- a/src/replica/storage/simple_kv/test/run.sh +++ b/src/replica/storage/simple_kv/test/run.sh @@ -52,8 +52,8 @@ function run_single() echo "${bin} ${prefix}.ini ${prefix}.act" ${bin} ${prefix}.ini ${prefix}.act ret=$? - if find . -name log.1.txt &>/dev/null; then - log=`find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + log=`find . -name pegasus.log.*` cat ${log} | grep -v FAILURE_DETECT | grep -v BEACON | grep -v beacon | grep -v THREAD_POOL_FD >${prefix}.log rm ${log} fi diff --git a/src/runtime/service_api_c.cpp b/src/runtime/service_api_c.cpp index fb4e5154eb..276fec4f3a 100644 --- a/src/runtime/service_api_c.cpp +++ b/src/runtime/service_api_c.cpp @@ -45,6 +45,7 @@ #endif #include "fmt/core.h" +#include "fmt/format.h" #include "perf_counter/perf_counters.h" #include "runtime/api_layer1.h" #include "runtime/api_task.h" @@ -457,8 +458,25 @@ bool run(const char *config_file, ::MallocExtension::instance()->SetMemoryReleaseRate(FLAGS_tcmalloc_release_rate); #endif - // init logging - dsn_log_init(spec.logging_factory_name, spec.log_dir, dsn_log_prefixed_message_func); + // Extract app_names. + std::list app_names_and_indexes; + ::dsn::utils::split_args(app_list.c_str(), app_names_and_indexes, ';'); + std::vector app_names; + for (const auto &app_name_and_index : app_names_and_indexes) { + std::vector name_and_index; + ::dsn::utils::split_args(app_name_and_index.c_str(), name_and_index, '@'); + if (name_and_index.empty()) { + fmt::print(stderr, "app_name should be specified in '{}'", app_name_and_index); + return false; + } + app_names.push_back(name_and_index[0]); + } + + // Initialize logging. + dsn_log_init(spec.logging_factory_name, + spec.log_dir, + fmt::format("{}", fmt::join(app_names, ".")), + dsn_log_prefixed_message_func); // Prepare the minimum necessary. ::dsn::service_engine::instance().init_before_toollets(spec); @@ -513,9 +531,6 @@ bool run(const char *config_file, if (app_list.empty()) { create_it = true; } else { - // Extract app_names. - std::list app_names_and_indexes; - ::dsn::utils::split_args(app_list.c_str(), app_names_and_indexes, ';'); for (const auto &app_name_and_index : app_names_and_indexes) { std::vector name_and_index; ::dsn::utils::split_args(app_name_and_index.c_str(), name_and_index, '@'); diff --git a/src/runtime/test/run.sh b/src/runtime/test/run.sh index 648ccb41b5..ebb7925b6a 100755 --- a/src/runtime/test/run.sh +++ b/src/runtime/test/run.sh @@ -40,9 +40,9 @@ while read -r -a line; do echo "run dsn_runtime_tests $test_case failed" echo "---- ls ----" ls -l - if find . -name log.1.txt; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 `find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find . -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn_runtime_tests core ----" diff --git a/src/runtime/tracer.cpp b/src/runtime/tracer.cpp index 2c1d24ec6f..e85d88ed5f 100644 --- a/src/runtime/tracer.cpp +++ b/src/runtime/tracer.cpp @@ -408,7 +408,7 @@ void tracer::install(service_spec &spec) "tracer.find", "Find related logs", "[forward|f|backward|b] [rpc|r|task|t] [trace_id|task_id(e.g., a023003920302390)] " - "", + "", tracer_log_flow); }); } diff --git a/src/utils/logging.cpp b/src/utils/logging.cpp index 0f0484c76d..1a3176e6fa 100644 --- a/src/utils/logging.cpp +++ b/src/utils/logging.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "runtime/tool_api.h" #include "simple_logger.h" @@ -61,7 +62,7 @@ std::function log_prefixed_message_func = []() -> std::string { r void set_log_prefixed_message_func(std::function func) { - log_prefixed_message_func = func; + log_prefixed_message_func = std::move(func); } } // namespace dsn @@ -73,7 +74,8 @@ static void log_on_sys_exit(::dsn::sys_exit_type) void dsn_log_init(const std::string &logging_factory_name, const std::string &log_dir, - std::function dsn_log_prefixed_message_func) + const std::string &role_name, + const std::function &dsn_log_prefixed_message_func) { log_start_level = enum_from_string(FLAGS_logging_start_level, LOG_LEVEL_INVALID); @@ -86,7 +88,7 @@ void dsn_log_init(const std::string &logging_factory_name, } dsn::logging_provider *logger = dsn::utils::factory_store::create( - logging_factory_name.c_str(), dsn::PROVIDER_TYPE_MAIN, log_dir.c_str()); + logging_factory_name.c_str(), dsn::PROVIDER_TYPE_MAIN, log_dir.c_str(), role_name.c_str()); dsn::logging_provider::set_logger(logger); if (dsn_log_prefixed_message_func != nullptr) { diff --git a/src/utils/logging_provider.h b/src/utils/logging_provider.h index 910e418a14..8adec47433 100644 --- a/src/utils/logging_provider.h +++ b/src/utils/logging_provider.h @@ -40,12 +40,12 @@ class logging_provider { public: template - static logging_provider *create(const char *log_dir) + static logging_provider *create(const char *log_dir, const char *role_name) { - return new T(log_dir); + return new T(log_dir, role_name); } - typedef logging_provider *(*factory)(const char *); + typedef logging_provider *(*factory)(const char *, const char *); public: virtual ~logging_provider() = default; @@ -88,4 +88,5 @@ bool register_component_provider(const char *name, extern void dsn_log_init(const std::string &logging_factory_name, const std::string &log_dir, - std::function dsn_log_prefixed_message_func); + const std::string &role_name, + const std::function &dsn_log_prefixed_message_func); diff --git a/src/utils/simple_logger.cpp b/src/utils/simple_logger.cpp index 2765f158b1..cb853f4ba7 100644 --- a/src/utils/simple_logger.cpp +++ b/src/utils/simple_logger.cpp @@ -88,6 +88,8 @@ DSN_DEFINE_validator(stderr_start_level, [](const char *value) -> bool { return LOG_LEVEL_DEBUG <= level && level <= LOG_LEVEL_FATAL; }); +DSN_DEFINE_string(tools.simple_logger, base_name, "pegasus", "The default base name for log file"); + DSN_DECLARE_string(logging_start_level); namespace dsn { @@ -204,14 +206,15 @@ void screen_logger::log( void screen_logger::flush() { ::fflush(stdout); } -simple_logger::simple_logger(const char *log_dir) +simple_logger::simple_logger(const char *log_dir, const char *role_name) : _log_dir(std::string(log_dir)), _log(nullptr), _file_bytes(0), _stderr_start_level(enum_from_string(FLAGS_stderr_start_level, LOG_LEVEL_INVALID)) { // Use 'role_name' if it is specified, otherwise use 'base_name'. - const std::string symlink_name("log"); + const std::string symlink_name( + fmt::format("{}.log", utils::is_empty(role_name) ? FLAGS_base_name : role_name)); _file_name_prefix = fmt::format("{}.", symlink_name); _symlink_path = utils::filesystem::path_combine(_log_dir, symlink_name); diff --git a/src/utils/simple_logger.h b/src/utils/simple_logger.h index 3a8b0015ab..d55320c929 100644 --- a/src/utils/simple_logger.h +++ b/src/utils/simple_logger.h @@ -45,6 +45,7 @@ class screen_logger : public logging_provider { public: explicit screen_logger(bool short_header); + explicit screen_logger(const char *, const char *) : _short_header(true) {} ~screen_logger() override = default; void log(const char *file, @@ -74,7 +75,9 @@ class screen_logger : public logging_provider class simple_logger : public logging_provider { public: - simple_logger(const char *log_dir); + // 'log_dir' is the directory to store log files, 'role_name' is the name of the process, + // such as 'replica_server', 'meta_server' in Pegasus. + simple_logger(const char *log_dir, const char *role_name); ~simple_logger() override; void log(const char *file, diff --git a/src/utils/test/clear.sh b/src/utils/test/clear.sh index d113af3dbb..cc0c873c1b 100755 --- a/src/utils/test/clear.sh +++ b/src/utils/test/clear.sh @@ -24,4 +24,4 @@ # THE SOFTWARE. -rm -rf dsn.utils.tests.xml log*.txt +rm -rf dsn.utils.tests.xml pegasus*.txt diff --git a/src/utils/test/logger.cpp b/src/utils/test/logger.cpp index 5e0f59a285..8f07b5736a 100644 --- a/src/utils/test/logger.cpp +++ b/src/utils/test/logger.cpp @@ -81,7 +81,7 @@ class simple_logger_test : public logger_test ASSERT_TRUE(utils::filesystem::get_subfiles(test_dir, sub_list, false)); file_names.clear(); - std::regex pattern(R"(log\.[0-9]{8}_[0-9]{6}_[0-9]{3})"); + std::regex pattern(R"(SimpleLogger\.log\.[0-9]{8}_[0-9]{6}_[0-9]{3})"); for (const auto &path : sub_list) { std::string name(utils::filesystem::get_file_name(path)); if (std::regex_match(name, pattern)) { @@ -161,7 +161,7 @@ TEST_F(simple_logger_test, redundant_log_test) std::set before_files; NO_FATALS(get_log_files(before_files)); - auto logger = std::make_unique(test_dir.c_str()); + auto logger = std::make_unique(test_dir.c_str(), "SimpleLogger"); for (unsigned int i = 0; i != 1000; ++i) { LOG_PRINT(logger.get(), "{}", "test_print"); } diff --git a/src/utils/test/main.cpp b/src/utils/test/main.cpp index 2be6302e1a..0bdfb7f7a5 100644 --- a/src/utils/test/main.cpp +++ b/src/utils/test/main.cpp @@ -25,7 +25,7 @@ GTEST_API_ int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); - dsn_log_init("dsn::tools::simple_logger", "./", nullptr); + dsn_log_init("dsn::tools::simple_logger", "./", "test", nullptr); dsn::flags_initialize(); diff --git a/src/utils/test/run.sh b/src/utils/test/run.sh index 90698a718f..06c8b72346 100755 --- a/src/utils/test/run.sh +++ b/src/utils/test/run.sh @@ -36,9 +36,9 @@ if [ $? -ne 0 ]; then echo "run dsn_utils_tests failed" echo "---- ls ----" ls -l - if find . -name log.1.txt; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 `find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find . -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn_utils_tests core ----" diff --git a/src/zookeeper/test/run.sh b/src/zookeeper/test/run.sh index 38608a28b3..820dbdab3c 100755 --- a/src/zookeeper/test/run.sh +++ b/src/zookeeper/test/run.sh @@ -37,9 +37,9 @@ if [ $? -ne 0 ]; then echo "run dsn.zookeeper.tests failed" echo "---- ls ----" ls -l - if find . -name log.1.txt; then - echo "---- tail -n 100 log.1.txt ----" - tail -n 100 `find . -name log.1.txt` + if [ `find . -name pegasus.log.* | wc -l` -ne 0 ]; then + echo "---- tail -n 100 pegasus.log.* ----" + tail -n 100 `find . -name pegasus.log.*` fi if [ -f core ]; then echo "---- gdb ./dsn.zookeeper.tests core ----"