From 4359683d4cf68bc4ad504c03d4801c1f608d24ce Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Fri, 19 Aug 2022 23:02:51 +0800 Subject: [PATCH 01/11] feat: support to dump jemalloc stats --- src/rdsn/src/replica/replica_stub.cpp | 52 ++++++++++++++ src/rdsn/src/replica/replica_stub.h | 2 + src/rdsn/src/utils/je_ctl.cpp | 98 +++++++++++++++++++++++++++ src/rdsn/src/utils/je_ctl.h | 40 +++++++++++ 4 files changed, 192 insertions(+) create mode 100644 src/rdsn/src/utils/je_ctl.cpp create mode 100644 src/rdsn/src/utils/je_ctl.h diff --git a/src/rdsn/src/replica/replica_stub.cpp b/src/rdsn/src/replica/replica_stub.cpp index 9e7625e648..81e8889c4d 100644 --- a/src/rdsn/src/replica/replica_stub.cpp +++ b/src/rdsn/src/replica/replica_stub.cpp @@ -57,6 +57,8 @@ #include #ifdef DSN_ENABLE_GPERF #include +#elif defined(DSN_USE_JEMALLOC) +#include "utils/je_ctl.h" #endif #include #include @@ -106,6 +108,11 @@ replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/, _get_tcmalloc_status_command = nullptr; _max_reserved_memory_percentage_command = nullptr; _release_all_reserved_memory_command = nullptr; +#elif defined(DSN_USE_JEMALLOC) + _get_jemalloc_summary_stats_command = nullptr; + _get_jemalloc_configs_command = nullptr; + _get_jemalloc_brief_arena_stats_command = nullptr; + _get_jemalloc_detailed_stats_command = nullptr; #endif _replica_state_subscriber = subscriber; _is_long_subscriber = is_long_subscriber; @@ -2261,6 +2268,51 @@ void replica_stub::open_service() register_ctrl_command(); } +#if !defined(DSN_ENABLE_GPERF) && defined(DSN_USE_JEMALLOC) +void replica_stub::register_jemalloc_ctrl_command() +{ + _get_jemalloc_configs_command = ::dsn::command_manager::instance().register_command( + {"replica.get-jemalloc-configs"}, + "replica.get-jemalloc-configs - get configs of jemalloc", + "get configs of jemalloc", + [](const std::vector &args) { + std::string stats("\n"); + dsn::utils::je_dump_configs(&stats); + return stats; + }); + + _get_jemalloc_summary_stats_command = ::dsn::command_manager::instance().register_command( + {"replica.get-jemalloc-summary-stats"}, + "replica.get-jemalloc-summary-stats - get summary stats of jemalloc", + "get summary stats of jemalloc", + [](const std::vector &args) { + std::string stats("\n"); + dsn::utils::je_dump_summary_stats(&stats); + return stats; + }); + + _get_jemalloc_brief_arena_stats_command = ::dsn::command_manager::instance().register_command( + {"replica.get-jemalloc-brief-arena-stats"}, + "replica.get-jemalloc-brief-arena-stats - get brief_arena stats of jemalloc", + "get brief arena stats of jemalloc", + [](const std::vector &args) { + std::string stats("\n"); + dsn::utils::je_dump_brief_arena_stats(&stats); + return stats; + }); + + _get_jemalloc_detailed_stats_command = ::dsn::command_manager::instance().register_command( + {"replica.get-jemalloc-detailed-stats"}, + "replica.get-jemalloc-detailed-stats - get detailed stats of jemalloc", + "get detailed stats of jemalloc", + [](const std::vector &args) { + std::string stats("\n"); + dsn::utils::je_dump_detailed_stats(&stats); + return stats; + }); +} +#endif + void replica_stub::register_ctrl_command() { /// In simple_kv test, three replica apps are created, which means that three replica_stubs are diff --git a/src/rdsn/src/replica/replica_stub.h b/src/rdsn/src/replica/replica_stub.h index 32a47584b4..9b37ffda88 100644 --- a/src/rdsn/src/replica/replica_stub.h +++ b/src/rdsn/src/replica/replica_stub.h @@ -307,6 +307,8 @@ class replica_stub : public serverlet, public ref_counter // Try to release tcmalloc memory back to operating system // If release_all = true, it will release all reserved-not-used memory uint64_t gc_tcmalloc_memory(bool release_all); +#elif defined(DSN_USE_JEMALLOC) + void register_jemalloc_ctrl_command(); #endif private: diff --git a/src/rdsn/src/utils/je_ctl.cpp b/src/rdsn/src/utils/je_ctl.cpp new file mode 100644 index 0000000000..dc1edb172e --- /dev/null +++ b/src/rdsn/src/utils/je_ctl.cpp @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef DSN_USE_JEMALLOC + +#include "je_ctl.h" + +#include + +#include + +#include +#include + +#define RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, array) \ + do { \ + const auto index = static_cast(type);\ + dcheck_lt(index, sizeof(array) / sizeof(array[0]));\ + return array[index];\ + } while (0); + +namespace dsn { + +namespace { + +template + +void je_stats_cb(void *opaque, const char *str) +{ + if (str == nullptr) { + return; + } + + auto stats = reinterpret_cast(opaque); + auto avail_capacity = stats->capacity() - stats->size(); + auto len = strlen(str); + if (len > avail_capacity) { + len = avail_capacity; + } + + stats->append(str, len); +} + +void je_dump_malloc_stats(const char *opts, size_t buf_sz, std::string &stats) +{ + // Avoid malloc in callback. + stats.reserve(buf_sz); + + malloc_stats_print(je_stats_cb, &stats, opts); +} + +const char *je_stats_type_to_opts(je_stats_type type) +{ + static const char *opts_map[] = { + "gmdablxe", "mdablxe", "gblxe", "", + }; + + RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, opts_map); +} + +size_t je_stats_type_to_default_buf_sz(je_stats_type type) +{ + static const size_t buf_sz_map[] = { + 2 * 1024, 4 * 1024, 1024 * 1024, 2 * 1024 * 1024, + }; + + RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, buf_sz_map); +} + +} // anonymous namespace + +void je_dump_stats(je_stats_type type, size_t buf_sz, std::string &stats) +{ + je_dump_malloc_stats(je_stats_type_to_opts(type), buf_sz, stats); +} + +void je_dump_stats(je_stats_type type, std::string &stats) +{ + je_dump_stats(type, je_stats_type_to_default_buf_sz(type), stats); +} + +} // namespace dsn + +#endif // DSN_USE_JEMALLOC diff --git a/src/rdsn/src/utils/je_ctl.h b/src/rdsn/src/utils/je_ctl.h new file mode 100644 index 0000000000..88a5781e92 --- /dev/null +++ b/src/rdsn/src/utils/je_ctl.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#ifdef DSN_USE_JEMALLOC + +#include + +namespace dsn { + +enum class je_stats_type : size_t +{ + kSummaryStats, + kConfigs, + kBriefArenaStats, + kDetailedStats, +}; + +void je_dump_stats(je_stats_type type, size_t buf_sz, std::string &stats); + +void je_dump_stats(je_stats_type type, std::string &stats); + +} // namespace dsn + +#endif // DSN_USE_JEMALLOC From 537311966915bc9b47ebe7dfb208c1d2a9cb68fc Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Sat, 20 Aug 2022 00:32:54 +0800 Subject: [PATCH 02/11] feat: support to dump jemalloc stats --- src/rdsn/src/replica/replica_stub.cpp | 65 ++++++++++++--------------- src/rdsn/src/replica/replica_stub.h | 2 + src/rdsn/src/utils/je_ctl.cpp | 21 ++++++--- src/rdsn/src/utils/je_ctl.h | 22 +++++++-- 4 files changed, 65 insertions(+), 45 deletions(-) diff --git a/src/rdsn/src/replica/replica_stub.cpp b/src/rdsn/src/replica/replica_stub.cpp index 81e8889c4d..c87a81da1d 100644 --- a/src/rdsn/src/replica/replica_stub.cpp +++ b/src/rdsn/src/replica/replica_stub.cpp @@ -109,10 +109,7 @@ replica_stub::replica_stub(replica_state_subscriber subscriber /*= nullptr*/, _max_reserved_memory_percentage_command = nullptr; _release_all_reserved_memory_command = nullptr; #elif defined(DSN_USE_JEMALLOC) - _get_jemalloc_summary_stats_command = nullptr; - _get_jemalloc_configs_command = nullptr; - _get_jemalloc_brief_arena_stats_command = nullptr; - _get_jemalloc_detailed_stats_command = nullptr; + _dump_jemalloc_stats_command = nullptr; #endif _replica_state_subscriber = subscriber; _is_long_subscriber = is_long_subscriber; @@ -2271,43 +2268,33 @@ void replica_stub::open_service() #if !defined(DSN_ENABLE_GPERF) && defined(DSN_USE_JEMALLOC) void replica_stub::register_jemalloc_ctrl_command() { - _get_jemalloc_configs_command = ::dsn::command_manager::instance().register_command( - {"replica.get-jemalloc-configs"}, - "replica.get-jemalloc-configs - get configs of jemalloc", - "get configs of jemalloc", + _dump_jemalloc_stats_command = ::dsn::command_manager::instance().register_command( + {"replica.dump-jemalloc-stats"}, + fmt::format("replica.dump-jemalloc-stats <{}> [buf_sz]", kAllKthPercentileTypes), + "dump stats of jemalloc", [](const std::vector &args) { - std::string stats("\n"); - dsn::utils::je_dump_configs(&stats); - return stats; - }); + if (args.empty()) { + return std::string("ERR: invalid arguments"); + } - _get_jemalloc_summary_stats_command = ::dsn::command_manager::instance().register_command( - {"replica.get-jemalloc-summary-stats"}, - "replica.get-jemalloc-summary-stats - get summary stats of jemalloc", - "get summary stats of jemalloc", - [](const std::vector &args) { - std::string stats("\n"); - dsn::utils::je_dump_summary_stats(&stats); - return stats; - }); + auto type = enum_from_string(args[0].c_str(), je_stats_type::INVALID); + if (type == je_stats_type::INVALID) { + return std::string("ERR: invalid stats type"); + } - _get_jemalloc_brief_arena_stats_command = ::dsn::command_manager::instance().register_command( - {"replica.get-jemalloc-brief-arena-stats"}, - "replica.get-jemalloc-brief-arena-stats - get brief_arena stats of jemalloc", - "get brief arena stats of jemalloc", - [](const std::vector &args) { std::string stats("\n"); - dsn::utils::je_dump_brief_arena_stats(&stats); - return stats; - }); - _get_jemalloc_detailed_stats_command = ::dsn::command_manager::instance().register_command( - {"replica.get-jemalloc-detailed-stats"}, - "replica.get-jemalloc-detailed-stats - get detailed stats of jemalloc", - "get detailed stats of jemalloc", - [](const std::vector &args) { - std::string stats("\n"); - dsn::utils::je_dump_detailed_stats(&stats); + if (args.size() == 1) { + dsn::je_dump_stats(type, stats); + return stats; + } + + uint64_t buf_sz; + if (!dsn::buf2uint64(args[1], buf_sz)) { + return std::string("ERR: invalid buffer size"); + } + + dsn::je_dump_stats(type, static_cast(buf_sz), stats); return stats; }); } @@ -2463,6 +2450,8 @@ void replica_stub::register_ctrl_command() auto release_bytes = gc_tcmalloc_memory(true); return "OK, release_bytes=" + std::to_string(release_bytes); }); +#elif defined(DSN_USE_JEMALLOC) + register_jemalloc_ctrl_command(); #endif _max_concurrent_bulk_load_downloading_count_command = dsn::command_manager::instance().register_command( @@ -2625,6 +2614,8 @@ void replica_stub::close() UNREGISTER_VALID_HANDLER(_get_tcmalloc_status_command); UNREGISTER_VALID_HANDLER(_max_reserved_memory_percentage_command); UNREGISTER_VALID_HANDLER(_release_all_reserved_memory_command); +#elif defined(DSN_USE_JEMALLOC) + UNREGISTER_VALID_HANDLER(_dump_jemalloc_stats_command); #endif UNREGISTER_VALID_HANDLER(_max_concurrent_bulk_load_downloading_count_command); @@ -2640,6 +2631,8 @@ void replica_stub::close() _get_tcmalloc_status_command = nullptr; _max_reserved_memory_percentage_command = nullptr; _release_all_reserved_memory_command = nullptr; +#elif defined(DSN_USE_JEMALLOC) + _dump_jemalloc_stats_command = nullptr; #endif _max_concurrent_bulk_load_downloading_count_command = nullptr; diff --git a/src/rdsn/src/replica/replica_stub.h b/src/rdsn/src/replica/replica_stub.h index 9b37ffda88..0db8bc10ad 100644 --- a/src/rdsn/src/replica/replica_stub.h +++ b/src/rdsn/src/replica/replica_stub.h @@ -384,6 +384,8 @@ class replica_stub : public serverlet, public ref_counter dsn_handle_t _get_tcmalloc_status_command; dsn_handle_t _max_reserved_memory_percentage_command; dsn_handle_t _release_all_reserved_memory_command; +#elif defined(DSN_USE_JEMALLOC) + dsn_handle_t _dump_jemalloc_stats_command; #endif dsn_handle_t _max_concurrent_bulk_load_downloading_count_command; diff --git a/src/rdsn/src/utils/je_ctl.cpp b/src/rdsn/src/utils/je_ctl.cpp index dc1edb172e..50d4314905 100644 --- a/src/rdsn/src/utils/je_ctl.cpp +++ b/src/rdsn/src/utils/je_ctl.cpp @@ -20,17 +20,19 @@ #include "je_ctl.h" #include +#include +#include #include #include #include -#define RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, array) \ - do { \ - const auto index = static_cast(type);\ - dcheck_lt(index, sizeof(array) / sizeof(array[0]));\ - return array[index];\ +#define RETURN_ARRAY_ELEM_BY_ENUM_TYPE(type, array) \ + do { \ + const auto index = static_cast(type); \ + dcheck_lt(index, sizeof(array) / sizeof(array[0])); \ + return array[index]; \ } while (0); namespace dsn { @@ -83,6 +85,15 @@ size_t je_stats_type_to_default_buf_sz(je_stats_type type) } // anonymous namespace +std::string get_all_je_stats_types_str() +{ + std::vector names; + for (size_t i = 0; i < static_cast(je_stats_type::COUNT); ++i) { + names.emplace_back(enum_to_string(static_cast(i))); + } + return boost::join(names, " | "); +} + void je_dump_stats(je_stats_type type, size_t buf_sz, std::string &stats) { je_dump_malloc_stats(je_stats_type_to_opts(type), buf_sz, stats); diff --git a/src/rdsn/src/utils/je_ctl.h b/src/rdsn/src/utils/je_ctl.h index 88a5781e92..8fdc8a9809 100644 --- a/src/rdsn/src/utils/je_ctl.h +++ b/src/rdsn/src/utils/je_ctl.h @@ -21,16 +21,30 @@ #include +#include + namespace dsn { enum class je_stats_type : size_t { - kSummaryStats, - kConfigs, - kBriefArenaStats, - kDetailedStats, + SUMMARY_STATS, + CONFIGS, + BRIEF_ARENA_STATS, + DETAILED_STATS, + COUNT, + INVALID, }; +ENUM_BEGIN(je_stats_type, je_stats_type::INVALID) +ENUM_REG(je_stats_type::SUMMARY_STATS) +ENUM_REG(je_stats_type::CONFIGS) +ENUM_REG(je_stats_type::BRIEF_ARENA_STATS) +ENUM_REG(je_stats_type::DETAILED_STATS) +ENUM_END(je_stats_type) + +std::string get_all_je_stats_types_str(); +const std::string kAllJeStatsTypesStr = get_all_je_stats_types_str(); + void je_dump_stats(je_stats_type type, size_t buf_sz, std::string &stats); void je_dump_stats(je_stats_type type, std::string &stats); From 5040e852f62805ad5144487c6c03bcd9dcb03914 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Sat, 20 Aug 2022 01:10:48 +0800 Subject: [PATCH 03/11] feat: support to dump jemalloc stats --- src/rdsn/src/replica/replica_stub.cpp | 2 +- src/rdsn/src/utils/je_ctl.cpp | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/rdsn/src/replica/replica_stub.cpp b/src/rdsn/src/replica/replica_stub.cpp index c87a81da1d..27c8e5eab6 100644 --- a/src/rdsn/src/replica/replica_stub.cpp +++ b/src/rdsn/src/replica/replica_stub.cpp @@ -2270,7 +2270,7 @@ void replica_stub::register_jemalloc_ctrl_command() { _dump_jemalloc_stats_command = ::dsn::command_manager::instance().register_command( {"replica.dump-jemalloc-stats"}, - fmt::format("replica.dump-jemalloc-stats <{}> [buf_sz]", kAllKthPercentileTypes), + fmt::format("replica.dump-jemalloc-stats <{}> [buf_sz]", kAllJeStatsTypesStr), "dump stats of jemalloc", [](const std::vector &args) { if (args.empty()) { diff --git a/src/rdsn/src/utils/je_ctl.cpp b/src/rdsn/src/utils/je_ctl.cpp index 50d4314905..92761d3c2f 100644 --- a/src/rdsn/src/utils/je_ctl.cpp +++ b/src/rdsn/src/utils/je_ctl.cpp @@ -39,8 +39,6 @@ namespace dsn { namespace { -template - void je_stats_cb(void *opaque, const char *str) { if (str == nullptr) { @@ -89,7 +87,7 @@ std::string get_all_je_stats_types_str() { std::vector names; for (size_t i = 0; i < static_cast(je_stats_type::COUNT); ++i) { - names.emplace_back(enum_to_string(static_cast(i))); + names.emplace_back(enum_to_string(static_cast(i))); } return boost::join(names, " | "); } From adfe58588bf5f1b0c3200f7b6bb45e4cc5e0adcb Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Sun, 21 Aug 2022 02:29:58 +0800 Subject: [PATCH 04/11] feat: support to dump jemalloc stats --- src/rdsn/include/dsn/utility/enum_helper.h | 1 + src/rdsn/src/utils/je_ctl.h | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/rdsn/include/dsn/utility/enum_helper.h b/src/rdsn/include/dsn/utility/enum_helper.h index 64d72638a2..e5d83e0d01 100644 --- a/src/rdsn/include/dsn/utility/enum_helper.h +++ b/src/rdsn/include/dsn/utility/enum_helper.h @@ -48,6 +48,7 @@ #define ENUM_BEGIN(type, invalid_value) ENUM_BEGIN2(type, type, invalid_value) +#define ENUM_REG2(type, name) helper->register_enum(#name, type::name); #define ENUM_REG(e) helper->register_enum(#e, e); #define ENUM_END2(type, name) \ diff --git a/src/rdsn/src/utils/je_ctl.h b/src/rdsn/src/utils/je_ctl.h index 8fdc8a9809..f69320cd45 100644 --- a/src/rdsn/src/utils/je_ctl.h +++ b/src/rdsn/src/utils/je_ctl.h @@ -36,10 +36,10 @@ enum class je_stats_type : size_t }; ENUM_BEGIN(je_stats_type, je_stats_type::INVALID) -ENUM_REG(je_stats_type::SUMMARY_STATS) -ENUM_REG(je_stats_type::CONFIGS) -ENUM_REG(je_stats_type::BRIEF_ARENA_STATS) -ENUM_REG(je_stats_type::DETAILED_STATS) +ENUM_REG2(je_stats_type, SUMMARY_STATS) +ENUM_REG2(je_stats_type, CONFIGS) +ENUM_REG2(je_stats_type, BRIEF_ARENA_STATS) +ENUM_REG2(je_stats_type, DETAILED_STATS) ENUM_END(je_stats_type) std::string get_all_je_stats_types_str(); From 07bb0510bcb75fb36050e16b675b0735eea7c13b Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 22 Aug 2022 00:36:03 +0800 Subject: [PATCH 05/11] feat: support to dump jemalloc stats --- src/rdsn/src/utils/test/je_ctl_test.cpp | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/rdsn/src/utils/test/je_ctl_test.cpp diff --git a/src/rdsn/src/utils/test/je_ctl_test.cpp b/src/rdsn/src/utils/test/je_ctl_test.cpp new file mode 100644 index 0000000000..deddfe3ac2 --- /dev/null +++ b/src/rdsn/src/utils/test/je_ctl_test.cpp @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifdef DSN_USE_JEMALLOC + +#include "utils/je_ctl.h" + +#include + +namespace dsn { + +namespace { + +void check_stats_marks(const std::string &stats) +{ + ASSERT_NE(stats.find("Begin jemalloc statistics"), std::string::npos); + ASSERT_NE(stats.find("End jemalloc statistics"), std::string::npos); +} + +} // anonymous namespace + +TEST(je_ctl_test, dump_summary_stats) +{ + std::string stats; + je_dump_stats(je_stats_type::SUMMARY_STATS, stats); + + check_stats_marks(stats); +} + +TEST(je_ctl_test, dump_configs) +{ + std::string stats; + je_dump_stats(je_stats_type::CONFIGS, stats); + + check_stats_marks(stats); + ASSERT_NE(stats.find("Build-time option settings"), std::string::npos); + ASSERT_NE(stats.find("Run-time option settings"), std::string::npos); + ASSERT_NE(stats.find("Profiling settings"), std::string::npos); +} + +TEST(je_ctl_test, dump_brief_arena_stats) +{ + std::string stats; + je_dump_stats(je_stats_type::BRIEF_ARENA_STATS, stats); + + check_stats_marks(stats); + ASSERT_NE(stats.find("arenas[0]:"), std::string::npos); + ASSERT_NE(stats.find("assigned threads:"), std::string::npos); + ASSERT_NE(stats.find("decaying:"), std::string::npos); +} + +} // namespace dsn + +#endif // DSN_USE_JEMALLOC From 1cea1d9bc1318af4b46f2ee5f525489dfb99de4e Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 22 Aug 2022 12:24:54 +0800 Subject: [PATCH 06/11] feat: support to dump jemalloc stats --- src/rdsn/src/utils/test/je_ctl_test.cpp | 60 ++++++++++++++++++++----- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/src/rdsn/src/utils/test/je_ctl_test.cpp b/src/rdsn/src/utils/test/je_ctl_test.cpp index deddfe3ac2..06e64be17c 100644 --- a/src/rdsn/src/utils/test/je_ctl_test.cpp +++ b/src/rdsn/src/utils/test/je_ctl_test.cpp @@ -25,12 +25,40 @@ namespace dsn { namespace { -void check_stats_marks(const std::string &stats) +// This function does not check "End" mark, since sometimes returned message is very long while +// the buffer may not have enough space: the message will be truncated. If it is known that the +// message can be held by the buffer completely, `check_base_stats_marks_with_end` should be used +// instead. +void check_base_stats_marks(const std::string &stats) { ASSERT_NE(stats.find("Begin jemalloc statistics"), std::string::npos); + ASSERT_NE(stats.find("Allocated:"), std::string::npos); + ASSERT_NE(stats.find("Background threads:"), std::string::npos); +} + +void check_base_stats_marks_with_end(const std::string &stats) +{ + check_base_stats_marks(stats); ASSERT_NE(stats.find("End jemalloc statistics"), std::string::npos); } +void check_configs_marks(const std::string &stats) +{ + ASSERT_NE(stats.find("Version:"), std::string::npos); + ASSERT_NE(stats.find("Build-time option settings"), std::string::npos); + ASSERT_NE(stats.find("Run-time option settings"), std::string::npos); + ASSERT_NE(stats.find("Profiling settings"), std::string::npos); +} + +void check_arena_marks(const std::string &stats) +{ + // Marks for merged arenas. + ASSERT_NE(stats.find("Merged arenas stats:"), std::string::npos); + + // Marks for each arena. + ASSERT_NE(stats.find("arenas[0]:"), std::string::npos); +} + } // anonymous namespace TEST(je_ctl_test, dump_summary_stats) @@ -38,7 +66,7 @@ TEST(je_ctl_test, dump_summary_stats) std::string stats; je_dump_stats(je_stats_type::SUMMARY_STATS, stats); - check_stats_marks(stats); + check_base_stats_marks_with_end(stats); } TEST(je_ctl_test, dump_configs) @@ -46,10 +74,8 @@ TEST(je_ctl_test, dump_configs) std::string stats; je_dump_stats(je_stats_type::CONFIGS, stats); - check_stats_marks(stats); - ASSERT_NE(stats.find("Build-time option settings"), std::string::npos); - ASSERT_NE(stats.find("Run-time option settings"), std::string::npos); - ASSERT_NE(stats.find("Profiling settings"), std::string::npos); + check_base_stats_marks_with_end(stats); + check_configs_marks(stats); } TEST(je_ctl_test, dump_brief_arena_stats) @@ -57,10 +83,24 @@ TEST(je_ctl_test, dump_brief_arena_stats) std::string stats; je_dump_stats(je_stats_type::BRIEF_ARENA_STATS, stats); - check_stats_marks(stats); - ASSERT_NE(stats.find("arenas[0]:"), std::string::npos); - ASSERT_NE(stats.find("assigned threads:"), std::string::npos); - ASSERT_NE(stats.find("decaying:"), std::string::npos); + // Since there may be many arenas, "End" mark is not required to be checked here. + check_base_stats_marks(stats); + check_arena_marks(stats); +} + +TEST(je_ctl_test, dump_detailed_stats) +{ + std::string stats; + je_dump_stats(je_stats_type::DETAILED_STATS, stats); + + // Since there may be many arenas, "End" mark is not required to be checked here. + check_base_stats_marks(stats); + + // Detailed stats will contain all information, therefore everything should be checked. + check_configs_marks(stats); + check_arena_marks(stats); + ASSERT_NE(stats.find("bins:"), std::string::npos); + ASSERT_NE(stats.find("extents:"), std::string::npos); } } // namespace dsn From 8da7cdc44fdf91eb75af0008711d13180ba6c8e0 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 22 Aug 2022 16:22:15 +0800 Subject: [PATCH 07/11] feat: support to dump jemalloc stats --- src/rdsn/src/utils/je_ctl.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/rdsn/src/utils/je_ctl.h b/src/rdsn/src/utils/je_ctl.h index f69320cd45..b26acfea9d 100644 --- a/src/rdsn/src/utils/je_ctl.h +++ b/src/rdsn/src/utils/je_ctl.h @@ -25,6 +25,13 @@ namespace dsn { +// `je_stats_type` defines the types of stats that are dumped for jemalloc allocator: +// * By SUMMARY_STATS, it will dump the briefest message of stats, which only referred the +// summary information; +// * By CONFIGS, it will dump the configurations of jemalloc; +// * By BRIEF_ARENA_STATS, it will dump necessary information about overall stats for all +// arenas and individual stats for each arena; +// * By DETAILED_STATS, it will dump the detailed stats for all arenas, bins, extents, etc. enum class je_stats_type : size_t { SUMMARY_STATS, @@ -45,8 +52,14 @@ ENUM_END(je_stats_type) std::string get_all_je_stats_types_str(); const std::string kAllJeStatsTypesStr = get_all_je_stats_types_str(); +// Dump the stats of specified type to a string, with specified buffer size. +// +// The buffer is used to read stats from jemalloc allocator. The reason why an extra buffer is +// involved is that the stats can only be accessed in the callback function for jemalloc where +// memory allocation should be avoided. void je_dump_stats(je_stats_type type, size_t buf_sz, std::string &stats); +// Dump the stats of specified type to a string, with default buffer size. void je_dump_stats(je_stats_type type, std::string &stats); } // namespace dsn From 1b7d458b1f90ed7c9452ca7cf6782b6b8bf62b7a Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Mon, 22 Aug 2022 16:37:18 +0800 Subject: [PATCH 08/11] feat: support to dump jemalloc stats --- src/rdsn/src/replica/replica_stub.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rdsn/src/replica/replica_stub.cpp b/src/rdsn/src/replica/replica_stub.cpp index 27c8e5eab6..b85722c56a 100644 --- a/src/rdsn/src/replica/replica_stub.cpp +++ b/src/rdsn/src/replica/replica_stub.cpp @@ -2270,16 +2270,16 @@ void replica_stub::register_jemalloc_ctrl_command() { _dump_jemalloc_stats_command = ::dsn::command_manager::instance().register_command( {"replica.dump-jemalloc-stats"}, - fmt::format("replica.dump-jemalloc-stats <{}> [buf_sz]", kAllJeStatsTypesStr), + fmt::format("replica.dump-jemalloc-stats <{}> [buffer size]", kAllJeStatsTypesStr), "dump stats of jemalloc", [](const std::vector &args) { if (args.empty()) { - return std::string("ERR: invalid arguments"); + return std::string("invalid arguments"); } auto type = enum_from_string(args[0].c_str(), je_stats_type::INVALID); if (type == je_stats_type::INVALID) { - return std::string("ERR: invalid stats type"); + return std::string("invalid stats type"); } std::string stats("\n"); @@ -2291,7 +2291,7 @@ void replica_stub::register_jemalloc_ctrl_command() uint64_t buf_sz; if (!dsn::buf2uint64(args[1], buf_sz)) { - return std::string("ERR: invalid buffer size"); + return std::string("invalid buffer size"); } dsn::je_dump_stats(type, static_cast(buf_sz), stats); From eb364f827d9ba221bbdbef7b54d4a7f6ebba1952 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 30 Aug 2022 21:00:58 +0800 Subject: [PATCH 09/11] feat: support to dump jemalloc stats --- .github/workflows/lint_and_test_cpp.yaml | 84 ++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index a19b6568fc..a5857b3976 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -373,6 +373,90 @@ jobs: ./scripts/config_hdfs.sh ./run.sh test --on_travis -m ${{ matrix.test_module }} + build_with_jemalloc: + name: Build with jemalloc + needs: cpp_clang_format_linter + runs-on: ubuntu-latest + container: + image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu1804-${{ github.base_ref }} + steps: + - uses: actions/checkout@v2 + - name: Setup cache + uses: actions/cache@v3 + with: + path: | + /github/home/.ccache + key: release_ccache + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + thirdparty: + - '.github/workflows/thirdparty-regular-push.yml' + - 'docker/thirdparties-src/**' + - 'docker/thirdparties-bin/**' + - 'thirdparty/**' + - name: Unpack prebuilt third-parties + if: steps.changes.outputs.thirdparty == 'false' + run: unzip /root/thirdparties-bin.zip -d ./thirdparty + - name: Rebuild third-parties + if: steps.changes.outputs.thirdparty == 'true' + working-directory: thirdparty + run: | + mkdir build + cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/ + cmake --build build/ -j $(nproc) + - name: Compilation + run: | + ccache -p + ccache -z + ./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc + ccache -s + - name: Pack Server + run: ./run.sh pack_server + - name: Pack Tools + run: ./run.sh pack_tools + - name: Tar files + run: | + rm -rf thirdparty + tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*' + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: release_artifact_${{ github.sha }} + path: release__builder.tar + + test_with_jemalloc: + name: Test with jemallc + strategy: + fail-fast: false + matrix: + test_module: + - dsn_utils_tests + needs: build_with_jemalloc + runs-on: ubuntu-latest + container: + image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu1804-${{ github.base_ref }} + options: --cap-add=SYS_PTRACE + steps: + - uses: actions/checkout@v2 + - name: Unpack prebuilt third-parties + run: unzip /root/thirdparties-bin.zip -d ./thirdparty + - name: Download Artifact + uses: actions/download-artifact@v3 + with: + name: release_artifact_${{ github.sha }} + path: . + - name: Tar files + run: | + tar -zxvf release__builder.tar + - name: Unit Testing + run: | + export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server + ulimit -s unlimited + ./scripts/config_hdfs.sh + ./run.sh test --on_travis -m ${{ matrix.test_module }} + build_pegasus_on_macos: name: macOS needs: cpp_clang_format_linter From ff10635c83d6ebedf2df2017c5c7ab2cd8f301bb Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Wed, 31 Aug 2022 11:28:22 +0800 Subject: [PATCH 10/11] feat: support to dump jemalloc stats --- .github/workflows/lint_and_test_cpp.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index a5857b3976..a8dc3c509e 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -413,18 +413,18 @@ jobs: ./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc ccache -s - name: Pack Server - run: ./run.sh pack_server + run: ./run.sh pack_server -j - name: Pack Tools - run: ./run.sh pack_tools + run: ./run.sh pack_tools -j - name: Tar files run: | rm -rf thirdparty - tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*' + tar -zcvhf release_jemalloc_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*' - name: Upload Artifact uses: actions/upload-artifact@v3 with: - name: release_artifact_${{ github.sha }} - path: release__builder.tar + name: release_jemalloc_artifact_${{ github.sha }} + path: release_jemalloc_builder.tar test_with_jemalloc: name: Test with jemallc @@ -445,11 +445,11 @@ jobs: - name: Download Artifact uses: actions/download-artifact@v3 with: - name: release_artifact_${{ github.sha }} + name: release_jemalloc_artifact_${{ github.sha }} path: . - name: Tar files run: | - tar -zxvf release__builder.tar + tar -zxvf release_jemalloc_builder.tar - name: Unit Testing run: | export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server From 3eeef904cbde7f97770a6aa3057f528134579384 Mon Sep 17 00:00:00 2001 From: Dan Wang Date: Tue, 6 Sep 2022 16:16:55 +0800 Subject: [PATCH 11/11] feat: support to dump jemalloc stats --- .github/workflows/lint_and_test_cpp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index a8dc3c509e..65d59e47e7 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -386,7 +386,7 @@ jobs: with: path: | /github/home/.ccache - key: release_ccache + key: jemalloc_ccache - uses: dorny/paths-filter@v2 id: changes with: