diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index acf75dbc04ad58..da7c45d1a19d5f 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1062,7 +1062,6 @@ DEFINE_mInt32(schema_cache_sweep_time_sec, "100"); DEFINE_mInt32(segment_cache_capacity, "-1"); DEFINE_mInt32(estimated_num_columns_per_segment, "200"); DEFINE_mInt32(estimated_mem_per_column_reader, "1024"); -// The value is calculate by storage_page_cache_limit * index_page_cache_percentage DEFINE_mInt32(segment_cache_memory_percentage, "2"); // enable feature binlog, default false diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index e79ca1bfb3aacb..1c0ab863e6caed 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -191,13 +191,21 @@ void MemTrackerLimiter::make_process_snapshots(std::vector } snapshot.type = "overview"; - snapshot.label = "tc/jemalloc cache"; + snapshot.label = "tc/jemalloc_cache"; snapshot.limit = -1; snapshot.cur_consumption = MemInfo::allocator_cache_mem(); snapshot.peak_consumption = -1; (*snapshots).emplace_back(snapshot); all_tracker_mem_sum += MemInfo::allocator_cache_mem(); + snapshot.type = "overview"; + snapshot.label = "tc/jemalloc_metadata"; + snapshot.limit = -1; + snapshot.cur_consumption = MemInfo::allocator_metadata_mem(); + snapshot.peak_consumption = -1; + (*snapshots).emplace_back(snapshot); + all_tracker_mem_sum += MemInfo::allocator_metadata_mem(); + snapshot.type = "overview"; snapshot.label = "sum of all trackers"; // is virtual memory snapshot.limit = -1; @@ -217,7 +225,7 @@ void MemTrackerLimiter::make_process_snapshots(std::vector (*snapshots).emplace_back(snapshot); snapshot.type = "overview"; - snapshot.label = "reserved memory"; + snapshot.label = "reserve_memory"; snapshot.limit = -1; snapshot.cur_consumption = GlobalMemoryArbitrator::process_reserved_memory(); snapshot.peak_consumption = -1; diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index b6f35df5d9c2f4..baddefcc27f828 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -52,6 +52,7 @@ std::atomic MemInfo::_s_mem_limit = std::numeric_limits::max() std::atomic MemInfo::_s_soft_mem_limit = std::numeric_limits::max(); std::atomic MemInfo::_s_allocator_cache_mem = 0; +std::atomic MemInfo::_s_allocator_metadata_mem = 0; std::atomic MemInfo::_s_je_dirty_pages_mem = std::numeric_limits::min(); std::atomic MemInfo::_s_je_dirty_pages_mem_limit = std::numeric_limits::max(); std::atomic MemInfo::_s_virtual_memory_used = 0; @@ -75,6 +76,10 @@ std::atomic MemInfo::je_purge_dirty_pages_notify {false}; void MemInfo::refresh_allocator_mem() { #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER) #elif defined(USE_JEMALLOC) + // jemalloc mallctl refer to : https://jemalloc.net/jemalloc.3.html + // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md + // Check the Doris BE web page `http://ip:webserver_port/memz` to get the Jemalloc Profile. + // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. @@ -82,14 +87,22 @@ void MemInfo::refresh_allocator_mem() { size_t sz = sizeof(epoch); jemallctl("epoch", &epoch, &sz, &epoch, sz); - // https://jemalloc.net/jemalloc.3.html - // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md - _s_allocator_cache_mem.store(get_je_all_arena_metrics("tcache_bytes") + - get_je_metrics("stats.metadata") + - get_je_all_arena_metrics("pdirty") * get_page_size(), + // Number of extents of the given type in this arena in the bucket corresponding to page size index. + // Large size class starts at 16384, the extents have three sizes before 16384: 4096, 8192, and 12288, so + 3 + int64_t dirty_pages_bytes = 0; + for (unsigned i = 0; i < get_je_unsigned_metrics("arenas.nlextents") + 3; i++) { + dirty_pages_bytes += get_je_all_arena_extents_metrics(i, "dirty_bytes"); + } + _s_je_dirty_pages_mem.store(dirty_pages_bytes, std::memory_order_relaxed); + + // Doris uses Jemalloc as default Allocator, Jemalloc Cache consists of two parts: + // - Thread Cache, cache a specified number of Pages in Thread Cache. + // - Dirty Page, memory Page that can be reused in all Arenas. + _s_allocator_cache_mem.store(get_je_all_arena_metrics("tcache_bytes") + dirty_pages_bytes, std::memory_order_relaxed); - _s_je_dirty_pages_mem.store(get_je_all_arena_metrics("pdirty") * get_page_size(), - std::memory_order_relaxed); + // Total number of bytes dedicated to metadata, which comprise base allocations used + // for bootstrap-sensitive allocator metadata structures. + _s_allocator_metadata_mem.store(get_je_metrics("stats.metadata"), std::memory_order_relaxed); _s_virtual_memory_used.store(get_je_metrics("stats.mapped"), std::memory_order_relaxed); #else _s_allocator_cache_mem.store(get_tc_metrics("tcmalloc.pageheap_free_bytes") + diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index 6b66d05033f32d..ee83239c1287cf 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -105,6 +105,17 @@ class MemInfo { return 0; } + static inline unsigned get_je_unsigned_metrics(const std::string& name) { +#ifdef USE_JEMALLOC + unsigned value = 0; + size_t sz = sizeof(value); + if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) { + return value; + } +#endif + return 0; + } + static inline int64_t get_je_all_arena_metrics(const std::string& name) { #ifdef USE_JEMALLOC return get_je_metrics(fmt::format("stats.arenas.{}.{}", MALLCTL_ARENAS_ALL, name)); @@ -112,6 +123,15 @@ class MemInfo { return 0; } + static inline int64_t get_je_all_arena_extents_metrics(int64_t page_size_index, + const std::string& extent_type) { +#ifdef USE_JEMALLOC + return get_je_metrics(fmt::format("stats.arenas.{}.extents.{}.{}", MALLCTL_ARENAS_ALL, + page_size_index, extent_type)); +#endif + return 0; + } + static inline void je_purge_all_arena_dirty_pages() { #ifdef USE_JEMALLOC // https://github.com/jemalloc/jemalloc/issues/2470 @@ -144,6 +164,9 @@ class MemInfo { static inline size_t allocator_cache_mem() { return _s_allocator_cache_mem.load(std::memory_order_relaxed); } + static inline size_t allocator_metadata_mem() { + return _s_allocator_metadata_mem.load(std::memory_order_relaxed); + } static inline int64_t je_dirty_pages_mem() { return _s_je_dirty_pages_mem.load(std::memory_order_relaxed); } @@ -184,6 +207,7 @@ class MemInfo { static std::atomic _s_soft_mem_limit; static std::atomic _s_allocator_cache_mem; + static std::atomic _s_allocator_metadata_mem; static std::atomic _s_je_dirty_pages_mem; static std::atomic _s_je_dirty_pages_mem_limit; static std::atomic _s_virtual_memory_used;