2

apache · Jul 10, 2024 · aa7d8fa · aa7d8fa
1 parent 77e462f
commit aa7d8fa
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 7 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1072,7 +1072,6 @@ DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
 DEFINE_mInt32(segment_cache_capacity, "-1");
 DEFINE_mInt32(estimated_num_columns_per_segment, "200");
 DEFINE_mInt32(estimated_mem_per_column_reader, "1024");
-// The value is calculate by storage_page_cache_limit * index_page_cache_percentage
 DEFINE_mInt32(segment_cache_memory_percentage, "2");
 
 // enable feature binlog, default false

diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
@@ -76,21 +76,33 @@ std::atomic<bool> MemInfo::je_purge_dirty_pages_notify {false};
 void MemInfo::refresh_allocator_mem() {
 #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
 #elif defined(USE_JEMALLOC)
+    // jemalloc mallctl refer to : https://jemalloc.net/jemalloc.3.html
+    // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md
+    //  Check the Doris BE web page `http://ip:webserver_port/memz` to get the Jemalloc Profile.
+
     // 'epoch' is a special mallctl -- it updates the statistics. Without it, all
     // the following calls will return stale values. It increments and returns
     // the current epoch number, which might be useful to log as a sanity check.
     uint64_t epoch = 0;
     size_t sz = sizeof(epoch);
     jemallctl("epoch", &epoch, &sz, &epoch, sz);
 
-    // https://jemalloc.net/jemalloc.3.html
-    // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md
-    _s_allocator_cache_mem.store(get_je_all_arena_metrics("tcache_bytes") +
-                                         get_je_all_arena_metrics("pdirty") * get_page_size(),
+    // Number of extents of the given type in this arena in the bucket corresponding to page size index.
+    // Large size class starts at 16384, the extents have three sizes before 16384: 4096, 8192, and 12288, so + 3
+    int64_t dirty_pages_bytes = 0;
+    for (unsigned i = 0; i < get_je_unsigned_metrics("arenas.nlextents") + 3; i++) {
+        dirty_pages_bytes += get_je_all_arena_extents_metrics(i, "dirty_bytes");
+    }
+    _s_je_dirty_pages_mem.store(dirty_pages_bytes, std::memory_order_relaxed);
+
+    // Doris uses Jemalloc as default Allocator, Jemalloc Cache consists of two parts:
+    // - Thread Cache, cache a specified number of Pages in Thread Cache.
+    // - Dirty Page, memory Page that can be reused in all Arenas.
+    _s_allocator_cache_mem.store(get_je_all_arena_metrics("tcache_bytes") + dirty_pages_bytes,
                                  std::memory_order_relaxed);
+    // Total number of bytes dedicated to metadata, which comprise base allocations used
+    // for bootstrap-sensitive allocator metadata structures.
     _s_allocator_metadata_mem.store(get_je_metrics("stats.metadata"), std::memory_order_relaxed);
-    _s_je_dirty_pages_mem.store(get_je_all_arena_metrics("pdirty") * get_page_size(),
-                                std::memory_order_relaxed);
     _s_virtual_memory_used.store(get_je_metrics("stats.mapped"), std::memory_order_relaxed);
 #else
     _s_allocator_cache_mem.store(get_tc_metrics("tcmalloc.pageheap_free_bytes") +

diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
@@ -108,13 +108,33 @@ class MemInfo {
         return 0;
     }
 
+    static inline unsigned get_je_unsigned_metrics(const std::string& name) {
+#ifdef USE_JEMALLOC
+        unsigned value = 0;
+        size_t sz = sizeof(value);
+        if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
+            return value;
+        }
+#endif
+        return 0;
+    }
+
     static inline int64_t get_je_all_arena_metrics(const std::string& name) {
 #ifdef USE_JEMALLOC
         return get_je_metrics(fmt::format("stats.arenas.{}.{}", MALLCTL_ARENAS_ALL, name));
 #endif
         return 0;
     }
 
+    static inline int64_t get_je_all_arena_extents_metrics(int64_t page_size_index,
+                                                           const std::string& extent_type) {
+#ifdef USE_JEMALLOC
+        return get_je_metrics(fmt::format("stats.arenas.{}.extents.{}.{}", MALLCTL_ARENAS_ALL,
+                                          page_size_index, extent_type));
+#endif
+        return 0;
+    }
+
     static inline void je_purge_all_arena_dirty_pages() {
 #ifdef USE_JEMALLOC
         // https://github.com/jemalloc/jemalloc/issues/2470