diff --git a/be/src/olap/tablet_column_cache.cpp b/be/src/olap/tablet_column_object_pool.cpp similarity index 86% rename from be/src/olap/tablet_column_cache.cpp rename to be/src/olap/tablet_column_object_pool.cpp index db40c727805cb78..429b48603b032a4 100644 --- a/be/src/olap/tablet_column_cache.cpp +++ b/be/src/olap/tablet_column_object_pool.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "olap/tablet_column_cache.h" +#include "olap/tablet_column_object_pool.h" #include #include @@ -24,7 +24,7 @@ namespace doris { -TabletColumnPtr TabletColumnCache::insert(const std::string& key) { +TabletColumnPtr TabletColumnObjectPool::insert(const std::string& key) { auto* lru_handle = lookup(key); TabletColumnPtr tablet_column_ptr; if (lru_handle) { @@ -46,10 +46,4 @@ TabletColumnPtr TabletColumnCache::insert(const std::string& key) { return tablet_column_ptr; } -void TabletColumnCache::release(Cache::Handle* lru_handle) { - LRUCachePolicy::release(lru_handle); -} - -TabletColumnCache::CacheValue::~CacheValue() = default; - } // namespace doris diff --git a/be/src/olap/tablet_column_cache.h b/be/src/olap/tablet_column_object_pool.h similarity index 76% rename from be/src/olap/tablet_column_cache.h rename to be/src/olap/tablet_column_object_pool.h index 658d2ddb348bfaa..1219b7921e53a02 100644 --- a/be/src/olap/tablet_column_cache.h +++ b/be/src/olap/tablet_column_object_pool.h @@ -23,36 +23,32 @@ namespace doris { -// TabletColumnCache is a cache for TabletColumn objects. It is used to reduce memory consumption +// TabletColumnObjectPool is a cache for TabletColumn objects. It is used to reduce memory consumption // when there are a large number of identical TabletColumns in the cluster, which usually occurs // when VARIANT type columns are modified and added, each Rowset has an individual TabletSchema. // Excessive TabletSchemas can lead to significant memory overhead. Reusing memory for identical // TabletColumns would greatly reduce this memory consumption. -class TabletColumnCache : public LRUCachePolicy { +class TabletColumnObjectPool : public LRUCachePolicy { public: - TabletColumnCache(size_t capacity) - : LRUCachePolicy(CachePolicy::CacheType::TABLET_COLUMN_CACHE, capacity, + TabletColumnObjectPool(size_t capacity) + : LRUCachePolicy(CachePolicy::CacheType::TABLET_COLUMN_OBJECT_POOL, capacity, LRUCacheType::NUMBER, config::tablet_schema_cache_recycle_interval) {} - static TabletColumnCache* create_global_column_cache(size_t capacity) { - auto* res = new TabletColumnCache(capacity); + static TabletColumnObjectPool* create_global_column_cache(size_t capacity) { + auto* res = new TabletColumnObjectPool(capacity); return res; } - static TabletColumnCache* instance() { - return ExecEnv::GetInstance()->get_tablet_column_cache(); + static TabletColumnObjectPool* instance() { + return ExecEnv::GetInstance()->get_tablet_column_object_pool(); } TabletColumnPtr insert(const std::string& key); - void release(Cache::Handle*); - private: class CacheValue : public LRUCacheValueBase { public: - ~CacheValue() override; - TabletColumnPtr tablet_column; }; }; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 78325d3b34d3bf6..1d7c71dd28d4788 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -38,7 +38,7 @@ #include "exec/tablet_info.h" #include "olap/inverted_index_parser.h" #include "olap/olap_define.h" -#include "olap/tablet_column_cache.h" +#include "olap/tablet_column_object_pool.h" #include "olap/types.h" #include "olap/utils.h" #include "runtime/thread_context.h" @@ -961,7 +961,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac for (auto& column_pb : schema.column()) { TabletColumnPtr column; if (reuse_cache_column) { - column = TabletColumnCache::instance()->insert( + column = TabletColumnObjectPool::instance()->insert( deterministic_string_serialize(column_pb)); } else { column = std::make_shared(); diff --git a/be/src/olap/tablet_schema_cache.cpp b/be/src/olap/tablet_schema_cache.cpp index 14e557632af138b..4b79049b0507bf3 100644 --- a/be/src/olap/tablet_schema_cache.cpp +++ b/be/src/olap/tablet_schema_cache.cpp @@ -18,6 +18,12 @@ #include "olap/tablet_schema_cache.h" #include +#include + +#include +#include +#include +#include #include "bvar/bvar.h" #include "olap/tablet_schema.h" @@ -27,12 +33,29 @@ bvar::Adder g_tablet_schema_cache_columns_count("tablet_schema_cache_co namespace doris { +// Get the key signature of the TabletSchemaPB, which is used as the key of LRUCache, +// to reduce the memory consumption of the serialized TabletSchema as key. +static std::string get_key_signature(const std::string& origin) { + // Use UUID namespace DNS to manually create + const static boost::uuids::uuid namespace_dns = + boost::uuids::string_generator()("6ba7b810-9dad-11d1-80b4-00c04fd430c8"); + + // base on MD5 hash algorithm to generate UUID + boost::uuids::name_generator_md5 gen(namespace_dns); + boost::uuids::uuid uuid = gen(origin); + + // return UUID as signature. + return boost::uuids::to_string(uuid); +} + std::pair TabletSchemaCache::insert(const std::string& key) { - auto* lru_handle = lookup(key); + std::string key_signature = get_key_signature(key); + auto* lru_handle = lookup(key_signature); TabletSchemaSPtr tablet_schema_ptr; if (lru_handle) { auto* value = (CacheValue*)LRUCachePolicy::value(lru_handle); tablet_schema_ptr = value->tablet_schema; + LOG_IF(FATAL, tablet_schema_ptr->to_key() != key) << "key signature not match"; } else { auto* value = new CacheValue; tablet_schema_ptr = std::make_shared(); @@ -41,8 +64,8 @@ std::pair TabletSchemaCache::insert(const std: // We should reuse the memory of the same TabletColumn object, set reuse_cached_column to true tablet_schema_ptr->init_from_pb(pb, false, true); value->tablet_schema = tablet_schema_ptr; - lru_handle = LRUCachePolicy::insert(key, value, tablet_schema_ptr->num_columns(), 0, - CachePriority::NORMAL); + lru_handle = LRUCachePolicy::insert(key_signature, value, tablet_schema_ptr->num_columns(), + 0, CachePriority::NORMAL); g_tablet_schema_cache_count << 1; g_tablet_schema_cache_columns_count << tablet_schema_ptr->num_columns(); } diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index fd285a89b9d6b40..b1617744eac6ba0 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -101,7 +101,7 @@ class FrontendServiceClient; class FileMetaCache; class GroupCommitMgr; class TabletSchemaCache; -class TabletColumnCache; +class TabletColumnObjectPool; class UserFunctionCache; class SchemaCache; class StoragePageCache; @@ -276,7 +276,9 @@ class ExecEnv { void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } - void set_tablet_column_cache(TabletColumnCache* c) { this->_tablet_column_cache = c; } + void set_tablet_column_object_pool(TabletColumnObjectPool* c) { + this->_tablet_column_object_pool = c; + } void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; } void set_segment_loader(SegmentLoader* sl) { this->_segment_loader = sl; } void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) { @@ -302,7 +304,7 @@ class ExecEnv { std::map get_running_frontends(); TabletSchemaCache* get_tablet_schema_cache() { return _tablet_schema_cache; } - TabletColumnCache* get_tablet_column_cache() { return _tablet_column_cache; } + TabletColumnObjectPool* get_tablet_column_object_pool() { return _tablet_column_object_pool; } SchemaCache* schema_cache() { return _schema_cache; } StoragePageCache* get_storage_page_cache() { return _storage_page_cache; } SegmentLoader* segment_loader() { return _segment_loader; } @@ -442,7 +444,7 @@ class ExecEnv { // these redundancy header could introduce potential bug, at least, more header means slow compile. // So we choose to use raw pointer, please remember to delete these pointer in deconstructor. TabletSchemaCache* _tablet_schema_cache = nullptr; - TabletColumnCache* _tablet_column_cache = nullptr; + TabletColumnObjectPool* _tablet_column_object_pool = nullptr; std::unique_ptr _storage_engine; SchemaCache* _schema_cache = nullptr; StoragePageCache* _storage_page_cache = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index a4e779a4510c021..ce8f8909e79b401 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -53,7 +53,7 @@ #include "olap/schema_cache.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" -#include "olap/tablet_column_cache.h" +#include "olap/tablet_column_object_pool.h" #include "olap/tablet_schema_cache.h" #include "olap/wal/wal_manager.h" #include "pipeline/pipeline_tracing.h" @@ -340,8 +340,8 @@ Status ExecEnv::_init(const std::vector& store_paths, _tablet_schema_cache = TabletSchemaCache::create_global_schema_cache(config::tablet_schema_cache_capacity); - _tablet_column_cache = - TabletColumnCache::create_global_column_cache(config::tablet_schema_cache_capacity); + _tablet_column_object_pool = TabletColumnObjectPool::create_global_column_cache( + config::tablet_schema_cache_capacity); // Storage engine doris::EngineOptions options; diff --git a/be/src/runtime/memory/cache_policy.h b/be/src/runtime/memory/cache_policy.h index a88d22e5c5e6636..e7e1c73e7cbb41d 100644 --- a/be/src/runtime/memory/cache_policy.h +++ b/be/src/runtime/memory/cache_policy.h @@ -49,7 +49,7 @@ class CachePolicy { NONE = 18, // not be used FOR_UT_CACHE_NUMBER = 19, QUERY_CACHE = 20, - TABLET_COLUMN_CACHE = 21, + TABLET_COLUMN_OBJECT_POOL = 21, }; static std::string type_string(CacheType type) { @@ -94,8 +94,8 @@ class CachePolicy { return "ForUTCacheNumber"; case CacheType::QUERY_CACHE: return "QueryCache"; - case CacheType::TABLET_COLUMN_CACHE: - return "TabletColumnCache"; + case CacheType::TABLET_COLUMN_OBJECT_POOL: + return "TabletColumnObjectPool"; default: LOG(FATAL) << "not match type of cache policy :" << static_cast(type); } @@ -123,7 +123,7 @@ class CachePolicy { {"CloudTabletCache", CacheType::CLOUD_TABLET_CACHE}, {"CloudTxnDeleteBitmapCache", CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE}, {"ForUTCacheNumber", CacheType::FOR_UT_CACHE_NUMBER}, - {"TabletColumnCache", CacheType::TABLET_COLUMN_CACHE}}; + {"TabletColumnObjectPool", CacheType::TABLET_COLUMN_OBJECT_POOL}}; static CacheType string_to_type(std::string type) { if (StringToType.contains(type)) { diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp index f1bef27d1a9eb60..59933db80e5bb93 100644 --- a/be/test/testutil/run_all_tests.cpp +++ b/be/test/testutil/run_all_tests.cpp @@ -31,7 +31,7 @@ #include "olap/page_cache.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" -#include "olap/tablet_column_cache.h" +#include "olap/tablet_column_object_pool.h" #include "olap/tablet_schema_cache.h" #include "runtime/exec_env.h" #include "runtime/memory/cache_manager.h" @@ -67,8 +67,8 @@ int main(int argc, char** argv) { doris::ExecEnv::GetInstance()->set_tablet_schema_cache( doris::TabletSchemaCache::create_global_schema_cache( doris::config::tablet_schema_cache_capacity)); - doris::ExecEnv::GetInstance()->set_tablet_column_cache( - doris::TabletColumnCache::create_global_column_cache( + doris::ExecEnv::GetInstance()->set_tablet_column_object_pool( + doris::TabletColumnObjectPool::create_global_column_cache( doris::config::tablet_schema_cache_capacity)); LOG(INFO) << "init config " << st; doris::Status s = doris::config::set_config("enable_stacktrace", "false");