diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default index 9c96a141fde..2105fa62bef 100644 --- a/conf/nebula-storaged.conf.default +++ b/conf/nebula-storaged.conf.default @@ -121,12 +121,9 @@ --enable_storage_cache=false # Total capacity reserved for storage in memory cache in MB --storage_cache_capacity=0 -# Number of buckets in base 2 logarithm. E.g., in case of 20, the total number of buckets will be 2^20. -# A good estimate can be ceil(log2(#cacheEntries * 1.6)). The maximum allowed is 32. ---storage_cache_buckets_power=20 -# Number of locks in base 2 logarithm. E.g., in case of 10, the total number of locks will be 2^10. -# A good estimate can be max(1, bucketsPower - 10). The maximum allowed is 32. ---storage_cache_locks_power=10 +# Estimated number of cache entries on this storage node in base 2 logarithm. E.g., in case of 20, the estimated number of entries will be 2^20. +# A good estimate can be log2(#vertices on this storage node * #tags). The maximum allowed is 31. +--storage_cache_entries_power=20 # Whether to add vertex pool in cache. Only valid when storage cache is enabled. --enable_vertex_pool=false diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production index 78bb4faf7d8..6e14da73a54 100644 --- a/conf/nebula-storaged.conf.production +++ b/conf/nebula-storaged.conf.production @@ -119,12 +119,9 @@ --enable_storage_cache=false # Total capacity reserved for storage in memory cache in MB --storage_cache_capacity=0 -# Number of buckets in base 2 logarithm. E.g., in case of 20, the total number of buckets will be 2^20. -# A good estimate can be ceil(log2(cache_entries * 1.6)). The maximum allowed is 32. ---storage_cache_buckets_power=20 -# Number of locks in base 2 logarithm. E.g., in case of 10, the total number of locks will be 2^10. -# A good estimate can be max(1, buckets_power - 10). The maximum allowed is 32. ---storage_cache_locks_power=10 +# Estimated number of cache entries on this storage node in base 2 logarithm. E.g., in case of 20, the estimated number of entries will be 2^20. +# A good estimate can be log2(#vertices on this storage node * #tags). The maximum allowed is 31. +--storage_cache_entries_power=20 # Whether to add vertex pool in cache. Only valid when storage cache is enabled. --enable_vertex_pool=false diff --git a/src/common/base/CacheLibLRU.h b/src/common/base/CacheLibLRU.h index e029172056f..76232bdad1d 100644 --- a/src/common/base/CacheLibLRU.h +++ b/src/common/base/CacheLibLRU.h @@ -18,11 +18,8 @@ using Cache = facebook::cachelib::LruAllocator; class CacheLibLRU { public: - explicit CacheLibLRU(std::string name, - uint32_t capacity, - uint32_t bucketsPower, - uint32_t locksPower) - : name_(name), capacity_(capacity), bucketsPower_(bucketsPower), locksPower_(locksPower) {} + explicit CacheLibLRU(std::string name, uint32_t capacity, uint32_t cacheEntriesPower) + : name_(name), capacity_(capacity), cacheEntriesPower_(cacheEntriesPower) {} /** * @brief Create cache instance. If there is any exception, we will allow the process continue. @@ -32,12 +29,18 @@ class CacheLibLRU { */ nebula::cpp2::ErrorCode initializeCache() { Cache::Config config; + if (cacheEntriesPower_ > kMaxCacheEntriesPower) { + LOG(WARNING) << "Estimated number of cache entries exceeds the cache limit. Nebula will trim " + "the the maximum allowed: " + << kMaxCacheEntriesPower; + cacheEntriesPower_ = kMaxCacheEntriesPower; + } try { config // size cannot exceed the maximum cache size (274'877'906'944 bytes) .setCacheSize(capacity_ * 1024 * 1024) .setCacheName(name_) - .setAccessConfig({bucketsPower_, locksPower_}) + .setAccessConfig(std::pow(2, cacheEntriesPower_)) .validate(); // will throw if bad config } catch (const std::exception& e) { // We do not stop the service. Users should refer to the log to determine whether to restart @@ -179,12 +182,12 @@ class CacheLibLRU { } private: + static constexpr uint32_t kMaxCacheEntriesPower = 31; // the cap by cachelib std::unique_ptr nebulaCache_ = nullptr; std::unordered_map poolIdMap_; std::string name_; - uint32_t capacity_ = 0; // in MB - uint32_t bucketsPower_ = 20; // bucketsPower number of buckets in base 2 logarithm - uint32_t locksPower_ = 10; // locksPower number of locks in base 2 logarithm + uint32_t capacity_ = 0; // in MB + uint32_t cacheEntriesPower_ = 20; // estimated number of cache entries in base 2 logarithm // CacheLib does not protect data at item level. We need to synchronize the access. folly::SharedMutex lock_; diff --git a/src/kvstore/cache/StorageCache.cpp b/src/kvstore/cache/StorageCache.cpp index 8835175ca24..d8f1875dc6f 100644 --- a/src/kvstore/cache/StorageCache.cpp +++ b/src/kvstore/cache/StorageCache.cpp @@ -11,15 +11,10 @@ DEFINE_uint32(storage_cache_capacity, 100, "Total capacity reservered for storage in memory cache in MB"); -DEFINE_uint32(storage_cache_buckets_power, - 10, - "Number of buckets in base 2 logarithm. " - "E.g., in case of 10, the total number of buckets will be 2^10."); - -DEFINE_uint32(storage_cache_locks_power, - 5, - "Number of locks in base 2 logarithm. " - "E.g., in case of 5, the total number of locks will be 2^5."); +DEFINE_uint32(storage_cache_entries_power, + 20, + "Estimated number of cache entries in base 2 logarithm. " + "E.g., in case of 20, the estimated number of entries will be 2^20."); DEFINE_uint32(vertex_pool_capacity, 50, "Vertex pool size in MB"); DEFINE_uint32(vertex_item_ttl, 300, "TTL for vertex item in the cache"); @@ -36,10 +31,8 @@ namespace kvstore { StorageCache::StorageCache() { capacity_ = FLAGS_storage_cache_capacity; - cacheInternal_ = std::make_unique(kStorageCacheName, - capacity_, - FLAGS_storage_cache_buckets_power, - FLAGS_storage_cache_locks_power); + cacheInternal_ = std::make_unique( + kStorageCacheName, capacity_, FLAGS_storage_cache_entries_power); } bool StorageCache::init() { diff --git a/src/kvstore/cache/test/StorageCacheTest.cpp b/src/kvstore/cache/test/StorageCacheTest.cpp index 1316475f1ec..88d47eeaf83 100644 --- a/src/kvstore/cache/test/StorageCacheTest.cpp +++ b/src/kvstore/cache/test/StorageCacheTest.cpp @@ -9,9 +9,7 @@ DECLARE_uint32(storage_cache_capacity); -DECLARE_uint32(storage_cache_buckets_power); - -DECLARE_uint32(storage_cache_locks_power); +DECLARE_uint32(storage_cache_entries_power); DECLARE_uint32(vertex_pool_capacity);