Change the way of configuring cache (vesoft-inc#758)

* use #cache entries to set cache * fix bug * update comments in config Co-authored-by: Sophie <[email protected]>
liwenhui-soul · Mar 31, 2022 · 3d2892b · 3d2892b
1 parent 8aef733
commit 3d2892b
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 37 deletions.
diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default
@@ -121,12 +121,9 @@
 --enable_storage_cache=false
 # Total capacity reserved for storage in memory cache in MB
 --storage_cache_capacity=0
-# Number of buckets in base 2 logarithm. E.g., in case of 20, the total number of buckets will be 2^20. 
-# A good estimate can be ceil(log2(#cacheEntries * 1.6)). The maximum allowed is 32.
---storage_cache_buckets_power=20
-# Number of locks in base 2 logarithm. E.g., in case of 10, the total number of locks will be 2^10. 
-# A good estimate can be max(1, bucketsPower - 10). The maximum allowed is 32.
---storage_cache_locks_power=10
+# Estimated number of cache entries on this storage node in base 2 logarithm. E.g., in case of 20, the estimated number of entries will be 2^20. 
+# A good estimate can be log2(#vertices on this storage node * #tags). The maximum allowed is 31.
+--storage_cache_entries_power=20
 
 # Whether to add vertex pool in cache. Only valid when storage cache is enabled.
 --enable_vertex_pool=false

diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production
@@ -119,12 +119,9 @@
 --enable_storage_cache=false
 # Total capacity reserved for storage in memory cache in MB
 --storage_cache_capacity=0
-# Number of buckets in base 2 logarithm. E.g., in case of 20, the total number of buckets will be 2^20. 
-# A good estimate can be ceil(log2(cache_entries * 1.6)). The maximum allowed is 32.
---storage_cache_buckets_power=20
-# Number of locks in base 2 logarithm. E.g., in case of 10, the total number of locks will be 2^10. 
-# A good estimate can be max(1, buckets_power - 10). The maximum allowed is 32.
---storage_cache_locks_power=10
+# Estimated number of cache entries on this storage node in base 2 logarithm. E.g., in case of 20, the estimated number of entries will be 2^20. 
+# A good estimate can be log2(#vertices on this storage node * #tags). The maximum allowed is 31.
+--storage_cache_entries_power=20
 
 # Whether to add vertex pool in cache. Only valid when storage cache is enabled.
 --enable_vertex_pool=false

diff --git a/src/common/base/CacheLibLRU.h b/src/common/base/CacheLibLRU.h
@@ -18,11 +18,8 @@ using Cache = facebook::cachelib::LruAllocator;
 
 class CacheLibLRU {
  public:
-  explicit CacheLibLRU(std::string name,
-                       uint32_t capacity,
-                       uint32_t bucketsPower,
-                       uint32_t locksPower)
-      : name_(name), capacity_(capacity), bucketsPower_(bucketsPower), locksPower_(locksPower) {}
+  explicit CacheLibLRU(std::string name, uint32_t capacity, uint32_t cacheEntriesPower)
+      : name_(name), capacity_(capacity), cacheEntriesPower_(cacheEntriesPower) {}
 
   /**
    * @brief Create cache instance. If there is any exception, we will allow the process continue.
@@ -32,12 +29,18 @@ class CacheLibLRU {
    */
   nebula::cpp2::ErrorCode initializeCache() {
     Cache::Config config;
+    if (cacheEntriesPower_ > kMaxCacheEntriesPower) {
+      LOG(WARNING) << "Estimated number of cache entries exceeds the cache limit. Nebula will trim "
+                      "the the maximum allowed: "
+                   << kMaxCacheEntriesPower;
+      cacheEntriesPower_ = kMaxCacheEntriesPower;
+    }
     try {
       config
           // size cannot exceed the maximum cache size (274'877'906'944 bytes)
           .setCacheSize(capacity_ * 1024 * 1024)
           .setCacheName(name_)
-          .setAccessConfig({bucketsPower_, locksPower_})
+          .setAccessConfig(std::pow(2, cacheEntriesPower_))
           .validate();  // will throw if bad config
     } catch (const std::exception& e) {
       // We do not stop the service. Users should refer to the log to determine whether to restart
@@ -179,12 +182,12 @@ class CacheLibLRU {
   }
 
  private:
+  static constexpr uint32_t kMaxCacheEntriesPower = 31;  // the cap by cachelib
   std::unique_ptr<Cache> nebulaCache_ = nullptr;
   std::unordered_map<std::string, facebook::cachelib::PoolId> poolIdMap_;
   std::string name_;
-  uint32_t capacity_ = 0;       // in MB
-  uint32_t bucketsPower_ = 20;  // bucketsPower number of buckets in base 2 logarithm
-  uint32_t locksPower_ = 10;    // locksPower number of locks in base 2 logarithm
+  uint32_t capacity_ = 0;            // in MB
+  uint32_t cacheEntriesPower_ = 20;  // estimated number of cache entries in base 2 logarithm
 
   // CacheLib does not protect data at item level. We need to synchronize the access.
   folly::SharedMutex lock_;

diff --git a/src/kvstore/cache/StorageCache.cpp b/src/kvstore/cache/StorageCache.cpp
@@ -11,15 +11,10 @@ DEFINE_uint32(storage_cache_capacity,
               100,
               "Total capacity reservered for storage in memory cache in MB");
 
-DEFINE_uint32(storage_cache_buckets_power,
-              10,
-              "Number of buckets in base 2 logarithm. "
-              "E.g., in case of 10, the total number of buckets will be 2^10.");
-
-DEFINE_uint32(storage_cache_locks_power,
-              5,
-              "Number of locks in base 2 logarithm. "
-              "E.g., in case of 5, the total number of locks will be 2^5.");
+DEFINE_uint32(storage_cache_entries_power,
+              20,
+              "Estimated number of cache entries in base 2 logarithm. "
+              "E.g., in case of 20, the estimated number of entries will be 2^20.");
 
 DEFINE_uint32(vertex_pool_capacity, 50, "Vertex pool size in MB");
 DEFINE_uint32(vertex_item_ttl, 300, "TTL for vertex item in the cache");
@@ -36,10 +31,8 @@ namespace kvstore {
 
 StorageCache::StorageCache() {
   capacity_ = FLAGS_storage_cache_capacity;
-  cacheInternal_ = std::make_unique<CacheLibLRU>(kStorageCacheName,
-                                                 capacity_,
-                                                 FLAGS_storage_cache_buckets_power,
-                                                 FLAGS_storage_cache_locks_power);
+  cacheInternal_ = std::make_unique<CacheLibLRU>(
+      kStorageCacheName, capacity_, FLAGS_storage_cache_entries_power);
 }
 
 bool StorageCache::init() {

diff --git a/src/kvstore/cache/test/StorageCacheTest.cpp b/src/kvstore/cache/test/StorageCacheTest.cpp
@@ -9,9 +9,7 @@
 
 DECLARE_uint32(storage_cache_capacity);
 
-DECLARE_uint32(storage_cache_buckets_power);
-
-DECLARE_uint32(storage_cache_locks_power);
+DECLARE_uint32(storage_cache_entries_power);
 
 DECLARE_uint32(vertex_pool_capacity);