Add a 5 byte compressed pointer for t2

Summary:
This diff adds 5-byte compressed pointer for T2 support.

It also adds a benchmark to explore further optimizations to 5-byte pointer to improve performance or to encode additional information about an item in the future.

However, note that CacheAllocator using 5-byte compressed pointer can only map 256GB of data. This is because the free allocation list in allocation classes use 4-byte compressed pointer. Changing this to a template type leads to a large number of complex changes in the codebase. All changes up to here should be a no-op change for current customers.

The next diff will set the 5-byte compressed pointer to be the default for the free list in allocation classes. It does not bloat the item size for caches <256GB as the item still uses 4-byte compressed pointer by default. This allows us to avoid complicated changes to the codebase with a minimal change to current users. CacheLib will support the new CacheAllocators with 5-byte compressed pointer which supports caches up to 32TB.

This diff also saves 1 bit for identifying where the compressed pointer belongs to an item in DRAM or NVM. The implementation is due.

This diff splits declaration of allocators in CacheAllocator.cpp into separate files which helps with build time.

Reviewed By: alikhtarov

Differential Revision: D62625822

fbshipit-source-id: 337b7cbcbeb7a8b68d124277ecbe97d1d850afca

cachelib/allocator/CacheAllocator.h

@@ -183,6 +183,7 @@ class CacheAllocator : public CacheBase {
   using CompressedPtrType = typename CacheTrait::CompressedPtrType;
   using MMType = typename CacheTrait::MMType;
   using AccessType = typename CacheTrait::AccessType;
+
   using Config = CacheAllocatorConfig<CacheT>;
 
   // configs for the MMtype and AccessType.
@@ -1336,7 +1337,9 @@ class CacheAllocator : public CacheBase {
                  sizeof(typename RefcountWithFlags::Value) + sizeof(uint32_t) +
                  sizeof(uint32_t) + sizeof(KAllocation)) == sizeof(Item),
                 "vtable overhead");
-  static_assert(32 == sizeof(Item), "item overhead is 32 bytes");
+  static_assert((20 + (3 * sizeof(CompressedPtrType))) == sizeof(Item),
+                "item overhead is 32 bytes for 4 byte compressed pointer and "
+                "35 bytes for 5 bytes compressed pointer.");
 
   // make sure there is no overhead in ChainedItem on top of a regular Item
   static_assert(sizeof(Item) == sizeof(ChainedItem),

cachelib/allocator/CacheAllocatorConfig.h

@@ -1095,7 +1095,7 @@ const CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::validate() const {
         "Tail hits tracking cannot be enabled on MMTypes except MM2Q.");
   }
 
-  size_t maxCacheSize = CompressedPtr4B::getMaxAddressableSize();
+  size_t maxCacheSize = T::CompressedPtrType::getMaxAddressableSize();
   // Configured cache size should not exceed the maximal addressable space for
   // cache.
   if (size > maxCacheSize) {

cachelib/allocator/CacheAllocatorLru2QCache.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<Lru2QCacheTrait>;
+}

cachelib/allocator/CacheAllocatorLru5B2QCache.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<Lru5B2QCacheTrait>;
+}

cachelib/allocator/CacheAllocatorLru5BCache.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<Lru5BCacheTrait>;
+}

cachelib/allocator/CacheAllocatorLru5BCacheWithSpinBuckets.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<Lru5BCacheWithSpinBucketsTrait>;
+}

cachelib/allocator/CacheAllocatorLruCache.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<LruCacheTrait>;
+}

cachelib/allocator/CacheAllocator.cpp → cachelib/allocator/CacheAllocatorLruCacheWithSpinBuckets.cpp

@@ -17,8 +17,5 @@
 #include "cachelib/allocator/CacheAllocator.h"
 
 namespace facebook::cachelib {
-template class CacheAllocator<LruCacheTrait>;
 template class CacheAllocator<LruCacheWithSpinBucketsTrait>;
-template class CacheAllocator<Lru2QCacheTrait>;
-template class CacheAllocator<TinyLFUCacheTrait>;
-} // namespace facebook::cachelib
+}

cachelib/allocator/CacheAllocatorTinyLFU5BCache.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<TinyLFU5BCacheTrait>;
+}

cachelib/allocator/CacheAllocatorTinyLFUCache.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/CacheAllocator.h"
+
+namespace facebook::cachelib {
+template class CacheAllocator<TinyLFUCacheTrait>;
+}

cachelib/allocator/CacheTraits.h

@@ -25,13 +25,13 @@
 namespace facebook {
 namespace cachelib {
 // The cache traits supported by CacheLib.
-// Cache trait is a combination of MMType, AccessType and AccesTypeLock.
-// MMType is the type of MM (memory management) container used by the cache,
-// which controls a cache item's life time.
-// AccessType is the type of access container, which controls how an item is
-// accessed.
-// AccessTypeLock is the lock type for the access container that supports
-// multiple locking primitives
+// Cache trait is a combination of MMType, AccessType, AccesTypeLock and
+// CompressedPtr. MMType is the type of MM (memory management) container used by
+// the cache, which controls a cache item's life time. AccessType is the type of
+// access container, which controls how an item is accessed. AccessTypeLock is
+// the lock type for the access container that supports multiple locking
+// primitives CompressedPtr maps slabs and allocations within slabs in cache
+// memory.
 struct LruCacheTrait {
   using MMType = MMLru;
   using AccessType = ChainedHashTable;
@@ -60,5 +60,33 @@ struct TinyLFUCacheTrait {
   using CompressedPtrType = CompressedPtr4B;
 };
 
+struct Lru5BCacheTrait {
+  using MMType = MMLru;
+  using AccessType = ChainedHashTable;
+  using AccessTypeLocks = SharedMutexBuckets;
+  using CompressedPtrType = CompressedPtr5B;
+};
+
+struct Lru5BCacheWithSpinBucketsTrait {
+  using MMType = MMLru;
+  using AccessType = ChainedHashTable;
+  using AccessTypeLocks = SpinBuckets;
+  using CompressedPtrType = CompressedPtr5B;
+};
+
+struct Lru5B2QCacheTrait {
+  using MMType = MM2Q;
+  using AccessType = ChainedHashTable;
+  using AccessTypeLocks = SharedMutexBuckets;
+  using CompressedPtrType = CompressedPtr5B;
+};
+
+struct TinyLFU5BCacheTrait {
+  using MMType = MMTinyLFU;
+  using AccessType = ChainedHashTable;
+  using AccessTypeLocks = SharedMutexBuckets;
+  using CompressedPtrType = CompressedPtr5B;
+};
+
 } // namespace cachelib
 } // namespace facebook