store raft data in dedicated blob file (#7719)

ref #6827

dbms/src/Interpreters/Settings.h

@@ -215,6 +215,7 @@ struct Settings
     M(SettingUInt64, dt_checksum_frame_size, DBMS_DEFAULT_BUFFER_SIZE, "Frame size for delta tree stable storage")                                                                                                                      \
                                                                                                                                                                                                                                         \
     M(SettingDouble, dt_page_gc_threshold, 0.5, "Max valid rate of deciding to do a GC in PageStorage")                                                                                                                                 \
+    M(SettingDouble, dt_page_gc_threshold_raft_data, 0.05, "Max valid rate of deciding to do a GC for BlobFile storing PageData in PageStorage")                                                                                        \
     M(SettingBool, dt_enable_read_thread, true, "Enable storage read thread or not")                                                                                                                                                    \
     M(SettingBool, dt_enable_bitmap_filter, true, "Use bitmap filter to read data or not")                                                                                                                                              \
     M(SettingDouble, dt_read_thread_count_scale, 1.0, "Number of read thread = number of logical cpu cores * dt_read_thread_count_scale.  Only has meaning at server startup.")                                                         \

dbms/src/Storages/Page/Config.h

@@ -99,6 +99,7 @@ struct PageStorageConfig
     SettingUInt64 blob_file_limit_size = BLOBFILE_LIMIT_SIZE;
     SettingUInt64 blob_spacemap_type = 2;
     SettingDouble blob_heavy_gc_valid_rate = 0.5;
+    SettingDouble blob_heavy_gc_valid_rate_raft_data = 0.05;
     SettingUInt64 blob_block_alignment_bytes = 0;
 
     SettingUInt64 wal_roll_size = PAGE_META_ROLL_SIZE;
@@ -122,6 +123,7 @@ struct PageStorageConfig
         blob_file_limit_size = rhs.blob_file_limit_size;
         blob_spacemap_type = rhs.blob_spacemap_type;
         blob_heavy_gc_valid_rate = rhs.blob_heavy_gc_valid_rate;
+        blob_heavy_gc_valid_rate_raft_data = rhs.blob_heavy_gc_valid_rate_raft_data;
         blob_block_alignment_bytes = rhs.blob_block_alignment_bytes;
 
         wal_roll_size = rhs.wal_roll_size;
@@ -150,11 +152,12 @@ struct PageStorageConfig
         return fmt::format(
             "PageStorageConfig {{"
             "blob_file_limit_size: {}, blob_spacemap_type: {}, "
-            "blob_heavy_gc_valid_rate: {:.3f}, blob_block_alignment_bytes: {}, "
-            "wal_roll_size: {}, wal_max_persisted_log_files: {}}}",
+            "blob_heavy_gc_valid_rate: {:.3f}, blob_heavy_gc_valid_rate_raft_data: {:.3f}, "
+            "blob_block_alignment_bytes: {}, wal_roll_size: {}, wal_max_persisted_log_files: {}}}",
             blob_file_limit_size.get(),
             blob_spacemap_type.get(),
             blob_heavy_gc_valid_rate.get(),
+            blob_heavy_gc_valid_rate_raft_data.get(),
             blob_block_alignment_bytes.get(),
             wal_roll_size.get(),
             wal_max_persisted_log_files.get());

dbms/src/Storages/Page/PageStorage.cpp

@@ -740,7 +740,7 @@ void PageWriter::writeIntoMixMode(WriteBatch && write_batch, WriteLimiterPtr wri
 
 void PageWriter::writeIntoUni(UniversalWriteBatch && write_batch, WriteLimiterPtr write_limiter) const
 {
-    uni_ps->write(std::move(write_batch), write_limiter);
+    uni_ps->write(std::move(write_batch), PageType::Normal, write_limiter);
 }
 
 PageStorageConfig PageWriter::getSettings() const

dbms/src/Storages/Page/V3/Blob/BlobConfig.h

@@ -28,17 +28,20 @@ struct BlobConfig
     SettingUInt64 spacemap_type = SpaceMap::SpaceMapType::SMAP64_STD_MAP;
     SettingUInt64 block_alignment_bytes = 0;
     SettingDouble heavy_gc_valid_rate = 0.2;
+    SettingDouble heavy_gc_valid_rate_raft_data = 0.05;
 
     String toString()
     {
         return fmt::format("BlobStore Config Info: "
                            "[file_limit_size={}] [spacemap_type={}] "
                            "[block_alignment_bytes={}] "
-                           "[heavy_gc_valid_rate={}]",
+                           "[heavy_gc_valid_rate={}]"
+                           "[heavy_gc_valid_rate_raft_data={}]",
                            file_limit_size,
                            spacemap_type,
                            block_alignment_bytes,
-                           heavy_gc_valid_rate);
+                           heavy_gc_valid_rate,
+                           heavy_gc_valid_rate_raft_data);
     }
 
     static BlobConfig from(const PageStorageConfig & config)
@@ -48,6 +51,7 @@ struct BlobConfig
         blob_config.file_limit_size = config.blob_file_limit_size;
         blob_config.spacemap_type = config.blob_spacemap_type;
         blob_config.heavy_gc_valid_rate = config.blob_heavy_gc_valid_rate;
+        blob_config.heavy_gc_valid_rate_raft_data = config.blob_heavy_gc_valid_rate_raft_data;
         blob_config.block_alignment_bytes = config.blob_block_alignment_bytes;
 
         return blob_config;

dbms/src/Storages/Page/V3/Blob/BlobStat.cpp

@@ -35,7 +35,10 @@ namespace DB::PS::V3
   * BlobStats methods *
   *********************/
 
-BlobStats::BlobStats(LoggerPtr log_, PSDiskDelegatorPtr delegator_, BlobConfig & config_)
+BlobStats::BlobStats(
+    LoggerPtr log_,
+    PSDiskDelegatorPtr delegator_,
+    BlobConfig & config_)
     : log(std::move(log_))
     , delegator(delegator_)
     , config(config_)
@@ -90,20 +93,15 @@ std::pair<BlobFileId, String> BlobStats::getBlobIdFromName(String blob_name)
 
 void BlobStats::restore()
 {
-    BlobFileId max_restored_file_id = 0;
-
     for (auto & [path, stats] : stats_map)
     {
         (void)path;
         for (const auto & stat : stats)
         {
             stat->recalculateSpaceMap();
-            max_restored_file_id = std::max(stat->id, max_restored_file_id);
+            cur_max_id = std::max(stat->id, cur_max_id);
         }
     }
-
-    // restore `roll_id`
-    roll_id = max_restored_file_id + 1;
 }
 
 std::lock_guard<std::mutex> BlobStats::lock() const
@@ -113,15 +111,6 @@ std::lock_guard<std::mutex> BlobStats::lock() const
 
 BlobStats::BlobStatPtr BlobStats::createStat(BlobFileId blob_file_id, UInt64 max_caps, const std::lock_guard<std::mutex> & guard)
 {
-    // New blob file id won't bigger than roll_id
-    if (blob_file_id > roll_id)
-    {
-        throw Exception(fmt::format("BlobStats won't create [blob_id={}], which is bigger than [roll_id={}]",
-                                    blob_file_id,
-                                    roll_id),
-                        ErrorCodes::LOGICAL_ERROR);
-    }
-
     for (auto & [path, stats] : stats_map)
     {
         (void)path;
@@ -137,15 +126,7 @@ BlobStats::BlobStatPtr BlobStats::createStat(BlobFileId blob_file_id, UInt64 max
     }
 
     // Create a stat without checking the file_id exist or not
-    auto stat = createStatNotChecking(blob_file_id, max_caps, guard);
-
-    // Roll to the next new blob id
-    if (blob_file_id == roll_id)
-    {
-        roll_id++;
-    }
-
-    return stat;
+    return createStatNotChecking(blob_file_id, max_caps, guard);
 }
 
 BlobStats::BlobStatPtr BlobStats::createStatNotChecking(BlobFileId blob_file_id, UInt64 max_caps, const std::lock_guard<std::mutex> &)
@@ -203,14 +184,16 @@ void BlobStats::eraseStat(BlobFileId blob_file_id, const std::lock_guard<std::mu
     eraseStat(std::move(stat), lock);
 }
 
-std::pair<BlobStats::BlobStatPtr, BlobFileId> BlobStats::chooseStat(size_t buf_size, const std::lock_guard<std::mutex> &)
+std::pair<BlobStats::BlobStatPtr, BlobFileId> BlobStats::chooseStat(size_t buf_size, PageType page_type, const std::lock_guard<std::mutex> &)
 {
     BlobStatPtr stat_ptr = nullptr;
 
     // No stats exist
     if (stats_map.empty())
     {
-        return std::make_pair(nullptr, roll_id);
+        auto next_id = PageTypeUtils::nextFileID(page_type, cur_max_id);
+        cur_max_id = next_id;
+        return std::make_pair(nullptr, next_id);
     }
 
     // If the stats_map size changes, or stats_map_path_index is out of range,
@@ -226,6 +209,9 @@ std::pair<BlobStats::BlobStatPtr, BlobFileId> BlobStats::chooseStat(size_t buf_s
         // Try to find a suitable stat under current path (path=`stats_iter->first`)
         for (const auto & stat : stats_iter->second)
         {
+            if (PageTypeUtils::getPageType(stat->id) != page_type)
+                continue;
+
             auto defer_lock = stat->defer_lock();
             if (defer_lock.try_lock() && stat->isNormal() && stat->sm_max_caps >= buf_size)
             {
@@ -245,7 +231,9 @@ std::pair<BlobStats::BlobStatPtr, BlobFileId> BlobStats::chooseStat(size_t buf_s
     stats_map_path_index += path_iter_idx + 1;
 
     // Can not find a suitable stat under all paths
-    return std::make_pair(nullptr, roll_id);
+    auto next_id = PageTypeUtils::nextFileID(page_type, cur_max_id);
+    cur_max_id = next_id;
+    return std::make_pair(nullptr, next_id);
 }
 
 BlobStats::BlobStatPtr BlobStats::blobIdToStat(BlobFileId file_id, bool ignore_not_exist)
@@ -358,5 +346,4 @@ void BlobStats::BlobStat::recalculateCapacity()
 {
     sm_max_caps = smap->updateAccurateMaxCapacity();
 }
-
 } // namespace DB::PS::V3

dbms/src/Storages/Page/V3/Blob/BlobStat.h

@@ -15,8 +15,10 @@
 #pragma once
 
 #include <Common/Logger.h>
+#include <Storages/Page/Page.h>
 #include <Storages/Page/V3/Blob/BlobConfig.h>
 #include <Storages/Page/V3/PageEntry.h>
+#include <Storages/Page/V3/PageType.h>
 #include <Storages/Page/V3/spacemap/SpaceMap.h>
 #include <Storages/PathPool.h>
 #include <common/types.h>
@@ -58,7 +60,11 @@ class BlobStats
         double sm_valid_rate = 0.0;
 
     public:
-        BlobStat(BlobFileId id_, SpaceMap::SpaceMapType sm_type, UInt64 sm_max_caps_, BlobStatType type_)
+        BlobStat(
+            BlobFileId id_,
+            SpaceMap::SpaceMapType sm_type,
+            UInt64 sm_max_caps_,
+            BlobStatType type_)
             : id(id_)
             , type(type_)
             , smap(SpaceMap::createSpaceMap(sm_type, 0, sm_max_caps_))
@@ -119,7 +125,10 @@ class BlobStats
     using BlobStatPtr = std::shared_ptr<BlobStat>;
 
 public:
-    BlobStats(LoggerPtr log_, PSDiskDelegatorPtr delegator_, BlobConfig & config);
+    BlobStats(
+        LoggerPtr log_,
+        PSDiskDelegatorPtr delegator_,
+        BlobConfig & config);
 
     // Don't require a lock from BlobStats When you already hold a BlobStat lock
     //
@@ -155,7 +164,7 @@ class BlobStats
          * The `INVALID_BLOBFILE_ID` means that you don't need create a new `BlobFile`.
          * 
          */
-    std::pair<BlobStatPtr, BlobFileId> chooseStat(size_t buf_size, const std::lock_guard<std::mutex> &);
+    std::pair<BlobStatPtr, BlobFileId> chooseStat(size_t buf_size, PageType page_type, const std::lock_guard<std::mutex> &);
 
     BlobStatPtr blobIdToStat(BlobFileId file_id, bool ignore_not_exist = false);
 
@@ -184,7 +193,8 @@ class BlobStats
     BlobConfig & config;
 
     mutable std::mutex lock_stats;
-    BlobFileId roll_id = 1;
+    const PageTypeAndConfig page_type_and_config;
+    BlobFileId cur_max_id = 1;
     // Index for selecting next path for creating new blobfile
     UInt32 stats_map_path_index = 0;
     std::map<String, std::list<BlobStatPtr>> stats_map;