refactor

src/common/src/config.rs

@@ -195,6 +195,10 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::full_gc_object_limit")]
     pub full_gc_object_limit: u64,
 
+    /// Duration in seconds to retain garbage collection history data.
+    #[serde(default = "default::meta::gc_history_retention_time_sec")]
+    pub gc_history_retention_time_sec: u64,
+
     /// Max number of inflight time travel query.
     #[serde(default = "default::meta::max_inflight_time_travel_query")]
     pub max_inflight_time_travel_query: u64,
@@ -1355,6 +1359,10 @@ pub mod default {
             3600 * 3
         }
 
+        pub fn gc_history_retention_time_sec() -> u64 {
+            3600 * 6
+        }
+
         pub fn full_gc_interval_sec() -> u64 {
             600
         }

src/config/docs.md

@@ -39,6 +39,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | event_log_enabled |  | true |
 | full_gc_interval_sec | Interval of automatic hummock full GC. | 600 |
 | full_gc_object_limit | Max number of object per full GC job can fetch. | 100000 |
+| gc_history_retention_time_sec | Duration in seconds to retain garbage collection history data. | 21600 |
 | hummock_time_travel_snapshot_interval | The interval at which a Hummock version snapshot is taken for time travel. Larger value indicates less storage overhead but worse query performance. | 100 |
 | hummock_version_checkpoint_interval_sec | Interval of hummock version checkpoint. | 30 |
 | hybrid_partition_vnode_count | Count of partitions of tables in default group and materialized view group. The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment. Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table. Set it zero to disable this feature. | 4 |

src/config/example.toml

@@ -17,6 +17,7 @@ dir = "./"
 min_sst_retention_time_sec = 10800
 full_gc_interval_sec = 600
 full_gc_object_limit = 100000
+gc_history_retention_time_sec = 21600
 max_inflight_time_travel_query = 1000
 periodic_compaction_interval_sec = 60
 vacuum_interval_sec = 30

src/meta/node/src/lib.rs

@@ -367,6 +367,7 @@ pub fn start(
                 min_sst_retention_time_sec: config.meta.min_sst_retention_time_sec,
                 full_gc_interval_sec: config.meta.full_gc_interval_sec,
                 full_gc_object_limit: config.meta.full_gc_object_limit,
+                gc_history_retention_time_sec: config.meta.gc_history_retention_time_sec,
                 max_inflight_time_travel_query: config.meta.max_inflight_time_travel_query,
                 enable_committed_sst_sanity_check: config.meta.enable_committed_sst_sanity_check,
                 periodic_compaction_interval_sec: config.meta.periodic_compaction_interval_sec,

src/meta/src/hummock/manager/context.rs

@@ -332,10 +332,11 @@ async fn check_gc_history(
     });
     let res: Vec<_> = stream::iter(futures).buffer_unordered(10).collect().await;
     let res: Result<Vec<_>> = res.into_iter().collect();
-    let expired_object_ids: Vec<_> = res?.into_iter().flatten().collect();
-    if expired_object_ids.is_empty() {
+    let mut expired_object_ids = res?.into_iter().flatten().peekable();
+    if expired_object_ids.peek().is_none() {
         return Ok(());
     }
+    let expired_object_ids: Vec<_> = expired_object_ids.collect();
     tracing::error!(
         ?expired_object_ids,
         "new SSTs are rejected because they have already been GCed"

src/meta/src/hummock/manager/gc.rs

@@ -362,9 +362,8 @@ impl HummockManager {
             mark_delete_at: Set(dt.naive_utc()),
         });
         let db = &self.meta_store_ref().conn;
-        let gc_history_retention_sec = self.env.opts.min_sst_retention_time_sec * 2;
         let gc_history_low_watermark = DateTime::from_timestamp(
-            now.saturating_sub(gc_history_retention_sec)
+            now.saturating_sub(self.env.opts.gc_history_retention_time_sec)
                 .try_into()
                 .unwrap(),
             0,

src/meta/src/manager/env.rs

@@ -126,6 +126,8 @@ pub struct MetaOpts {
     pub full_gc_interval_sec: u64,
     /// Max number of object per full GC job can fetch.
     pub full_gc_object_limit: u64,
+    /// Duration in seconds to retain garbage collection history data.
+    pub gc_history_retention_time_sec: u64,
     /// Max number of inflight time travel query.
     pub max_inflight_time_travel_query: u64,
     /// Enable sanity check when SSTs are committed
@@ -262,6 +264,7 @@ impl MetaOpts {
             min_sst_retention_time_sec: 3600 * 24 * 7,
             full_gc_interval_sec: 3600 * 24 * 7,
             full_gc_object_limit: 100_000,
+            gc_history_retention_time_sec: 3600 * 24 * 7,
             max_inflight_time_travel_query: 1000,
             enable_committed_sst_sanity_check: false,
             periodic_compaction_interval_sec: 60,