From a4e37a3e423c00bce676ad319ad5f265e04dd424 Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Fri, 19 Jan 2024 13:25:46 +0800 Subject: [PATCH] Storages: Update the length of Block queue and the number of read threads (#8702) (#8706) close pingcap/tiflash#8703 --- dbms/src/Interpreters/Settings.h | 2 +- dbms/src/Storages/DeltaMerge/SegmentReadTaskPool.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 19908f987e4..5550fca69e9 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -219,7 +219,7 @@ struct Settings M(SettingUInt64, dt_max_sharing_column_bytes_for_all, 2048 * Constant::MB, "Memory limitation for data sharing of all requests. 0 means disable data sharing") \ M(SettingUInt64, dt_max_sharing_column_count, 5, "ColumnPtr object limitation for data sharing of each DMFileReader::Stream. 0 means disable data sharing") \ M(SettingBool, dt_enable_bitmap_filter, true, "Use bitmap filter to read data or not") \ - M(SettingDouble, dt_read_thread_count_scale, 1.0, "Number of read thread = number of logical cpu cores * dt_read_thread_count_scale. Only has meaning at server startup.") \ + M(SettingDouble, dt_read_thread_count_scale, 2.0, "Number of read thread = number of logical cpu cores * dt_read_thread_count_scale. Only has meaning at server startup.") \ M(SettingDouble, dt_filecache_max_downloading_count_scale, 1.0, "Max downloading task count of FileCache = io thread count * dt_filecache_max_downloading_count_scale.") \ M(SettingUInt64, dt_filecache_min_age_seconds, 1800, "Files of the same priority can only be evicted from files that were not accessed within `dt_filecache_min_age_seconds` seconds.") \ M(SettingUInt64, dt_small_file_size_threshold, 128 * 1024, "When S3 is enabled, file size less than dt_small_file_size_threshold will be merged before uploading to S3") \ diff --git a/dbms/src/Storages/DeltaMerge/SegmentReadTaskPool.h b/dbms/src/Storages/DeltaMerge/SegmentReadTaskPool.h index 6a445b01f7c..84a1e1592e5 100644 --- a/dbms/src/Storages/DeltaMerge/SegmentReadTaskPool.h +++ b/dbms/src/Storages/DeltaMerge/SegmentReadTaskPool.h @@ -165,10 +165,10 @@ class SegmentReadTaskPool : private boost::noncopyable , unordered_input_stream_ref_count(0) , exception_happened(false) , mem_tracker(current_memory_tracker == nullptr ? nullptr : current_memory_tracker->shared_from_this()) - // If the queue is too short, only 1 in the extreme case, it may cause the computation thread - // to encounter empty queues frequently, resulting in too much waiting and thread context - // switching, so we limit the lower limit to 3, which provides two blocks of buffer space. - , block_slot_limit(std::max(num_streams_, 3)) + // to encounter empty queues frequently, resulting in too much waiting and thread context switching. + // We limit the length of block queue to be 1.5 times of `num_streams_`, and in the extreme case, + // when `num_streams_` is 1, `block_slot_limit` is at least 2. + , block_slot_limit(std::ceil(num_streams_ * 1.5)) // Limiting the minimum number of reading segments to 2 is to avoid, as much as possible, // situations where the computation may be faster and the storage layer may not be able to keep up. , active_segment_limit(std::max(num_streams_, 2))