From 8cda48a96ab31d8e8c25c796f102846858c1f550 Mon Sep 17 00:00:00 2001 From: "yihao.dai" Date: Tue, 24 Sep 2024 12:21:13 +0800 Subject: [PATCH] enhance: Use mmap.scalarIndex config for text index (#36400) issue: https://github.com/milvus-io/milvus/issues/35273 --------- Signed-off-by: bigsheeper --- internal/core/src/common/type_c.h | 2 +- internal/core/src/segcore/SegmentSealedImpl.cpp | 2 +- internal/core/src/storage/Types.h | 13 +++++++------ internal/core/src/storage/storage_c.cpp | 3 ++- internal/util/initcore/init_core.go | 12 ++++++------ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/internal/core/src/common/type_c.h b/internal/core/src/common/type_c.h index 6aeba50689ad7..af751e5da1378 100644 --- a/internal/core/src/common/type_c.h +++ b/internal/core/src/common/type_c.h @@ -100,7 +100,7 @@ typedef struct CMmapConfig { uint64_t disk_limit; uint64_t fix_file_size; bool growing_enable_mmap; - bool enable_mmap; + bool scalar_index_enable_mmap; } CMmapConfig; typedef struct CTraceConfig { diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp index 71e6185c74eb0..da6c0b86cb38f 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.cpp +++ b/internal/core/src/segcore/SegmentSealedImpl.cpp @@ -2031,7 +2031,7 @@ SegmentSealedImpl::CreateTextIndex(FieldId field_id) { const auto& field_meta = schema_->operator[](field_id); auto& cfg = storage::MmapManager::GetInstance().GetMmapConfig(); std::unique_ptr index; - if (!cfg.GetEnableMmap()) { + if (!cfg.GetScalarIndexEnableMmap()) { // build text index in ram. index = std::make_unique( std::numeric_limits::max(), diff --git a/internal/core/src/storage/Types.h b/internal/core/src/storage/Types.h index e33f5d803f297..cde1484fd1050 100644 --- a/internal/core/src/storage/Types.h +++ b/internal/core/src/storage/Types.h @@ -125,7 +125,7 @@ struct MmapConfig { uint64_t disk_limit; uint64_t fix_file_size; bool growing_enable_mmap; - bool enable_mmap; + bool scalar_index_enable_mmap; bool GetEnableGrowingMmap() const { return growing_enable_mmap; @@ -135,12 +135,12 @@ struct MmapConfig { this->growing_enable_mmap = flag; } bool - GetEnableMmap() const { - return enable_mmap; + GetScalarIndexEnableMmap() const { + return scalar_index_enable_mmap; } void - SetEnableMmap(bool flag) { - this->enable_mmap = flag; + SetScalarIndexEnableMmap(bool flag) { + this->scalar_index_enable_mmap = flag; } std::string GetMmapPath() { @@ -154,7 +154,8 @@ struct MmapConfig { << ", disk_limit=" << disk_limit / (1024 * 1024) << "MB" << ", fix_file_size=" << fix_file_size / (1024 * 1024) << "MB" << ", growing_enable_mmap=" << std::boolalpha << growing_enable_mmap - << ", enable_mmap=" << std::boolalpha << enable_mmap << "]"; + << ", scalar_index_enable_mmap=" << std::boolalpha + << scalar_index_enable_mmap << "]"; return ss.str(); } }; diff --git a/internal/core/src/storage/storage_c.cpp b/internal/core/src/storage/storage_c.cpp index 8a337b66fe8fa..bab1e7616425c 100644 --- a/internal/core/src/storage/storage_c.cpp +++ b/internal/core/src/storage/storage_c.cpp @@ -95,7 +95,8 @@ InitMmapManager(CMmapConfig c_mmap_config) { mmap_config.disk_limit = c_mmap_config.disk_limit; mmap_config.fix_file_size = c_mmap_config.fix_file_size; mmap_config.growing_enable_mmap = c_mmap_config.growing_enable_mmap; - mmap_config.enable_mmap = c_mmap_config.enable_mmap; + mmap_config.scalar_index_enable_mmap = + c_mmap_config.scalar_index_enable_mmap; milvus::storage::MmapManager::GetInstance().Init(mmap_config); return milvus::SuccessCStatus(); } catch (std::exception& e) { diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index 93075073e2603..a43a16e2dcf9d 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -181,12 +181,12 @@ func InitMmapManager(params *paramtable.ComponentParam) error { diskLimit := uint64(float64(params.QueryNodeCfg.MaxMmapDiskPercentageForMmapManager.GetAsUint64()*diskCapacity) * 0.01) mmapFileSize := params.QueryNodeCfg.FixedFileSizeForMmapManager.GetAsFloat() * 1024 * 1024 mmapConfig := C.CMmapConfig{ - cache_read_ahead_policy: cCacheReadAheadPolicy, - mmap_path: cMmapChunkManagerDir, - disk_limit: C.uint64_t(diskLimit), - fix_file_size: C.uint64_t(mmapFileSize), - growing_enable_mmap: C.bool(params.QueryNodeCfg.GrowingMmapEnabled.GetAsBool()), - enable_mmap: C.bool(params.QueryNodeCfg.MmapEnabled.GetAsBool()), + cache_read_ahead_policy: cCacheReadAheadPolicy, + mmap_path: cMmapChunkManagerDir, + disk_limit: C.uint64_t(diskLimit), + fix_file_size: C.uint64_t(mmapFileSize), + growing_enable_mmap: C.bool(params.QueryNodeCfg.GrowingMmapEnabled.GetAsBool()), + scalar_index_enable_mmap: C.bool(params.QueryNodeCfg.MmapScalarIndex.GetAsBool()), } status := C.InitMmapManager(mmapConfig) return HandleCStatus(&status, "InitMmapManager failed")