enhance: rename some params and reduce default bitmapCardinalityLimit… (

milvus-io#36138) milvus-io#32900 Signed-off-by: luzhang <[email protected]> Co-authored-by: luzhang <[email protected]>
tedxu · Sep 12, 2024 · 5e5e87c · 5e5e87c
1 parent 11dbe1e
commit 5e5e87c
Show file tree

Hide file tree

Showing 8 changed files with 35 additions and 28 deletions.
diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -768,7 +768,6 @@ common:
     BeamWidthRatio: 4
   gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency.
   gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
-  bitmapIndexCardinalityBound: 500
   storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead
   # Default value: auto
   # Valid values: [auto, avx512, avx2, avx, sse4_2]

diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h
@@ -70,6 +70,8 @@ const int64_t DEFAULT_MAX_OUTPUT_SIZE = 67108864;  // bytes, 64MB
 
 const int64_t DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS = 10000;
 
-const int64_t DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND = 500;
+const int64_t DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND = 500;
+
+const int64_t DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT = 100;
 
 const size_t MARISA_NULL_KEY_ID = -1;
diff --git a/internal/core/src/index/BitmapIndex.cpp b/internal/core/src/index/BitmapIndex.cpp
@@ -97,7 +97,7 @@ BitmapIndex<T>::Build(size_t n, const T* data) {
         valid_bitset.set(i);
     }
 
-    if (data_.size() < DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND) {
+    if (data_.size() < DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND) {
         for (auto it = data_.begin(); it != data_.end(); ++it) {
             bitsets_[it->first] = ConvertRoaringToBitset(it->second);
         }
@@ -330,7 +330,7 @@ BitmapIndex<T>::DeserializeIndexMeta(const uint8_t* data_ptr,
 template <typename T>
 void
 BitmapIndex<T>::ChooseIndexLoadMode(int64_t index_length) {
-    if (index_length <= DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND) {
+    if (index_length <= DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND) {
         LOG_DEBUG("load bitmap index with bitset mode");
         build_mode_ = BitmapIndexBuildMode::BITSET;
     } else {

diff --git a/internal/core/src/index/HybridScalarIndex.cpp b/internal/core/src/index/HybridScalarIndex.cpp
@@ -32,7 +32,8 @@ HybridScalarIndex<T>::HybridScalarIndex(
     const storage::FileManagerContext& file_manager_context)
     : ScalarIndex<T>(HYBRID_INDEX_TYPE),
       is_built_(false),
-      bitmap_index_cardinality_limit_(DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND),
+      bitmap_index_cardinality_limit_(
+          DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT),
       file_manager_context_(file_manager_context) {
     if (file_manager_context.Valid()) {
         mem_file_manager_ =

diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go
@@ -454,7 +454,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
 	if indexType == indexparamcheck.IndexHybrid {
 		_, exist := indexParams[common.BitmapCardinalityLimitKey]
 		if !exist {
-			indexParams[common.BitmapCardinalityLimitKey] = paramtable.Get().CommonCfg.BitmapIndexCardinalityBound.GetValue()
+			indexParams[common.BitmapCardinalityLimitKey] = paramtable.Get().AutoIndexConfig.BitmapCardinalityLimit.GetValue()
 		}
 	}
 	checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType)

diff --git a/internal/proxy/task_index_test.go b/internal/proxy/task_index_test.go
@@ -664,7 +664,7 @@ func Test_parseIndexParams(t *testing.T) {
 		sortKeyValuePairs(cit.newIndexParams)
 		assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
 			{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
-			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
+			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
 		})
 	})
 
@@ -708,7 +708,7 @@ func Test_parseIndexParams(t *testing.T) {
 		sortKeyValuePairs(cit.newIndexParams)
 		assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
 			{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
-			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
+			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
 		})
 	})
 
@@ -937,7 +937,7 @@ func Test_parseIndexParams(t *testing.T) {
 		sortKeyValuePairs(cit.newIndexParams)
 		assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
 			{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
-			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
+			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
 		})
 	})
 
@@ -967,7 +967,7 @@ func Test_parseIndexParams(t *testing.T) {
 		sortKeyValuePairs(cit.newIndexParams)
 		assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
 			{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
-			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
+			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
 		})
 	})
 
@@ -997,7 +997,7 @@ func Test_parseIndexParams(t *testing.T) {
 		sortKeyValuePairs(cit.newIndexParams)
 		assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
 			{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
-			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
+			{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
 		})
 	})
 

diff --git a/pkg/util/paramtable/autoindex_param.go b/pkg/util/paramtable/autoindex_param.go
@@ -18,6 +18,7 @@ package paramtable
 
 import (
 	"fmt"
+	"strconv"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/pkg/common"
@@ -50,8 +51,14 @@ type autoIndexConfig struct {
 	ScalarVarcharIndexType ParamItem `refreshable:"true"`
 	ScalarBoolIndexType    ParamItem `refreshable:"true"`
 	ScalarFloatIndexType   ParamItem `refreshable:"true"`
+
+	BitmapCardinalityLimit ParamItem `refreshable:"true"`
 }
 
+const (
+	DefaultBitmapCardinalityLimit = 100
+)
+
 func (p *autoIndexConfig) init(base *BaseTable) {
 	p.Enable = ParamItem{
 		Key:          "autoIndex.enable",
@@ -195,6 +202,14 @@ func (p *autoIndexConfig) init(base *BaseTable) {
 	}
 	p.ScalarFloatIndexType.Init(base.mgr)
 
+	p.BitmapCardinalityLimit = ParamItem{
+		Key:          "scalarAutoIndex.params.bitmapCardinalityLimit",
+		Version:      "2.5.0",
+		DefaultValue: strconv.Itoa(DefaultBitmapCardinalityLimit),
+		Export:       true,
+	}
+	p.BitmapCardinalityLimit.Init(base.mgr)
+
 	p.ScalarVarcharIndexType = ParamItem{
 		Version: "2.4.0",
 		Formatter: func(v string) string {

diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go
@@ -48,14 +48,13 @@ const (
 	DefaultSessionTTL        = 30 // s
 	DefaultSessionRetryTimes = 30
 
-	DefaultMaxDegree                   = 56
-	DefaultSearchListSize              = 100
-	DefaultPQCodeBudgetGBRatio         = 0.125
-	DefaultBuildNumThreadsRatio        = 1.0
-	DefaultSearchCacheBudgetGBRatio    = 0.10
-	DefaultLoadNumThreadRatio          = 8.0
-	DefaultBeamWidthRatio              = 4.0
-	DefaultBitmapIndexCardinalityBound = 500
+	DefaultMaxDegree                = 56
+	DefaultSearchListSize           = 100
+	DefaultPQCodeBudgetGBRatio      = 0.125
+	DefaultBuildNumThreadsRatio     = 1.0
+	DefaultSearchCacheBudgetGBRatio = 0.10
+	DefaultLoadNumThreadRatio       = 8.0
+	DefaultBeamWidthRatio           = 4.0
 )
 
 // ComponentParam is used to quickly and easily access all components' configurations.
@@ -229,7 +228,6 @@ type commonConfig struct {
 	BeamWidthRatio                      ParamItem `refreshable:"true"`
 	GracefulTime                        ParamItem `refreshable:"true"`
 	GracefulStopTimeout                 ParamItem `refreshable:"true"`
-	BitmapIndexCardinalityBound         ParamItem `refreshable:"false"`
 
 	StorageType ParamItem `refreshable:"false"`
 	SimdType    ParamItem `refreshable:"false"`
@@ -502,14 +500,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
 	}
 	p.IndexSliceSize.Init(base.mgr)
 
-	p.BitmapIndexCardinalityBound = ParamItem{
-		Key:          "common.bitmapIndexCardinalityBound",
-		Version:      "2.5.0",
-		DefaultValue: strconv.Itoa(DefaultBitmapIndexCardinalityBound),
-		Export:       true,
-	}
-	p.BitmapIndexCardinalityBound.Init(base.mgr)
-
 	p.EnableMaterializedView = ParamItem{
 		Key:          "common.materializedView.enabled",
 		Version:      "2.4.6",