From 679ad1dd2cec6c4840120d1e2a216cdffd19f139 Mon Sep 17 00:00:00 2001 From: lixinguo Date: Fri, 6 Sep 2024 10:50:11 +0800 Subject: [PATCH] fix: to filldefault value when nullable and default value are enable both Signed-off-by: lixinguo --- internal/proxy/validate_util.go | 46 ++-- internal/proxy/validate_util_test.go | 329 +++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 28 deletions(-) diff --git a/internal/proxy/validate_util.go b/internal/proxy/validate_util.go index abf6a59b9d7a4..323e7d7eeeec4 100644 --- a/internal/proxy/validate_util.go +++ b/internal/proxy/validate_util.go @@ -353,7 +353,12 @@ func (v *validateUtil) fillWithNullValue(field *schemapb.FieldData, fieldSchema } case *schemapb.ScalarField_ArrayData: - // Todo: support it + if fieldSchema.GetNullable() { + sd.ArrayData.Data, err = fillWithNullValueImpl(sd.ArrayData.Data, field.GetValidData()) + if err != nil { + return err + } + } case *schemapb.ScalarField_JsonData: if fieldSchema.GetNullable() { @@ -391,10 +396,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } - case *schemapb.ScalarField_IntData: if len(field.GetValidData()) != numRows { msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName()) @@ -406,10 +407,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } - case *schemapb.ScalarField_LongData: if len(field.GetValidData()) != numRows { msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName()) @@ -420,9 +417,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche if err != nil { return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } case *schemapb.ScalarField_FloatData: if len(field.GetValidData()) != numRows { @@ -435,10 +429,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } - case *schemapb.ScalarField_DoubleData: if len(field.GetValidData()) != numRows { msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName()) @@ -450,10 +440,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } - case *schemapb.ScalarField_StringData: if len(field.GetValidData()) != numRows { msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName()) @@ -465,10 +451,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } - case *schemapb.ScalarField_ArrayData: // Todo: support it log.Error("array type not support default value", zap.String("fieldSchemaName", field.GetFieldName())) @@ -485,10 +467,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return err } - if !fieldSchema.GetNullable() { - field.ValidData = []bool{} - } - default: return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String())) } @@ -501,6 +479,18 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String())) } + if !typeutil.IsVectorType(field.Type) { + if fieldSchema.GetNullable() { + validData := make([]bool, numRows) + for i := range validData { + validData[i] = true + } + field.ValidData = validData + } else { + field.ValidData = []bool{} + } + } + err = checkValidData(field, fieldSchema, numRows) if err != nil { return err diff --git a/internal/proxy/validate_util_test.go b/internal/proxy/validate_util_test.go index 3e56fa1f45a9c..6438a9a5481e1 100644 --- a/internal/proxy/validate_util_test.go +++ b/internal/proxy/validate_util_test.go @@ -3089,6 +3089,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) { assert.NoError(t, err) flag := checkfillWithValueData(data[0].GetScalars().GetBoolData().Data, schema.Fields[0].GetDefaultValue().GetBoolData(), 2) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.True(t, flag) + }) + + t.Run("bool scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_Bool, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_BoolData{ + BoolData: &schemapb.BoolArray{ + Data: []bool{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + key := true + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_BoolData{ + BoolData: key, + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + + flag := checkfillWithValueData(data[0].GetScalars().GetBoolData().Data, schema.Fields[0].GetDefaultValue().GetBoolData(), 2) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.True(t, flag) }) @@ -3475,6 +3522,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) { assert.NoError(t, err) flag := checkfillWithValueData(data[0].GetScalars().GetIntData().Data, schema.Fields[0].GetDefaultValue().GetIntData(), 2) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.True(t, flag) + }) + + t.Run("int scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_Int32, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: []int32{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DataType: schemapb.DataType_Int32, + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_IntData{ + IntData: 1, + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + + flag := checkfillWithValueData(data[0].GetScalars().GetIntData().Data, schema.Fields[0].GetDefaultValue().GetIntData(), 2) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.True(t, flag) }) @@ -3862,6 +3956,52 @@ func Test_validateUtil_fillWithValue(t *testing.T) { assert.NoError(t, err) flag := checkfillWithValueData(data[0].GetScalars().GetLongData().Data, schema.Fields[0].GetDefaultValue().GetLongData(), 2) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.True(t, flag) + }) + + t.Run("long scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_Int64, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_LongData{ + LongData: &schemapb.LongArray{ + Data: []int64{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DataType: schemapb.DataType_Int32, + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_LongData{ + LongData: 1, + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + flag := checkfillWithValueData(data[0].GetScalars().GetLongData().Data, schema.Fields[0].GetDefaultValue().GetLongData(), 2) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.True(t, flag) }) @@ -4251,6 +4391,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) { assert.NoError(t, err) flag := checkfillWithValueData(data[0].GetScalars().GetFloatData().Data, schema.Fields[0].GetDefaultValue().GetFloatData(), 2) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.True(t, flag) + }) + + t.Run("float scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_Float, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_FloatData{ + FloatData: &schemapb.FloatArray{ + Data: []float32{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DataType: schemapb.DataType_Float, + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_FloatData{ + FloatData: 1, + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + + flag := checkfillWithValueData(data[0].GetScalars().GetFloatData().Data, schema.Fields[0].GetDefaultValue().GetFloatData(), 2) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.True(t, flag) }) @@ -4639,6 +4826,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) { assert.NoError(t, err) flag := checkfillWithValueData(data[0].GetScalars().GetDoubleData().Data, schema.Fields[0].GetDefaultValue().GetDoubleData(), 2) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.True(t, flag) + }) + + t.Run("double scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_Double, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_DoubleData{ + DoubleData: &schemapb.DoubleArray{ + Data: []float64{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DataType: schemapb.DataType_Double, + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_DoubleData{ + DoubleData: 1, + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + + flag := checkfillWithValueData(data[0].GetScalars().GetDoubleData().Data, schema.Fields[0].GetDefaultValue().GetDoubleData(), 2) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.True(t, flag) }) @@ -5027,6 +5261,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) { assert.NoError(t, err) flag := checkfillWithValueData(data[0].GetScalars().GetStringData().Data, schema.Fields[0].GetDefaultValue().GetStringData(), 2) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.True(t, flag) + }) + + t.Run("string scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_VarChar, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_StringData{ + StringData: &schemapb.StringArray{ + Data: []string{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DataType: schemapb.DataType_VarChar, + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_StringData{ + StringData: "b", + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + + flag := checkfillWithValueData(data[0].GetScalars().GetStringData().Data, schema.Fields[0].GetDefaultValue().GetStringData(), 2) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.True(t, flag) }) @@ -5409,6 +5690,54 @@ func Test_validateUtil_fillWithValue(t *testing.T) { flag, err := checkJsonfillWithValueData(data[0].GetScalars().GetJsonData().Data, schema.Fields[0].GetDefaultValue().GetBytesData(), 2) assert.True(t, flag) + assert.Equal(t, 0, len(data[0].GetValidData())) + assert.NoError(t, err) + }) + + t.Run("json scalars schema set both nullable==true and default value", func(t *testing.T) { + data := []*schemapb.FieldData{ + { + FieldName: "test", + Type: schemapb.DataType_JSON, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_JsonData{ + JsonData: &schemapb.JSONArray{ + Data: [][]byte{}, + }, + }, + }, + }, + ValidData: []bool{false, false}, + }, + } + + schema := &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "test", + DataType: schemapb.DataType_BinaryVector, + DefaultValue: &schemapb.ValueField{ + Data: &schemapb.ValueField_BytesData{ + BytesData: []byte("{\"Hello\":\"world\"}"), + }, + }, + Nullable: true, + }, + }, + } + h, err := typeutil.CreateSchemaHelper(schema) + assert.NoError(t, err) + + v := newValidateUtil() + + err = v.fillWithValue(data, h, 2) + + assert.NoError(t, err) + + flag, err := checkJsonfillWithValueData(data[0].GetScalars().GetJsonData().Data, schema.Fields[0].GetDefaultValue().GetBytesData(), 2) + assert.True(t, flag) + assert.Equal(t, []bool{true, true}, data[0].GetValidData()) assert.NoError(t, err) })