diff --git a/client/column/array.go b/client/column/array.go index 5eb701b75f4a6..90251fe689e5d 100644 --- a/client/column/array.go +++ b/client/column/array.go @@ -17,62 +17,18 @@ package column import ( - "fmt" - - "github.com/cockroachdb/errors" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/client/v2/entity" ) -// ColumnVarCharArray generated columns type for VarChar -type ColumnVarCharArray struct { - ColumnBase - name string - values [][][]byte -} - -// Name returns column name -func (c *ColumnVarCharArray) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnVarCharArray) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnVarCharArray) Len() int { - return len(c.values) -} - -func (c *ColumnVarCharArray) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnVarCharArray{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } +// columnArrayBase implement `Column` interface +// it provided specified `FieldData` behavior for Array columns. +type columnArrayBase[T any] struct { + *genericColumnBase[[]T] + elementType entity.FieldType } -// Get returns value at index as interface{}. -func (c *ColumnVarCharArray) Get(idx int) (interface{}, error) { - var r []string // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnVarCharArray) FieldData() *schemapb.FieldData { +func (c *columnArrayBase[T]) FieldData() *schemapb.FieldData { fd := &schemapb.FieldData{ Type: schemapb.DataType_Array, FieldName: c.name, @@ -80,24 +36,15 @@ func (c *ColumnVarCharArray) FieldData() *schemapb.FieldData { data := make([]*schemapb.ScalarField, 0, c.Len()) for _, arr := range c.values { - converted := make([]string, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, string(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: converted, - }, - }, - }) + data = append(data, slice2Scalar(arr, c.elementType)) } + fd.Field = &schemapb.FieldData_Scalars{ Scalars: &schemapb.ScalarField{ Data: &schemapb.ScalarField_ArrayData{ ArrayData: &schemapb.ArrayArray{ Data: data, - ElementType: schemapb.DataType_VarChar, + ElementType: schemapb.DataType(c.elementType), }, }, }, @@ -105,36 +52,162 @@ func (c *ColumnVarCharArray) FieldData() *schemapb.FieldData { return fd } -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnVarCharArray) ValueByIdx(idx int) ([][]byte, error) { - var r [][]byte // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") +func (c *columnArrayBase[T]) ElementType() entity.FieldType { + return c.elementType +} + +func (c *columnArrayBase[T]) slice(start, end int) *columnArrayBase[T] { + return &columnArrayBase[T]{ + genericColumnBase: c.genericColumnBase.slice(start, end), + elementType: c.elementType, + } +} + +func newArrayBase[T any](fieldName string, data [][]T, elementType entity.FieldType) *columnArrayBase[T] { + return &columnArrayBase[T]{ + genericColumnBase: &genericColumnBase[[]T]{ + name: fieldName, + fieldType: entity.FieldTypeArray, + values: data, + }, + elementType: elementType, + } +} + +/* bool array */ + +type ColumnBoolArray struct { + *columnArrayBase[bool] +} + +func NewColumnBoolArray(fieldName string, data [][]bool) *ColumnBoolArray { + return &ColumnBoolArray{ + columnArrayBase: newArrayBase[bool](fieldName, data, entity.FieldTypeBool), + } +} + +func (c *ColumnBoolArray) Slice(start, end int) Column { + return &ColumnBoolArray{ + columnArrayBase: c.columnArrayBase.slice(start, end), + } +} + +/* int8 array */ + +type ColumnInt8Array struct { + *columnArrayBase[int8] +} + +func NewColumnInt8Array(fieldName string, data [][]int8) *ColumnInt8Array { + return &ColumnInt8Array{ + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt8), + } +} + +func (c *ColumnInt8Array) Slice(start, end int) Column { + return &ColumnInt8Array{ + columnArrayBase: c.columnArrayBase.slice(start, end), + } +} + +/* int16 array */ + +type ColumnInt16Array struct { + *columnArrayBase[int16] +} + +func NewColumnInt16Array(fieldName string, data [][]int16) *ColumnInt16Array { + return &ColumnInt16Array{ + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt16), + } +} + +func (c *ColumnInt16Array) Slice(start, end int) Column { + return &ColumnInt16Array{ + columnArrayBase: c.columnArrayBase.slice(start, end), + } +} + +/* int32 array */ + +type ColumnInt32Array struct { + *columnArrayBase[int32] +} + +func NewColumnInt32Array(fieldName string, data [][]int32) *ColumnInt32Array { + return &ColumnInt32Array{ + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt32), + } +} + +func (c *ColumnInt32Array) Slice(start, end int) Column { + return &ColumnInt32Array{ + columnArrayBase: c.columnArrayBase.slice(start, end), + } +} + +/* int64 array */ + +type ColumnInt64Array struct { + *columnArrayBase[int64] +} + +func NewColumnInt64Array(fieldName string, data [][]int64) *ColumnInt64Array { + return &ColumnInt64Array{ + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeInt64), + } +} + +func (c *ColumnInt64Array) Slice(start, end int) Column { + return &ColumnInt64Array{ + columnArrayBase: c.columnArrayBase.slice(start, end), + } +} + +/* float32 array */ + +type ColumnFloatArray struct { + *columnArrayBase[float32] +} + +func NewColumnFloatArray(fieldName string, data [][]float32) *ColumnFloatArray { + return &ColumnFloatArray{ + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeFloat), } - return c.values[idx], nil } -// AppendValue append value into column -func (c *ColumnVarCharArray) AppendValue(i interface{}) error { - v, ok := i.([][]byte) - if !ok { - return fmt.Errorf("invalid type, expected []string, got %T", i) +func (c *ColumnFloatArray) Slice(start, end int) Column { + return &ColumnFloatArray{ + columnArrayBase: c.columnArrayBase.slice(start, end), } - c.values = append(c.values, v) +} + +/* float64 array */ - return nil +type ColumnDoubleArray struct { + *columnArrayBase[float64] +} + +func NewColumnDoubleArray(fieldName string, data [][]float64) *ColumnDoubleArray { + return &ColumnDoubleArray{ + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeDouble), + } } -// Data returns column data -func (c *ColumnVarCharArray) Data() [][][]byte { - return c.values +func (c *ColumnDoubleArray) Slice(start, end int) Column { + return &ColumnDoubleArray{ + columnArrayBase: c.columnArrayBase.slice(start, end), + } +} + +/* varchar array */ + +type ColumnVarCharArray struct { + *columnArrayBase[string] } -// NewColumnVarChar auto generated constructor -func NewColumnVarCharArray(name string, values [][][]byte) *ColumnVarCharArray { +func NewColumnVarCharArray(fieldName string, data [][]string) *ColumnVarCharArray { return &ColumnVarCharArray{ - name: name, - values: values, + columnArrayBase: newArrayBase(fieldName, data, entity.FieldTypeVarChar), } } diff --git a/client/column/array_gen.go b/client/column/array_gen.go deleted file mode 100644 index 393f8d636ba98..0000000000000 --- a/client/column/array_gen.go +++ /dev/null @@ -1,813 +0,0 @@ -// Code generated by go generate; DO NOT EDIT -// This file is generated by go generate - -package column - -import ( - "fmt" - - "github.com/cockroachdb/errors" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" -) - -// ColumnBoolArray generated columns type for Bool -type ColumnBoolArray struct { - ColumnBase - name string - values [][]bool -} - -// Name returns column name -func (c *ColumnBoolArray) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnBoolArray) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnBoolArray) Len() int { - return len(c.values) -} - -func (c *ColumnBoolArray) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - if end == -1 || end > l { - end = l - } - return &ColumnBoolArray{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnBoolArray) Get(idx int) (interface{}, error) { - var r []bool // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnBoolArray) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]bool, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, bool(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_BoolData{ - BoolData: &schemapb.BoolArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Bool, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnBoolArray) ValueByIdx(idx int) ([]bool, error) { - var r []bool // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnBoolArray) AppendValue(i interface{}) error { - v, ok := i.([]bool) - if !ok { - return fmt.Errorf("invalid type, expected []bool, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnBoolArray) Data() [][]bool { - return c.values -} - -// NewColumnBool auto generated constructor -func NewColumnBoolArray(name string, values [][]bool) *ColumnBoolArray { - return &ColumnBoolArray{ - name: name, - values: values, - } -} - -// ColumnInt8Array generated columns type for Int8 -type ColumnInt8Array struct { - ColumnBase - name string - values [][]int8 -} - -// Name returns column name -func (c *ColumnInt8Array) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt8Array) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnInt8Array) Len() int { - return len(c.values) -} - -func (c *ColumnInt8Array) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt8Array{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt8Array) Get(idx int) (interface{}, error) { - var r []int8 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt8Array) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]int32, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, int32(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Int8, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt8Array) ValueByIdx(idx int) ([]int8, error) { - var r []int8 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt8Array) AppendValue(i interface{}) error { - v, ok := i.([]int8) - if !ok { - return fmt.Errorf("invalid type, expected []int8, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt8Array) Data() [][]int8 { - return c.values -} - -// NewColumnInt8 auto generated constructor -func NewColumnInt8Array(name string, values [][]int8) *ColumnInt8Array { - return &ColumnInt8Array{ - name: name, - values: values, - } -} - -// ColumnInt16Array generated columns type for Int16 -type ColumnInt16Array struct { - ColumnBase - name string - values [][]int16 -} - -// Name returns column name -func (c *ColumnInt16Array) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt16Array) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnInt16Array) Len() int { - return len(c.values) -} - -func (c *ColumnInt16Array) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt16Array{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt16Array) Get(idx int) (interface{}, error) { - var r []int16 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt16Array) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]int32, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, int32(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Int16, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt16Array) ValueByIdx(idx int) ([]int16, error) { - var r []int16 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt16Array) AppendValue(i interface{}) error { - v, ok := i.([]int16) - if !ok { - return fmt.Errorf("invalid type, expected []int16, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt16Array) Data() [][]int16 { - return c.values -} - -// NewColumnInt16 auto generated constructor -func NewColumnInt16Array(name string, values [][]int16) *ColumnInt16Array { - return &ColumnInt16Array{ - name: name, - values: values, - } -} - -// ColumnInt32Array generated columns type for Int32 -type ColumnInt32Array struct { - ColumnBase - name string - values [][]int32 -} - -// Name returns column name -func (c *ColumnInt32Array) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt32Array) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnInt32Array) Len() int { - return len(c.values) -} - -func (c *ColumnInt32Array) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt32Array{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt32Array) Get(idx int) (interface{}, error) { - var r []int32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt32Array) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]int32, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, int32(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Int32, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt32Array) ValueByIdx(idx int) ([]int32, error) { - var r []int32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt32Array) AppendValue(i interface{}) error { - v, ok := i.([]int32) - if !ok { - return fmt.Errorf("invalid type, expected []int32, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt32Array) Data() [][]int32 { - return c.values -} - -// NewColumnInt32 auto generated constructor -func NewColumnInt32Array(name string, values [][]int32) *ColumnInt32Array { - return &ColumnInt32Array{ - name: name, - values: values, - } -} - -// ColumnInt64Array generated columns type for Int64 -type ColumnInt64Array struct { - ColumnBase - name string - values [][]int64 -} - -// Name returns column name -func (c *ColumnInt64Array) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt64Array) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnInt64Array) Len() int { - return len(c.values) -} - -func (c *ColumnInt64Array) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt64Array{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt64Array) Get(idx int) (interface{}, error) { - var r []int64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt64Array) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]int64, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, int64(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_LongData{ - LongData: &schemapb.LongArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Int64, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt64Array) ValueByIdx(idx int) ([]int64, error) { - var r []int64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt64Array) AppendValue(i interface{}) error { - v, ok := i.([]int64) - if !ok { - return fmt.Errorf("invalid type, expected []int64, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt64Array) Data() [][]int64 { - return c.values -} - -// NewColumnInt64 auto generated constructor -func NewColumnInt64Array(name string, values [][]int64) *ColumnInt64Array { - return &ColumnInt64Array{ - name: name, - values: values, - } -} - -// ColumnFloatArray generated columns type for Float -type ColumnFloatArray struct { - ColumnBase - name string - values [][]float32 -} - -// Name returns column name -func (c *ColumnFloatArray) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnFloatArray) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnFloatArray) Len() int { - return len(c.values) -} - -func (c *ColumnFloatArray) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnFloatArray{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnFloatArray) Get(idx int) (interface{}, error) { - var r []float32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnFloatArray) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]float32, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, float32(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_FloatData{ - FloatData: &schemapb.FloatArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Float, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnFloatArray) ValueByIdx(idx int) ([]float32, error) { - var r []float32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnFloatArray) AppendValue(i interface{}) error { - v, ok := i.([]float32) - if !ok { - return fmt.Errorf("invalid type, expected []float32, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnFloatArray) Data() [][]float32 { - return c.values -} - -// NewColumnFloat auto generated constructor -func NewColumnFloatArray(name string, values [][]float32) *ColumnFloatArray { - return &ColumnFloatArray{ - name: name, - values: values, - } -} - -// ColumnDoubleArray generated columns type for Double -type ColumnDoubleArray struct { - ColumnBase - name string - values [][]float64 -} - -// Name returns column name -func (c *ColumnDoubleArray) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnDoubleArray) Type() entity.FieldType { - return entity.FieldTypeArray -} - -// Len returns column values length -func (c *ColumnDoubleArray) Len() int { - return len(c.values) -} - -func (c *ColumnDoubleArray) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnDoubleArray{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnDoubleArray) Get(idx int) (interface{}, error) { - var r []float64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnDoubleArray) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Array, - FieldName: c.name, - } - - data := make([]*schemapb.ScalarField, 0, c.Len()) - for _, arr := range c.values { - converted := make([]float64, 0, c.Len()) - for i := 0; i < len(arr); i++ { - converted = append(converted, float64(arr[i])) - } - data = append(data, &schemapb.ScalarField{ - Data: &schemapb.ScalarField_DoubleData{ - DoubleData: &schemapb.DoubleArray{ - Data: converted, - }, - }, - }) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_ArrayData{ - ArrayData: &schemapb.ArrayArray{ - Data: data, - ElementType: schemapb.DataType_Double, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnDoubleArray) ValueByIdx(idx int) ([]float64, error) { - var r []float64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnDoubleArray) AppendValue(i interface{}) error { - v, ok := i.([]float64) - if !ok { - return fmt.Errorf("invalid type, expected []float64, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnDoubleArray) Data() [][]float64 { - return c.values -} - -// NewColumnDouble auto generated constructor -func NewColumnDoubleArray(name string, values [][]float64) *ColumnDoubleArray { - return &ColumnDoubleArray{ - name: name, - values: values, - } -} diff --git a/client/column/array_test.go b/client/column/array_test.go new file mode 100644 index 0000000000000..07f7883203c4a --- /dev/null +++ b/client/column/array_test.go @@ -0,0 +1,299 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/client/v2/entity" +) + +type ArraySuite struct { + suite.Suite +} + +func (s *ArraySuite) TestBasic() { + s.Run("bool_array", func() { + data := [][]bool{ + {true, false}, + {false, true}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnBoolArray(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeBool, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Bool, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + s.Equal(row, sf.GetBoolData().GetData()) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnBoolArray) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeBool, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("int8_array", func() { + data := [][]int8{ + {1, 2}, + {3, 4}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnInt8Array(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt8, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Int8, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + for j, item := range row { + s.EqualValues(item, sf.GetIntData().GetData()[j]) + } + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt8Array) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt8, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("int16_array", func() { + data := [][]int16{ + {1, 2}, + {3, 4}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnInt16Array(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt16, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Int16, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + for j, item := range row { + s.EqualValues(item, sf.GetIntData().GetData()[j]) + } + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt16Array) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt16, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("int32_array", func() { + data := [][]int32{ + {1, 2}, + {3, 4}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnInt32Array(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt32, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Int32, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + s.Equal(row, sf.GetIntData().GetData()) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt32Array) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt32, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("int64_array", func() { + data := [][]int64{ + {1, 2}, + {3, 4}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnInt64Array(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt64, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Int64, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + s.Equal(row, sf.GetLongData().GetData()) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt64Array) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeInt64, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("float_array", func() { + data := [][]float32{ + {0.1, 0.2}, + {1.3, 1.4}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnFloatArray(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeFloat, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Float, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + s.Equal(row, sf.GetFloatData().GetData()) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnFloatArray) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeFloat, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("double_array", func() { + data := [][]float64{ + {0.1, 0.2}, + {1.3, 1.4}, + } + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnDoubleArray(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeDouble, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_Double, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + s.Equal(row, sf.GetDoubleData().GetData()) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnDoubleArray) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeDouble, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) + + s.Run("varchar_array", func() { + data := [][]string{ + {"abc", "def"}, + {"xyz"}, + } + + name := fmt.Sprintf("field_%d", rand.Intn(100)) + + column := NewColumnVarCharArray(name, data) + s.Equal(name, column.Name()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeVarChar, column.ElementType()) + + fd := column.FieldData() + arrayData := fd.GetScalars().GetArrayData() + s.Equal(schemapb.DataType_VarChar, arrayData.GetElementType()) + for i, row := range data { + sf := arrayData.GetData()[i] + s.Equal(row, sf.GetStringData().GetData()) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnVarCharArray) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeArray, column.Type()) + s.Equal(entity.FieldTypeVarChar, column.ElementType()) + s.Equal(data, parsed.Data()) + } + }) +} + +func TestArrays(t *testing.T) { + suite.Run(t, new(ArraySuite)) +} diff --git a/client/column/columns.go b/client/column/columns.go index 905cc8c2d9ca1..6ee7f89603c45 100644 --- a/client/column/columns.go +++ b/client/column/columns.go @@ -42,25 +42,6 @@ type Column interface { GetAsBool(int) (bool, error) } -// ColumnBase adds conversion methods support for fixed-type columns. -type ColumnBase struct{} - -func (b ColumnBase) GetAsInt64(_ int) (int64, error) { - return 0, errors.New("conversion between fixed-type column not support") -} - -func (b ColumnBase) GetAsString(_ int) (string, error) { - return "", errors.New("conversion between fixed-type column not support") -} - -func (b ColumnBase) GetAsDouble(_ int) (float64, error) { - return 0, errors.New("conversion between fixed-type column not support") -} - -func (b ColumnBase) GetAsBool(_ int) (bool, error) { - return false, errors.New("conversion between fixed-type column not support") -} - var errFieldDataTypeNotMatch = errors.New("FieldData type not matched") // IDColumns converts schemapb.IDs to corresponding column @@ -393,12 +374,12 @@ func parseArrayData(fieldName string, elementType schemapb.DataType, fieldDataLi return NewColumnDoubleArray(fieldName, data), nil case schemapb.DataType_VarChar, schemapb.DataType_String: - var data [][][]byte + var data [][]string for _, fd := range fieldDataList { strs := fd.GetStringData().GetData() - bytesData := make([][]byte, 0, len(strs)) + bytesData := make([]string, 0, len(strs)) for _, str := range strs { - bytesData = append(bytesData, []byte(str)) + bytesData = append(bytesData, str) } data = append(data, bytesData) } @@ -428,104 +409,3 @@ func getIntData(fd *schemapb.FieldData) (*schemapb.ScalarField_IntData, bool) { return nil, false } } - -// FieldDataColumn converts schemapb.FieldData to vector Column -func FieldDataVector(fd *schemapb.FieldData) (Column, error) { - switch fd.GetType() { - case schemapb.DataType_FloatVector: - vectors := fd.GetVectors() - x, ok := vectors.GetData().(*schemapb.VectorField_FloatVector) - if !ok { - return nil, errFieldDataTypeNotMatch - } - data := x.FloatVector.GetData() - dim := int(vectors.GetDim()) - vector := make([][]float32, 0, len(data)/dim) // shall not have remanunt - for i := 0; i < len(data)/dim; i++ { - v := make([]float32, dim) - copy(v, data[i*dim:(i+1)*dim]) - vector = append(vector, v) - } - return NewColumnFloatVector(fd.GetFieldName(), dim, vector), nil - case schemapb.DataType_BinaryVector: - vectors := fd.GetVectors() - x, ok := vectors.GetData().(*schemapb.VectorField_BinaryVector) - if !ok { - return nil, errFieldDataTypeNotMatch - } - data := x.BinaryVector - if data == nil { - return nil, errFieldDataTypeNotMatch - } - dim := int(vectors.GetDim()) - blen := dim / 8 - vector := make([][]byte, 0, len(data)/blen) - for i := 0; i < len(data)/blen; i++ { - v := make([]byte, blen) - copy(v, data[i*blen:(i+1)*blen]) - vector = append(vector, v) - } - return NewColumnBinaryVector(fd.GetFieldName(), dim, vector), nil - case schemapb.DataType_Float16Vector: - vectors := fd.GetVectors() - x, ok := vectors.GetData().(*schemapb.VectorField_Float16Vector) - if !ok { - return nil, errFieldDataTypeNotMatch - } - data := x.Float16Vector - dim := int(vectors.GetDim()) - vector := make([][]byte, 0, len(data)/dim) // shall not have remanunt - for i := 0; i < len(data)/dim; i++ { - v := make([]byte, dim) - copy(v, data[i*dim:(i+1)*dim]) - vector = append(vector, v) - } - return NewColumnFloat16Vector(fd.GetFieldName(), dim, vector), nil - case schemapb.DataType_BFloat16Vector: - vectors := fd.GetVectors() - x, ok := vectors.GetData().(*schemapb.VectorField_Bfloat16Vector) - if !ok { - return nil, errFieldDataTypeNotMatch - } - data := x.Bfloat16Vector - dim := int(vectors.GetDim()) - vector := make([][]byte, 0, len(data)/dim) // shall not have remanunt - for i := 0; i < len(data)/dim; i++ { - v := make([]byte, dim) - copy(v, data[i*dim:(i+1)*dim]) - vector = append(vector, v) - } - return NewColumnBFloat16Vector(fd.GetFieldName(), dim, vector), nil - default: - return nil, errors.New("unsupported data type") - } -} - -// defaultValueColumn will return the empty scalars column which will be fill with default value -func DefaultValueColumn(name string, dataType entity.FieldType) (Column, error) { - switch dataType { - case entity.FieldTypeBool: - return NewColumnBool(name, nil), nil - case entity.FieldTypeInt8: - return NewColumnInt8(name, nil), nil - case entity.FieldTypeInt16: - return NewColumnInt16(name, nil), nil - case entity.FieldTypeInt32: - return NewColumnInt32(name, nil), nil - case entity.FieldTypeInt64: - return NewColumnInt64(name, nil), nil - case entity.FieldTypeFloat: - return NewColumnFloat(name, nil), nil - case entity.FieldTypeDouble: - return NewColumnDouble(name, nil), nil - case entity.FieldTypeString: - return NewColumnString(name, nil), nil - case entity.FieldTypeVarChar: - return NewColumnVarChar(name, nil), nil - case entity.FieldTypeJSON: - return NewColumnJSONBytes(name, nil), nil - - default: - return nil, fmt.Errorf("default value unsupported data type %s", dataType) - } -} diff --git a/client/column/conversion.go b/client/column/conversion.go index 43c61a016de5f..0eedccadbecdd 100644 --- a/client/column/conversion.go +++ b/client/column/conversion.go @@ -16,38 +16,269 @@ package column -func (c *ColumnInt8) GetAsInt64(idx int) (int64, error) { - v, err := c.ValueByIdx(idx) - return int64(v), err -} +import ( + "fmt" -func (c *ColumnInt16) GetAsInt64(idx int) (int64, error) { - v, err := c.ValueByIdx(idx) - return int64(v), err -} + "github.com/cockroachdb/errors" + "github.com/samber/lo" -func (c *ColumnInt32) GetAsInt64(idx int) (int64, error) { - v, err := c.ValueByIdx(idx) - return int64(v), err -} + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/client/v2/entity" +) -func (c *ColumnInt64) GetAsInt64(idx int) (int64, error) { - return c.ValueByIdx(idx) +func slice2Scalar[T any](values []T, elementType entity.FieldType) *schemapb.ScalarField { + var ok bool + scalarField := &schemapb.ScalarField{} + switch elementType { + case entity.FieldTypeBool: + var bools []bool + bools, ok = any(values).([]bool) + scalarField.Data = &schemapb.ScalarField_BoolData{ + BoolData: &schemapb.BoolArray{ + Data: bools, + }, + } + case entity.FieldTypeInt8: + var int8s []int8 + int8s, ok = any(values).([]int8) + int32s := lo.Map(int8s, func(i8 int8, _ int) int32 { return int32(i8) }) + scalarField.Data = &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: int32s, + }, + } + case entity.FieldTypeInt16: + var int16s []int16 + int16s, ok = any(values).([]int16) + int32s := lo.Map(int16s, func(i16 int16, _ int) int32 { return int32(i16) }) + scalarField.Data = &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: int32s, + }, + } + case entity.FieldTypeInt32: + var int32s []int32 + int32s, ok = any(values).([]int32) + scalarField.Data = &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: int32s, + }, + } + case entity.FieldTypeInt64: + var int64s []int64 + int64s, ok = any(values).([]int64) + scalarField.Data = &schemapb.ScalarField_LongData{ + LongData: &schemapb.LongArray{ + Data: int64s, + }, + } + case entity.FieldTypeFloat: + var floats []float32 + floats, ok = any(values).([]float32) + scalarField.Data = &schemapb.ScalarField_FloatData{ + FloatData: &schemapb.FloatArray{ + Data: floats, + }, + } + case entity.FieldTypeDouble: + var doubles []float64 + doubles, ok = any(values).([]float64) + scalarField.Data = &schemapb.ScalarField_DoubleData{ + DoubleData: &schemapb.DoubleArray{ + Data: doubles, + }, + } + case entity.FieldTypeVarChar, entity.FieldTypeString: + var strings []string + strings, ok = any(values).([]string) + scalarField.Data = &schemapb.ScalarField_StringData{ + StringData: &schemapb.StringArray{ + Data: strings, + }, + } + } + + if !ok { + panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, elementType)) + } + return scalarField } -func (c *ColumnString) GetAsString(idx int) (string, error) { - return c.ValueByIdx(idx) +func values2FieldData[T any](values []T, fieldType entity.FieldType, dim int) *schemapb.FieldData { + fd := &schemapb.FieldData{} + switch fieldType { + // scalars + case entity.FieldTypeBool, + entity.FieldTypeFloat, + entity.FieldTypeDouble, + entity.FieldTypeInt8, + entity.FieldTypeInt16, + entity.FieldTypeInt32, + entity.FieldTypeInt64, + entity.FieldTypeVarChar, + entity.FieldTypeString, + entity.FieldTypeJSON: + fd.Field = &schemapb.FieldData_Scalars{ + Scalars: values2Scalars(values, fieldType), // scalars, + } + // vectors + case entity.FieldTypeFloatVector, + entity.FieldTypeFloat16Vector, + entity.FieldTypeBFloat16Vector, + entity.FieldTypeBinaryVector, + entity.FieldTypeSparseVector: + fd.Field = &schemapb.FieldData_Vectors{ + Vectors: values2Vectors(values, fieldType, int64(dim)), + } + default: + panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, fieldType)) + } + return fd } -func (c *ColumnFloat) GetAsDouble(idx int) (float64, error) { - v, err := c.ValueByIdx(idx) - return float64(v), err +func values2Scalars[T any](values []T, fieldType entity.FieldType) *schemapb.ScalarField { + scalars := &schemapb.ScalarField{} + var ok bool + switch fieldType { + case entity.FieldTypeBool: + var bools []bool + bools, ok = any(values).([]bool) + scalars.Data = &schemapb.ScalarField_BoolData{ + BoolData: &schemapb.BoolArray{Data: bools}, + } + case entity.FieldTypeInt8: + var int8s []int8 + int8s, ok = any(values).([]int8) + int32s := lo.Map(int8s, func(i8 int8, _ int) int32 { return int32(i8) }) + scalars.Data = &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{Data: int32s}, + } + case entity.FieldTypeInt16: + var int16s []int16 + int16s, ok = any(values).([]int16) + int32s := lo.Map(int16s, func(i16 int16, _ int) int32 { return int32(i16) }) + scalars.Data = &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{Data: int32s}, + } + case entity.FieldTypeInt32: + var int32s []int32 + int32s, ok = any(values).([]int32) + scalars.Data = &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{Data: int32s}, + } + case entity.FieldTypeInt64: + var int64s []int64 + int64s, ok = any(values).([]int64) + scalars.Data = &schemapb.ScalarField_LongData{ + LongData: &schemapb.LongArray{Data: int64s}, + } + case entity.FieldTypeVarChar, entity.FieldTypeString: + var strVals []string + strVals, ok = any(values).([]string) + scalars.Data = &schemapb.ScalarField_StringData{ + StringData: &schemapb.StringArray{Data: strVals}, + } + case entity.FieldTypeFloat: + var floats []float32 + floats, ok = any(values).([]float32) + scalars.Data = &schemapb.ScalarField_FloatData{ + FloatData: &schemapb.FloatArray{Data: floats}, + } + case entity.FieldTypeDouble: + var data []float64 + data, ok = any(values).([]float64) + scalars.Data = &schemapb.ScalarField_DoubleData{ + DoubleData: &schemapb.DoubleArray{Data: data}, + } + case entity.FieldTypeJSON: + var data [][]byte + data, ok = any(values).([][]byte) + scalars.Data = &schemapb.ScalarField_JsonData{ + JsonData: &schemapb.JSONArray{ + Data: data, + }, + } + } + // shall not be accessed + if !ok { + panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, fieldType)) + } + return scalars } -func (c *ColumnDouble) GetAsDouble(idx int) (float64, error) { - return c.ValueByIdx(idx) +func values2Vectors[T any](values []T, fieldType entity.FieldType, dim int64) *schemapb.VectorField { + vectorField := &schemapb.VectorField{ + Dim: dim, + } + var ok bool + switch fieldType { + case entity.FieldTypeFloatVector: + var vectors []entity.FloatVector + vectors, ok = any(values).([]entity.FloatVector) + data := make([]float32, 0, int64(len(vectors))*dim) + for _, vector := range vectors { + data = append(data, vector...) + } + vectorField.Data = &schemapb.VectorField_FloatVector{ + FloatVector: &schemapb.FloatArray{ + Data: data, + }, + } + case entity.FieldTypeFloat16Vector: + var vectors []entity.Float16Vector + vectors, ok = any(values).([]entity.Float16Vector) + data := make([]byte, 0, int64(len(vectors))*dim*2) + for _, vector := range vectors { + data = append(data, vector.Serialize()...) + } + vectorField.Data = &schemapb.VectorField_Float16Vector{ + Float16Vector: data, + } + case entity.FieldTypeBFloat16Vector: + var vectors []entity.BFloat16Vector + vectors, ok = any(values).([]entity.BFloat16Vector) + data := make([]byte, 0, int64(len(vectors))*dim*2) + for _, vector := range vectors { + data = append(data, vector.Serialize()...) + } + vectorField.Data = &schemapb.VectorField_Bfloat16Vector{ + Bfloat16Vector: data, + } + case entity.FieldTypeBinaryVector: + var vectors []entity.BinaryVector + vectors, ok = any(values).([]entity.BinaryVector) + data := make([]byte, 0, int64(len(vectors))*dim/8) + for _, vector := range vectors { + data = append(data, vector.Serialize()...) + } + vectorField.Data = &schemapb.VectorField_BinaryVector{ + BinaryVector: data, + } + case entity.FieldTypeSparseVector: + var vectors []entity.SparseEmbedding + vectors, ok = any(values).([]entity.SparseEmbedding) + data := lo.Map(vectors, func(row entity.SparseEmbedding, _ int) []byte { + return row.Serialize() + }) + vectorField.Data = &schemapb.VectorField_SparseFloatVector{ + SparseFloatVector: &schemapb.SparseFloatArray{ + Contents: data, + }, + } + } + + if !ok { + panic(fmt.Sprintf("unexpected values type(%T) of fieldType %v", values, fieldType)) + } + return vectorField } -func (c *ColumnBool) GetAsBool(idx int) (bool, error) { - return c.ValueByIdx(idx) +func value2Type[T any, U any](v T) (U, error) { + var z U + switch v := any(v).(type) { + case U: + return v, nil + default: + return z, errors.Newf("cannot automatically convert %T to %T", v, z) + } } diff --git a/client/column/dynamic.go b/client/column/dynamic.go index 663bf175e3169..e23b965af989b 100644 --- a/client/column/dynamic.go +++ b/client/column/dynamic.go @@ -41,7 +41,7 @@ func (c *ColumnDynamic) Name() string { // Get returns element at idx as interface{}. // Overrides internal json column behavior, returns raw json data. func (c *ColumnDynamic) Get(idx int) (interface{}, error) { - bs, err := c.ColumnJSONBytes.ValueByIdx(idx) + bs, err := c.ColumnJSONBytes.Value(idx) if err != nil { return 0, err } @@ -53,7 +53,7 @@ func (c *ColumnDynamic) Get(idx int) (interface{}, error) { } func (c *ColumnDynamic) GetAsInt64(idx int) (int64, error) { - bs, err := c.ColumnJSONBytes.ValueByIdx(idx) + bs, err := c.ColumnJSONBytes.Value(idx) if err != nil { return 0, err } @@ -68,7 +68,7 @@ func (c *ColumnDynamic) GetAsInt64(idx int) (int64, error) { } func (c *ColumnDynamic) GetAsString(idx int) (string, error) { - bs, err := c.ColumnJSONBytes.ValueByIdx(idx) + bs, err := c.ColumnJSONBytes.Value(idx) if err != nil { return "", err } @@ -83,7 +83,7 @@ func (c *ColumnDynamic) GetAsString(idx int) (string, error) { } func (c *ColumnDynamic) GetAsBool(idx int) (bool, error) { - bs, err := c.ColumnJSONBytes.ValueByIdx(idx) + bs, err := c.ColumnJSONBytes.Value(idx) if err != nil { return false, err } @@ -98,7 +98,7 @@ func (c *ColumnDynamic) GetAsBool(idx int) (bool, error) { } func (c *ColumnDynamic) GetAsDouble(idx int) (float64, error) { - bs, err := c.ColumnJSONBytes.ValueByIdx(idx) + bs, err := c.ColumnJSONBytes.Value(idx) if err != nil { return 0, err } diff --git a/client/column/dynamic_test.go b/client/column/dynamic_test.go index b65e5868997b0..865ed6c4344d9 100644 --- a/client/column/dynamic_test.go +++ b/client/column/dynamic_test.go @@ -40,9 +40,7 @@ func (s *ColumnDynamicSuite) TestGetInt() { for _, c := range cases { s.Run(c.input, func() { - column := NewColumnDynamic(&ColumnJSONBytes{ - values: [][]byte{[]byte(c.input)}, - }, "field") + column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field") v, err := column.GetAsInt64(0) if c.expectErr { s.Error(err) @@ -68,9 +66,10 @@ func (s *ColumnDynamicSuite) TestGetString() { for _, c := range cases { s.Run(c.input, func() { - column := NewColumnDynamic(&ColumnJSONBytes{ - values: [][]byte{[]byte(c.input)}, - }, "field") + // column := NewColumnDynamic(&ColumnJSONBytes{ + // values: [][]byte{[]byte(c.input)}, + // }, "field") + column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field") v, err := column.GetAsString(0) if c.expectErr { s.Error(err) @@ -96,9 +95,10 @@ func (s *ColumnDynamicSuite) TestGetBool() { for _, c := range cases { s.Run(c.input, func() { - column := NewColumnDynamic(&ColumnJSONBytes{ - values: [][]byte{[]byte(c.input)}, - }, "field") + // column := NewColumnDynamic(&ColumnJSONBytes{ + // values: [][]byte{[]byte(c.input)}, + // }, "field") + column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field") v, err := column.GetAsBool(0) if c.expectErr { s.Error(err) @@ -124,9 +124,10 @@ func (s *ColumnDynamicSuite) TestGetDouble() { for _, c := range cases { s.Run(c.input, func() { - column := NewColumnDynamic(&ColumnJSONBytes{ - values: [][]byte{[]byte(c.input)}, - }, "field") + // column := NewColumnDynamic(&ColumnJSONBytes{ + // values: [][]byte{[]byte(c.input)}, + // }, "field") + column := NewColumnDynamic(NewColumnJSONBytes("", [][]byte{[]byte(c.input)}), "field") v, err := column.GetAsDouble(0) if c.expectErr { s.Error(err) @@ -140,7 +141,7 @@ func (s *ColumnDynamicSuite) TestGetDouble() { func (s *ColumnDynamicSuite) TestIndexOutOfRange() { var err error - column := NewColumnDynamic(&ColumnJSONBytes{}, "field") + column := NewColumnDynamic(NewColumnJSONBytes("", nil), "field") s.Equal("field", column.Name()) diff --git a/client/column/generic_base.go b/client/column/generic_base.go new file mode 100644 index 0000000000000..d6f014f2dd408 --- /dev/null +++ b/client/column/generic_base.go @@ -0,0 +1,146 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import ( + "github.com/cockroachdb/errors" + + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/client/v2/entity" +) + +var _ Column = (*genericColumnBase[any])(nil) + +// genericColumnBase implements `Column` interface +// it provides the basic function for each scalar params +type genericColumnBase[T any] struct { + name string + fieldType entity.FieldType + values []T +} + +// Name returns column name. +func (c *genericColumnBase[T]) Name() string { + return c.name +} + +// Type returns corresponding field type. +// note that: it is not necessary to be 1-on-1 mapping +// say, `[]byte` could be lots of field type. +func (c *genericColumnBase[T]) Type() entity.FieldType { + return c.fieldType +} + +func (c *genericColumnBase[T]) Len() int { + return len(c.values) +} + +func (c *genericColumnBase[T]) AppendValue(a any) error { + v, ok := a.(T) + if !ok { + return errors.Newf("unexpected append value type %T, field type %v", a, c.fieldType) + } + c.values = append(c.values, v) + return nil +} + +func (c *genericColumnBase[T]) Slice(start, end int) Column { + return c.slice(start, end) +} + +func (c *genericColumnBase[T]) slice(start, end int) *genericColumnBase[T] { + l := c.Len() + if start > l { + start = l + } + if end == -1 || end > l { + end = l + } + return &genericColumnBase[T]{ + name: c.name, + fieldType: c.fieldType, + values: c.values[start:end], + } +} + +func (c *genericColumnBase[T]) FieldData() *schemapb.FieldData { + fd := values2FieldData(c.values, c.fieldType, 0) + fd.FieldName = c.name + fd.Type = schemapb.DataType(c.fieldType) + return fd +} + +func (c *genericColumnBase[T]) rangeCheck(idx int) error { + if idx < 0 || idx >= c.Len() { + return errors.Newf("index %d out of range[0, %d)", idx, c.Len()) + } + return nil +} + +func (c *genericColumnBase[T]) Get(idx int) (any, error) { + if err := c.rangeCheck(idx); err != nil { + return nil, err + } + return c.values[idx], nil +} + +func (c *genericColumnBase[T]) GetAsInt64(idx int) (int64, error) { + if err := c.rangeCheck(idx); err != nil { + return 0, err + } + return value2Type[T, int64](c.values[idx]) +} + +func (c *genericColumnBase[T]) GetAsString(idx int) (string, error) { + if err := c.rangeCheck(idx); err != nil { + return "", err + } + return value2Type[T, string](c.values[idx]) +} + +func (c *genericColumnBase[T]) GetAsDouble(idx int) (float64, error) { + if err := c.rangeCheck(idx); err != nil { + return 0, err + } + return value2Type[T, float64](c.values[idx]) +} + +func (c *genericColumnBase[T]) GetAsBool(idx int) (bool, error) { + if err := c.rangeCheck(idx); err != nil { + return false, err + } + return value2Type[T, bool](c.values[idx]) +} + +func (c *genericColumnBase[T]) Value(idx int) (T, error) { + var z T + if err := c.rangeCheck(idx); err != nil { + return z, err + } + return c.values[idx], nil +} + +func (c *genericColumnBase[T]) Data() []T { + return c.values +} + +func (c *genericColumnBase[T]) MustValue(idx int) T { + if idx < 0 || idx > c.Len() { + panic("index out of range") + } + return c.values[idx] +} diff --git a/client/column/generic_base_test.go b/client/column/generic_base_test.go new file mode 100644 index 0000000000000..991560e0e21ae --- /dev/null +++ b/client/column/generic_base_test.go @@ -0,0 +1,153 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/client/v2/entity" +) + +type GenericBaseSuite struct { + suite.Suite +} + +func (s *GenericBaseSuite) TestBasic() { + name := fmt.Sprintf("test_%d", rand.Intn(10)) + gb := &genericColumnBase[int64]{ + name: name, + fieldType: entity.FieldTypeInt64, + values: []int64{1, 2, 3}, + } + + s.Equal(name, gb.Name()) + s.Equal(entity.FieldTypeInt64, gb.Type()) + s.EqualValues(3, gb.Len()) + + err := gb.AppendValue("abc") + s.Error(err) + s.EqualValues(3, gb.Len()) + + err = gb.AppendValue(int64(4)) + s.NoError(err) + s.EqualValues(4, gb.Len()) +} + +func (s *GenericBaseSuite) TestIndexAccess() { + name := fmt.Sprintf("test_%d", rand.Intn(10)) + values := []int64{1, 2, 3} + gb := &genericColumnBase[int64]{ + name: name, + fieldType: entity.FieldTypeInt64, + values: values, + } + + for idx, value := range values { + v, err := gb.Value(idx) + s.NoError(err) + s.Equal(value, v) + + s.NotPanics(func() { + v = gb.MustValue(idx) + }) + s.Equal(value, v) + } + + s.Panics(func() { + gb.MustValue(-1) + }, "out of range, negative index") + + s.Panics(func() { + gb.MustValue(3) + }, "out of range, LTE len") + + s.NotPanics(func() { + _, err := gb.Value(-1) + s.Error(err) + + _, err = gb.Value(3) + s.Error(err) + }) +} + +func (s *GenericBaseSuite) TestSlice() { + name := fmt.Sprintf("test_%d", rand.Intn(10)) + values := []int64{1, 2, 3} + gb := &genericColumnBase[int64]{ + name: name, + fieldType: entity.FieldTypeInt64, + values: values, + } + + l := rand.Intn(3) + another := gb.Slice(0, l) + s.Equal(l, another.Len()) + agb, ok := another.(*genericColumnBase[int64]) + s.Require().True(ok) + + for i := 0; i < l; i++ { + s.Equal(gb.MustValue(i), agb.MustValue(i)) + } + + s.NotPanics(func() { + agb := gb.Slice(10, 10) + s.Equal(0, agb.Len()) + }) +} + +func (s *GenericBaseSuite) TestFieldData() { + name := fmt.Sprintf("test_%d", rand.Intn(10)) + values := []int64{1, 2, 3} + gb := &genericColumnBase[int64]{ + name: name, + fieldType: entity.FieldTypeInt64, + values: values, + } + + fd := gb.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(schemapb.DataType_Int64, fd.GetType()) +} + +func (s *GenericBaseSuite) TestConversion() { + name := fmt.Sprintf("test_%d", rand.Intn(10)) + values := []int64{1, 2, 3} + gb := &genericColumnBase[int64]{ + name: name, + fieldType: entity.FieldTypeInt64, + values: values, + } + + val, err := gb.GetAsInt64(0) + s.NoError(err) + s.EqualValues(1, val) + + _, err = gb.GetAsBool(0) + s.Error(err) + + _, err = gb.GetAsBool(0) + s.Error(err) +} + +func TestGenericBase(t *testing.T) { + suite.Run(t, new(GenericBaseSuite)) +} diff --git a/client/column/json.go b/client/column/json.go index 0471b05548344..ea7d0fdb48fa4 100644 --- a/client/column/json.go +++ b/client/column/json.go @@ -21,98 +21,44 @@ import ( "fmt" "reflect" - "github.com/cockroachdb/errors" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/client/v2/entity" ) -var _ (Column) = (*ColumnJSONBytes)(nil) +var _ (Column) = (*ColumnVarChar)(nil) -// ColumnJSONBytes column type for JSON. -// all items are marshaled json bytes. type ColumnJSONBytes struct { - ColumnBase - name string - values [][]byte + *genericColumnBase[[]byte] isDynamic bool } -// Name returns column name. -func (c *ColumnJSONBytes) Name() string { - return c.name -} - -// Type returns column entity.FieldType. -func (c *ColumnJSONBytes) Type() entity.FieldType { - return entity.FieldTypeJSON -} - -// Len returns column values length. -func (c *ColumnJSONBytes) Len() int { - return len(c.values) -} - -func (c *ColumnJSONBytes) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } +func NewColumnJSONBytes(name string, values [][]byte) *ColumnJSONBytes { return &ColumnJSONBytes{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], + genericColumnBase: &genericColumnBase[[]byte]{ + name: name, + fieldType: entity.FieldTypeJSON, + values: values, + }, } } -// Get returns value at index as interface{}. -func (c *ColumnJSONBytes) Get(idx int) (interface{}, error) { - if idx < 0 || idx > c.Len() { - return nil, errors.New("index out of range") +func (c *ColumnJSONBytes) Slice(start, end int) Column { + return &ColumnJSONBytes{ + genericColumnBase: c.genericColumnBase.slice(start, end), } - return c.values[idx], nil } -func (c *ColumnJSONBytes) GetAsString(idx int) (string, error) { - bs, err := c.ValueByIdx(idx) - if err != nil { - return "", err - } - return string(bs), nil +func (c *ColumnJSONBytes) WithIsDynamic(isDynamic bool) *ColumnJSONBytes { + c.isDynamic = isDynamic + return c } -// FieldData return column data mapped to schemapb.FieldData. func (c *ColumnJSONBytes) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_JSON, - FieldName: c.name, - IsDynamic: c.isDynamic, - } - - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_JsonData{ - JsonData: &schemapb.JSONArray{ - Data: c.values, - }, - }, - }, - } - + fd := c.genericColumnBase.FieldData() + fd.IsDynamic = c.isDynamic return fd } -// ValueByIdx returns value of the provided index. -func (c *ColumnJSONBytes) ValueByIdx(idx int) ([]byte, error) { - if idx < 0 || idx >= c.Len() { - return nil, errors.New("index out of range") - } - return c.values[idx], nil -} - // AppendValue append value into column. func (c *ColumnJSONBytes) AppendValue(i interface{}) error { var v []byte @@ -141,21 +87,3 @@ func (c *ColumnJSONBytes) AppendValue(i interface{}) error { return nil } - -// Data returns column data. -func (c *ColumnJSONBytes) Data() [][]byte { - return c.values -} - -func (c *ColumnJSONBytes) WithIsDynamic(isDynamic bool) *ColumnJSONBytes { - c.isDynamic = isDynamic - return c -} - -// NewColumnJSONBytes composes a Column with json bytes. -func NewColumnJSONBytes(name string, values [][]byte) *ColumnJSONBytes { - return &ColumnJSONBytes{ - name: name, - values: values, - } -} diff --git a/client/column/json_test.go b/client/column/json_test.go index b627639d0d424..0ed4141c715c9 100644 --- a/client/column/json_test.go +++ b/client/column/json_test.go @@ -62,15 +62,24 @@ func (s *ColumnJSONBytesSuite) TestAttrMethods() { fd := column.FieldData() s.NotNil(fd) s.Equal(fd.GetFieldName(), columnName) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnJSONBytes) + if s.True(ok) { + s.Equal(columnName, parsed.Name()) + s.Equal(entity.FieldTypeJSON, parsed.Type()) + s.Equal(v, parsed.Data()) + } }) s.Run("test_column_valuer_by_idx", func() { - _, err := column.ValueByIdx(-1) + _, err := column.Value(-1) s.Error(err) - _, err = column.ValueByIdx(columnLen) + _, err = column.Value(columnLen) s.Error(err) for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) + v, err := column.Value(i) s.NoError(err) s.Equal(column.values[i], v) } @@ -81,7 +90,7 @@ func (s *ColumnJSONBytesSuite) TestAttrMethods() { err := column.AppendValue(item) s.NoError(err) s.Equal(columnLen+1, column.Len()) - val, err := column.ValueByIdx(columnLen) + val, err := column.Value(columnLen) s.NoError(err) s.Equal(item, val) diff --git a/client/column/scalar.go b/client/column/scalar.go new file mode 100644 index 0000000000000..8508ef7f91166 --- /dev/null +++ b/client/column/scalar.go @@ -0,0 +1,256 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import "github.com/milvus-io/milvus/client/v2/entity" + +/* bool */ + +var _ Column = (*ColumnBool)(nil) + +type ColumnBool struct { + *genericColumnBase[bool] +} + +func NewColumnBool(name string, values []bool) *ColumnBool { + return &ColumnBool{ + genericColumnBase: &genericColumnBase[bool]{ + name: name, + fieldType: entity.FieldTypeBool, + values: values, + }, + } +} + +func (c *ColumnBool) Slice(start, end int) Column { + return &ColumnBool{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +/* Int8 */ + +var _ Column = (*ColumnInt8)(nil) + +type ColumnInt8 struct { + *genericColumnBase[int8] +} + +func NewColumnInt8(name string, values []int8) *ColumnInt8 { + return &ColumnInt8{ + genericColumnBase: &genericColumnBase[int8]{ + name: name, + fieldType: entity.FieldTypeInt8, + values: values, + }, + } +} + +func (c *ColumnInt8) Slice(start, end int) Column { + return &ColumnInt8{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +func (c *ColumnInt8) GetAsInt64(idx int) (int64, error) { + v, err := c.Value(idx) + return int64(v), err +} + +/* Int16 */ + +var _ Column = (*ColumnInt16)(nil) + +type ColumnInt16 struct { + *genericColumnBase[int16] +} + +func NewColumnInt16(name string, values []int16) *ColumnInt16 { + return &ColumnInt16{ + genericColumnBase: &genericColumnBase[int16]{ + name: name, + fieldType: entity.FieldTypeInt16, + values: values, + }, + } +} + +func (c *ColumnInt16) Slice(start, end int) Column { + return &ColumnInt16{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +func (c *ColumnInt16) GetAsInt64(idx int) (int64, error) { + v, err := c.Value(idx) + return int64(v), err +} + +/* Int32 */ + +var _ Column = (*ColumnInt32)(nil) + +type ColumnInt32 struct { + *genericColumnBase[int32] +} + +func NewColumnInt32(name string, values []int32) *ColumnInt32 { + return &ColumnInt32{ + genericColumnBase: &genericColumnBase[int32]{ + name: name, + fieldType: entity.FieldTypeInt32, + values: values, + }, + } +} + +func (c *ColumnInt32) Slice(start, end int) Column { + return &ColumnInt32{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +func (c *ColumnInt32) GetAsInt64(idx int) (int64, error) { + v, err := c.Value(idx) + return int64(v), err +} + +/* Int64 */ + +var _ Column = (*ColumnInt64)(nil) + +type ColumnInt64 struct { + *genericColumnBase[int64] +} + +func NewColumnInt64(name string, values []int64) *ColumnInt64 { + return &ColumnInt64{ + genericColumnBase: &genericColumnBase[int64]{ + name: name, + fieldType: entity.FieldTypeInt64, + values: values, + }, + } +} + +func (c *ColumnInt64) Slice(start, end int) Column { + return &ColumnInt64{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +/* Float */ + +var _ Column = (*ColumnFloat)(nil) + +type ColumnFloat struct { + *genericColumnBase[float32] +} + +func NewColumnFloat(name string, values []float32) *ColumnFloat { + return &ColumnFloat{ + genericColumnBase: &genericColumnBase[float32]{ + name: name, + fieldType: entity.FieldTypeFloat, + values: values, + }, + } +} + +func (c *ColumnFloat) Slice(start, end int) Column { + return &ColumnFloat{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +func (c *ColumnFloat) GetAsDouble(idx int) (float64, error) { + v, err := c.Value(idx) + return float64(v), err +} + +/* Double */ + +var _ Column = (*ColumnFloat)(nil) + +type ColumnDouble struct { + *genericColumnBase[float64] +} + +func NewColumnDouble(name string, values []float64) *ColumnDouble { + return &ColumnDouble{ + genericColumnBase: &genericColumnBase[float64]{ + name: name, + fieldType: entity.FieldTypeDouble, + values: values, + }, + } +} + +func (c *ColumnDouble) Slice(start, end int) Column { + return &ColumnDouble{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +/* Varchar */ + +var _ (Column) = (*ColumnVarChar)(nil) + +type ColumnVarChar struct { + *genericColumnBase[string] +} + +func NewColumnVarChar(name string, values []string) *ColumnVarChar { + return &ColumnVarChar{ + genericColumnBase: &genericColumnBase[string]{ + name: name, + fieldType: entity.FieldTypeVarChar, + values: values, + }, + } +} + +func (c *ColumnVarChar) Slice(start, end int) Column { + return &ColumnVarChar{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} + +/* String */ +/* NOT USED */ + +var _ (Column) = (*ColumnString)(nil) + +type ColumnString struct { + *genericColumnBase[string] +} + +func NewColumnString(name string, values []string) *ColumnString { + return &ColumnString{ + genericColumnBase: &genericColumnBase[string]{ + name: name, + fieldType: entity.FieldTypeString, + values: values, + }, + } +} + +func (c *ColumnString) Slice(start, end int) Column { + return &ColumnString{ + genericColumnBase: c.genericColumnBase.slice(start, end), + } +} diff --git a/client/column/scalar_gen.go b/client/column/scalar_gen.go deleted file mode 100644 index 734ee127dc737..0000000000000 --- a/client/column/scalar_gen.go +++ /dev/null @@ -1,828 +0,0 @@ -// Code generated by go generate; DO NOT EDIT -// This file is generated by go generate - -package column - -import ( - "errors" - "fmt" - - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" -) - -// ColumnBool generated columns type for Bool -type ColumnBool struct { - ColumnBase - name string - values []bool -} - -// Name returns column name -func (c *ColumnBool) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnBool) Type() entity.FieldType { - return entity.FieldTypeBool -} - -// Len returns column values length -func (c *ColumnBool) Len() int { - return len(c.values) -} - -func (c *ColumnBool) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnBool{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnBool) Get(idx int) (interface{}, error) { - var r bool // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnBool) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Bool, - FieldName: c.name, - } - data := make([]bool, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, bool(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_BoolData{ - BoolData: &schemapb.BoolArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnBool) ValueByIdx(idx int) (bool, error) { - var r bool // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnBool) AppendValue(i interface{}) error { - v, ok := i.(bool) - if !ok { - return fmt.Errorf("invalid type, expected bool, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnBool) Data() []bool { - return c.values -} - -// NewColumnBool auto generated constructor -func NewColumnBool(name string, values []bool) *ColumnBool { - return &ColumnBool{ - name: name, - values: values, - } -} - -// ColumnInt8 generated columns type for Int8 -type ColumnInt8 struct { - ColumnBase - name string - values []int8 -} - -// Name returns column name -func (c *ColumnInt8) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt8) Type() entity.FieldType { - return entity.FieldTypeInt8 -} - -// Len returns column values length -func (c *ColumnInt8) Len() int { - return len(c.values) -} - -func (c *ColumnInt8) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt8{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt8) Get(idx int) (interface{}, error) { - var r int8 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt8) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int8, - FieldName: c.name, - } - data := make([]int32, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, int32(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt8) ValueByIdx(idx int) (int8, error) { - var r int8 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt8) AppendValue(i interface{}) error { - v, ok := i.(int8) - if !ok { - return fmt.Errorf("invalid type, expected int8, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt8) Data() []int8 { - return c.values -} - -// NewColumnInt8 auto generated constructor -func NewColumnInt8(name string, values []int8) *ColumnInt8 { - return &ColumnInt8{ - name: name, - values: values, - } -} - -// ColumnInt16 generated columns type for Int16 -type ColumnInt16 struct { - ColumnBase - name string - values []int16 -} - -// Name returns column name -func (c *ColumnInt16) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt16) Type() entity.FieldType { - return entity.FieldTypeInt16 -} - -// Len returns column values length -func (c *ColumnInt16) Len() int { - return len(c.values) -} - -func (c *ColumnInt16) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt16{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt16) Get(idx int) (interface{}, error) { - var r int16 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt16) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int16, - FieldName: c.name, - } - data := make([]int32, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, int32(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt16) ValueByIdx(idx int) (int16, error) { - var r int16 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt16) AppendValue(i interface{}) error { - v, ok := i.(int16) - if !ok { - return fmt.Errorf("invalid type, expected int16, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt16) Data() []int16 { - return c.values -} - -// NewColumnInt16 auto generated constructor -func NewColumnInt16(name string, values []int16) *ColumnInt16 { - return &ColumnInt16{ - name: name, - values: values, - } -} - -// ColumnInt32 generated columns type for Int32 -type ColumnInt32 struct { - ColumnBase - name string - values []int32 -} - -// Name returns column name -func (c *ColumnInt32) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt32) Type() entity.FieldType { - return entity.FieldTypeInt32 -} - -// Len returns column values length -func (c *ColumnInt32) Len() int { - return len(c.values) -} - -func (c *ColumnInt32) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt32{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt32) Get(idx int) (interface{}, error) { - var r int32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt32) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int32, - FieldName: c.name, - } - data := make([]int32, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, int32(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt32) ValueByIdx(idx int) (int32, error) { - var r int32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt32) AppendValue(i interface{}) error { - v, ok := i.(int32) - if !ok { - return fmt.Errorf("invalid type, expected int32, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt32) Data() []int32 { - return c.values -} - -// NewColumnInt32 auto generated constructor -func NewColumnInt32(name string, values []int32) *ColumnInt32 { - return &ColumnInt32{ - name: name, - values: values, - } -} - -// ColumnInt64 generated columns type for Int64 -type ColumnInt64 struct { - ColumnBase - name string - values []int64 -} - -// Name returns column name -func (c *ColumnInt64) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnInt64) Type() entity.FieldType { - return entity.FieldTypeInt64 -} - -// Len returns column values length -func (c *ColumnInt64) Len() int { - return len(c.values) -} - -func (c *ColumnInt64) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnInt64{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnInt64) Get(idx int) (interface{}, error) { - var r int64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnInt64) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int64, - FieldName: c.name, - } - data := make([]int64, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, int64(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_LongData{ - LongData: &schemapb.LongArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnInt64) ValueByIdx(idx int) (int64, error) { - var r int64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnInt64) AppendValue(i interface{}) error { - v, ok := i.(int64) - if !ok { - return fmt.Errorf("invalid type, expected int64, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnInt64) Data() []int64 { - return c.values -} - -// NewColumnInt64 auto generated constructor -func NewColumnInt64(name string, values []int64) *ColumnInt64 { - return &ColumnInt64{ - name: name, - values: values, - } -} - -// ColumnFloat generated columns type for Float -type ColumnFloat struct { - ColumnBase - name string - values []float32 -} - -// Name returns column name -func (c *ColumnFloat) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnFloat) Type() entity.FieldType { - return entity.FieldTypeFloat -} - -// Len returns column values length -func (c *ColumnFloat) Len() int { - return len(c.values) -} - -func (c *ColumnFloat) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnFloat{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnFloat) Get(idx int) (interface{}, error) { - var r float32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnFloat) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Float, - FieldName: c.name, - } - data := make([]float32, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, float32(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_FloatData{ - FloatData: &schemapb.FloatArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnFloat) ValueByIdx(idx int) (float32, error) { - var r float32 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnFloat) AppendValue(i interface{}) error { - v, ok := i.(float32) - if !ok { - return fmt.Errorf("invalid type, expected float32, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnFloat) Data() []float32 { - return c.values -} - -// NewColumnFloat auto generated constructor -func NewColumnFloat(name string, values []float32) *ColumnFloat { - return &ColumnFloat{ - name: name, - values: values, - } -} - -// ColumnDouble generated columns type for Double -type ColumnDouble struct { - ColumnBase - name string - values []float64 -} - -// Name returns column name -func (c *ColumnDouble) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnDouble) Type() entity.FieldType { - return entity.FieldTypeDouble -} - -// Len returns column values length -func (c *ColumnDouble) Len() int { - return len(c.values) -} - -func (c *ColumnDouble) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnDouble{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnDouble) Get(idx int) (interface{}, error) { - var r float64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnDouble) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Double, - FieldName: c.name, - } - data := make([]float64, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, float64(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_DoubleData{ - DoubleData: &schemapb.DoubleArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnDouble) ValueByIdx(idx int) (float64, error) { - var r float64 // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnDouble) AppendValue(i interface{}) error { - v, ok := i.(float64) - if !ok { - return fmt.Errorf("invalid type, expected float64, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnDouble) Data() []float64 { - return c.values -} - -// NewColumnDouble auto generated constructor -func NewColumnDouble(name string, values []float64) *ColumnDouble { - return &ColumnDouble{ - name: name, - values: values, - } -} - -// ColumnString generated columns type for String -type ColumnString struct { - ColumnBase - name string - values []string -} - -// Name returns column name -func (c *ColumnString) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnString) Type() entity.FieldType { - return entity.FieldTypeString -} - -// Len returns column values length -func (c *ColumnString) Len() int { - return len(c.values) -} - -func (c *ColumnString) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnString{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnString) Get(idx int) (interface{}, error) { - var r string // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnString) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_String, - FieldName: c.name, - } - data := make([]string, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, string(c.values[i])) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnString) ValueByIdx(idx int) (string, error) { - var r string // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnString) AppendValue(i interface{}) error { - v, ok := i.(string) - if !ok { - return fmt.Errorf("invalid type, expected string, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnString) Data() []string { - return c.values -} - -// NewColumnString auto generated constructor -func NewColumnString(name string, values []string) *ColumnString { - return &ColumnString{ - name: name, - values: values, - } -} diff --git a/client/column/scalar_gen_test.go b/client/column/scalar_gen_test.go deleted file mode 100644 index 5e325a640ba3a..0000000000000 --- a/client/column/scalar_gen_test.go +++ /dev/null @@ -1,855 +0,0 @@ -// Code generated by go generate; DO NOT EDIT -// This file is generated by go generated - -package column - -import ( - "fmt" - "math/rand" - "testing" - "time" - - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" - "github.com/stretchr/testify/assert" -) - -func TestColumnBool(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Bool_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]bool, columnLen) - column := NewColumnBool(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeBool - assert.Equal(t, "Bool", ft.Name()) - assert.Equal(t, "bool", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Bool", pbName) - assert.Equal(t, "bool", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeBool, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataBoolColumn(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Bool_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Bool, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_BoolData{ - BoolData: &schemapb.BoolArray{ - Data: make([]bool, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeBool, column.Type()) - - var ev bool - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_BoolData{ - BoolData: &schemapb.BoolArray{ - Data: make([]bool, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeBool, column.Type()) - }) -} - -func TestColumnInt8(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Int8_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]int8, columnLen) - column := NewColumnInt8(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeInt8 - assert.Equal(t, "Int8", ft.Name()) - assert.Equal(t, "int8", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Int", pbName) - assert.Equal(t, "int32", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeInt8, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataInt8Column(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Int8_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int8, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: make([]int32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt8, column.Type()) - - var ev int8 - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: make([]int32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt8, column.Type()) - }) -} - -func TestColumnInt16(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Int16_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]int16, columnLen) - column := NewColumnInt16(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeInt16 - assert.Equal(t, "Int16", ft.Name()) - assert.Equal(t, "int16", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Int", pbName) - assert.Equal(t, "int32", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeInt16, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataInt16Column(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Int16_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int16, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: make([]int32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt16, column.Type()) - - var ev int16 - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: make([]int32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt16, column.Type()) - }) -} - -func TestColumnInt32(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Int32_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]int32, columnLen) - column := NewColumnInt32(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeInt32 - assert.Equal(t, "Int32", ft.Name()) - assert.Equal(t, "int32", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Int", pbName) - assert.Equal(t, "int32", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeInt32, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataInt32Column(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Int32_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int32, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: make([]int32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt32, column.Type()) - - var ev int32 - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_IntData{ - IntData: &schemapb.IntArray{ - Data: make([]int32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt32, column.Type()) - }) -} - -func TestColumnInt64(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Int64_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]int64, columnLen) - column := NewColumnInt64(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeInt64 - assert.Equal(t, "Int64", ft.Name()) - assert.Equal(t, "int64", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Long", pbName) - assert.Equal(t, "int64", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeInt64, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataInt64Column(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Int64_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Int64, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_LongData{ - LongData: &schemapb.LongArray{ - Data: make([]int64, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt64, column.Type()) - - var ev int64 - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_LongData{ - LongData: &schemapb.LongArray{ - Data: make([]int64, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeInt64, column.Type()) - }) -} - -func TestColumnFloat(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Float_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]float32, columnLen) - column := NewColumnFloat(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeFloat - assert.Equal(t, "Float", ft.Name()) - assert.Equal(t, "float32", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Float", pbName) - assert.Equal(t, "float32", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeFloat, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataFloatColumn(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Float_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Float, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_FloatData{ - FloatData: &schemapb.FloatArray{ - Data: make([]float32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeFloat, column.Type()) - - var ev float32 - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_FloatData{ - FloatData: &schemapb.FloatArray{ - Data: make([]float32, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeFloat, column.Type()) - }) -} - -func TestColumnDouble(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Double_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]float64, columnLen) - column := NewColumnDouble(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeDouble - assert.Equal(t, "Double", ft.Name()) - assert.Equal(t, "float64", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "Double", pbName) - assert.Equal(t, "float64", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeDouble, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataDoubleColumn(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_Double_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Double, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_DoubleData{ - DoubleData: &schemapb.DoubleArray{ - Data: make([]float64, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeDouble, column.Type()) - - var ev float64 - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_DoubleData{ - DoubleData: &schemapb.DoubleArray{ - Data: make([]float64, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeDouble, column.Type()) - }) -} - -func TestColumnString(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_String_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]string, columnLen) - column := NewColumnString(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeString - assert.Equal(t, "String", ft.Name()) - assert.Equal(t, "string", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "String", pbName) - assert.Equal(t, "string", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeString, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataStringColumn(t *testing.T) { - len := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_String_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_String, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: make([]string, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, len) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeString, column.Type()) - - var ev string - err = column.AppendValue(ev) - assert.Equal(t, len+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, len+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, len) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: make([]string, len), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, len, column.Len()) - assert.Equal(t, entity.FieldTypeString, column.Type()) - }) -} diff --git a/client/column/scalar_test.go b/client/column/scalar_test.go new file mode 100644 index 0000000000000..4763d64bf9e27 --- /dev/null +++ b/client/column/scalar_test.go @@ -0,0 +1,219 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/milvus-io/milvus/client/v2/entity" +) + +type ScalarSuite struct { + suite.Suite +} + +func (s *ScalarSuite) TestBasic() { + s.Run("column_bool", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []bool{true, false} + column := NewColumnBool(name, data) + s.Equal(entity.FieldTypeBool, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(data, fd.GetScalars().GetBoolData().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnBool) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeBool, column.Type()) + } + }) + + s.Run("column_int8", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []int8{1, 2, 3} + column := NewColumnInt8(name, data) + s.Equal(entity.FieldTypeInt8, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + fdData := fd.GetScalars().GetIntData().GetData() + for i, row := range data { + s.EqualValues(row, fdData[i]) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt8) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeInt8, column.Type()) + } + }) + + s.Run("column_int16", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []int16{1, 2, 3} + column := NewColumnInt16(name, data) + s.Equal(entity.FieldTypeInt16, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + fdData := fd.GetScalars().GetIntData().GetData() + for i, row := range data { + s.EqualValues(row, fdData[i]) + } + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt16) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeInt16, column.Type()) + } + }) + + s.Run("column_int32", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []int32{1, 2, 3} + column := NewColumnInt32(name, data) + s.Equal(entity.FieldTypeInt32, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(data, fd.GetScalars().GetIntData().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt32) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeInt32, column.Type()) + } + }) + + s.Run("column_int64", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []int64{1, 2, 3} + column := NewColumnInt64(name, data) + s.Equal(entity.FieldTypeInt64, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(data, fd.GetScalars().GetLongData().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnInt64) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeInt64, column.Type()) + } + }) + + s.Run("column_float", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []float32{1.1, 2.2, 3.3} + column := NewColumnFloat(name, data) + s.Equal(entity.FieldTypeFloat, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(data, fd.GetScalars().GetFloatData().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnFloat) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeFloat, column.Type()) + } + }) + + s.Run("column_double", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []float64{1.1, 2.2, 3.3} + column := NewColumnDouble(name, data) + s.Equal(entity.FieldTypeDouble, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(data, fd.GetScalars().GetDoubleData().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnDouble) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeDouble, column.Type()) + } + }) + + s.Run("column_varchar", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + data := []string{"a", "b", "c"} + column := NewColumnVarChar(name, data) + s.Equal(entity.FieldTypeVarChar, column.Type()) + s.Equal(name, column.Name()) + s.Equal(data, column.Data()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(data, fd.GetScalars().GetStringData().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnVarChar) + if s.True(ok) { + s.Equal(name, parsed.Name()) + s.Equal(data, parsed.Data()) + s.Equal(entity.FieldTypeVarChar, column.Type()) + } + }) +} + +func TestScalarColumn(t *testing.T) { + suite.Run(t, new(ScalarSuite)) +} diff --git a/client/column/sparse.go b/client/column/sparse.go index 96cb51d84918a..df99694a6b563 100644 --- a/client/column/sparse.go +++ b/client/column/sparse.go @@ -17,11 +17,7 @@ package column import ( - "encoding/binary" - "fmt" - "math" - - "github.com/cockroachdb/errors" + "github.com/samber/lo" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/client/v2/entity" @@ -30,112 +26,22 @@ import ( var _ (Column) = (*ColumnSparseFloatVector)(nil) type ColumnSparseFloatVector struct { - ColumnBase - - vectors []entity.SparseEmbedding - name string + *vectorBase[entity.SparseEmbedding] } -// Name returns column name. -func (c *ColumnSparseFloatVector) Name() string { - return c.name -} - -// Type returns column FieldType. -func (c *ColumnSparseFloatVector) Type() entity.FieldType { - return entity.FieldTypeSparseVector -} - -// Len returns column values length. -func (c *ColumnSparseFloatVector) Len() int { - return len(c.vectors) -} - -func (c *ColumnSparseFloatVector) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } +func NewColumnSparseVectors(name string, values []entity.SparseEmbedding) *ColumnSparseFloatVector { return &ColumnSparseFloatVector{ - ColumnBase: c.ColumnBase, - name: c.name, - vectors: c.vectors[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnSparseFloatVector) Get(idx int) (interface{}, error) { - if idx < 0 || idx >= c.Len() { - return nil, errors.New("index out of range") - } - return c.vectors[idx], nil -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnSparseFloatVector) ValueByIdx(idx int) (entity.SparseEmbedding, error) { - var r entity.SparseEmbedding // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") + // sparse embedding need not specify dimension + vectorBase: newVectorBase(name, 0, values, entity.FieldTypeSparseVector), } - return c.vectors[idx], nil } func (c *ColumnSparseFloatVector) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_SparseFloatVector, - FieldName: c.name, - } - - dim := int(0) - data := make([][]byte, 0, len(c.vectors)) - for _, vector := range c.vectors { - row := make([]byte, 8*vector.Len()) - for idx := 0; idx < vector.Len(); idx++ { - pos, value, _ := vector.Get(idx) - binary.LittleEndian.PutUint32(row[idx*8:], pos) - binary.LittleEndian.PutUint32(row[idx*8+4:], math.Float32bits(value)) - } - data = append(data, row) - if vector.Dim() > dim { - dim = vector.Dim() - } - } - - fd.Field = &schemapb.FieldData_Vectors{ - Vectors: &schemapb.VectorField{ - Dim: int64(dim), - Data: &schemapb.VectorField_SparseFloatVector{ - SparseFloatVector: &schemapb.SparseFloatArray{ - Dim: int64(dim), - Contents: data, - }, - }, - }, - } + fd := c.vectorBase.FieldData() + max := lo.MaxBy(c.values, func(a, b entity.SparseEmbedding) bool { + return a.Dim() > b.Dim() + }) + vectors := fd.GetVectors() + vectors.Dim = int64(max.Dim()) return fd } - -func (c *ColumnSparseFloatVector) AppendValue(i interface{}) error { - v, ok := i.(entity.SparseEmbedding) - if !ok { - return fmt.Errorf("invalid type, expect SparseEmbedding interface, got %T", i) - } - c.vectors = append(c.vectors, v) - - return nil -} - -func (c *ColumnSparseFloatVector) Data() []entity.SparseEmbedding { - return c.vectors -} - -func NewColumnSparseVectors(name string, values []entity.SparseEmbedding) *ColumnSparseFloatVector { - return &ColumnSparseFloatVector{ - name: name, - vectors: values, - } -} diff --git a/client/column/sparse_test.go b/client/column/sparse_test.go index 564f223ff1532..bd3b6c89fb5d1 100644 --- a/client/column/sparse_test.go +++ b/client/column/sparse_test.go @@ -57,12 +57,24 @@ func TestColumnSparseEmbedding(t *testing.T) { fd := column.FieldData() assert.NotNil(t, fd) assert.Equal(t, fd.GetFieldName(), columnName) + + result, err := FieldDataColumn(fd, 0, -1) + assert.NoError(t, err) + + parsed, ok := result.(*ColumnSparseFloatVector) + if assert.True(t, ok) { + assert.Equal(t, columnName, parsed.Name()) + assert.Equal(t, entity.FieldTypeSparseVector, parsed.Type()) + assert.Equal(t, columnLen, parsed.Len()) + // dim not equal + // assert.EqualValues(t, v, parsed.Data()) + } }) t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) + _, err := column.Value(-1) assert.Error(t, err) - _, err = column.ValueByIdx(columnLen) + _, err = column.Value(columnLen) assert.Error(t, err) _, err = column.Get(-1) @@ -71,9 +83,9 @@ func TestColumnSparseEmbedding(t *testing.T) { assert.Error(t, err) for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) + v, err := column.Value(i) assert.NoError(t, err) - assert.Equal(t, column.vectors[i], v) + assert.Equal(t, column.values[i], v) getV, err := column.Get(i) assert.NoError(t, err) assert.Equal(t, v, getV) diff --git a/client/column/varchar.go b/client/column/varchar.go deleted file mode 100644 index de96e3fe47f5b..0000000000000 --- a/client/column/varchar.go +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package column - -import ( - "fmt" - - "github.com/cockroachdb/errors" - - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" -) - -// ColumnVarChar generated columns type for VarChar -type ColumnVarChar struct { - ColumnBase - name string - values []string -} - -// Name returns column name -func (c *ColumnVarChar) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnVarChar) Type() entity.FieldType { - return entity.FieldTypeVarChar -} - -// Len returns column values length -func (c *ColumnVarChar) Len() int { - return len(c.values) -} - -func (c *ColumnVarChar) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - return &ColumnVarChar{ - ColumnBase: c.ColumnBase, - name: c.name, - values: c.values[start:end], - } -} - -// Get returns value at index as interface{}. -func (c *ColumnVarChar) Get(idx int) (interface{}, error) { - if idx < 0 || idx > c.Len() { - return "", errors.New("index out of range") - } - return c.values[idx], nil -} - -// GetAsString returns value at idx. -func (c *ColumnVarChar) GetAsString(idx int) (string, error) { - if idx < 0 || idx > c.Len() { - return "", errors.New("index out of range") - } - return c.values[idx], nil -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnVarChar) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_VarChar, - FieldName: c.name, - } - data := make([]string, 0, c.Len()) - for i := 0; i < c.Len(); i++ { - data = append(data, c.values[i]) - } - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// ValueByIdx returns value of the provided index -// error occurs when index out of range -func (c *ColumnVarChar) ValueByIdx(idx int) (string, error) { - var r string // use default value - if idx < 0 || idx >= c.Len() { - return r, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnVarChar) AppendValue(i interface{}) error { - v, ok := i.(string) - if !ok { - return fmt.Errorf("invalid type, expected string, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnVarChar) Data() []string { - return c.values -} - -// NewColumnVarChar auto generated constructor -func NewColumnVarChar(name string, values []string) *ColumnVarChar { - return &ColumnVarChar{ - name: name, - values: values, - } -} diff --git a/client/column/varchar_test.go b/client/column/varchar_test.go deleted file mode 100644 index 8e2535c8154b2..0000000000000 --- a/client/column/varchar_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package column - -import ( - "fmt" - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" -) - -func TestColumnVarChar(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_VarChar_%d", rand.Int()) - columnLen := 8 + rand.Intn(10) - - v := make([]string, columnLen) - column := NewColumnVarChar(columnName, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeVarChar - assert.Equal(t, "VarChar", ft.Name()) - assert.Equal(t, "string", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "VarChar", pbName) - assert.Equal(t, "string", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeVarChar, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.EqualValues(t, v, column.Data()) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - }) - - t.Run("test column value by idx", func(t *testing.T) { - _, err := column.ValueByIdx(-1) - assert.NotNil(t, err) - _, err = column.ValueByIdx(columnLen) - assert.NotNil(t, err) - for i := 0; i < columnLen; i++ { - v, err := column.ValueByIdx(i) - assert.Nil(t, err) - assert.Equal(t, column.values[i], v) - } - }) -} - -func TestFieldDataVarCharColumn(t *testing.T) { - colLen := rand.Intn(10) + 8 - name := fmt.Sprintf("fd_VarChar_%d", rand.Int()) - fd := &schemapb.FieldData{ - Type: schemapb.DataType_VarChar, - FieldName: name, - } - - t.Run("normal usage", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: make([]string, colLen), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, colLen) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, colLen, column.Len()) - assert.Equal(t, entity.FieldTypeVarChar, column.Type()) - - var ev string - err = column.AppendValue(ev) - assert.Equal(t, colLen+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, colLen+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("nil data", func(t *testing.T) { - fd.Field = nil - _, err := FieldDataColumn(fd, 0, colLen) - assert.NotNil(t, err) - }) - - t.Run("get all data", func(t *testing.T) { - fd.Field = &schemapb.FieldData_Scalars{ - Scalars: &schemapb.ScalarField{ - Data: &schemapb.ScalarField_StringData{ - StringData: &schemapb.StringArray{ - Data: make([]string, colLen), - }, - }, - }, - } - column, err := FieldDataColumn(fd, 0, -1) - assert.Nil(t, err) - assert.NotNil(t, column) - - assert.Equal(t, name, column.Name()) - assert.Equal(t, colLen, column.Len()) - assert.Equal(t, entity.FieldTypeVarChar, column.Type()) - }) -} diff --git a/client/column/vector.go b/client/column/vector.go new file mode 100644 index 0000000000000..6e3a3e1dac7df --- /dev/null +++ b/client/column/vector.go @@ -0,0 +1,158 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import ( + "github.com/cockroachdb/errors" + "github.com/samber/lo" + + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/client/v2/entity" +) + +type vectorBase[T entity.Vector] struct { + *genericColumnBase[T] + dim int +} + +func (b *vectorBase[T]) Dim() int { + return b.dim +} + +func (b *vectorBase[T]) FieldData() *schemapb.FieldData { + fd := b.genericColumnBase.FieldData() + vectors := fd.GetVectors() + vectors.Dim = int64(b.dim) + return fd +} + +func newVectorBase[T entity.Vector](fieldName string, dim int, vectors []T, fieldType entity.FieldType) *vectorBase[T] { + return &vectorBase[T]{ + genericColumnBase: &genericColumnBase[T]{ + name: fieldName, + fieldType: fieldType, + values: vectors, + }, + dim: dim, + } +} + +/* float vector */ + +type ColumnFloatVector struct { + *vectorBase[entity.FloatVector] +} + +func NewColumnFloatVector(fieldName string, dim int, data [][]float32) *ColumnFloatVector { + vectors := lo.Map(data, func(row []float32, _ int) entity.FloatVector { return entity.FloatVector(row) }) + return &ColumnFloatVector{ + vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeFloatVector), + } +} + +// AppendValue appends vector value into values. +// override default type constrains, add `[]float32` conversion +func (c *ColumnFloatVector) AppendValue(i interface{}) error { + switch vector := i.(type) { + case entity.FloatVector: + c.values = append(c.values, vector) + case []float32: + c.values = append(c.values, vector) + default: + return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType) + } + return nil +} + +/* binary vector */ + +type ColumnBinaryVector struct { + *vectorBase[entity.BinaryVector] +} + +func NewColumnBinaryVector(fieldName string, dim int, data [][]byte) *ColumnBinaryVector { + vectors := lo.Map(data, func(row []byte, _ int) entity.BinaryVector { return entity.BinaryVector(row) }) + return &ColumnBinaryVector{ + vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeBinaryVector), + } +} + +// AppendValue appends vector value into values. +// override default type constrains, add `[]byte` conversion +func (c *ColumnBinaryVector) AppendValue(i interface{}) error { + switch vector := i.(type) { + case entity.BinaryVector: + c.values = append(c.values, vector) + case []byte: + c.values = append(c.values, vector) + default: + return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType) + } + return nil +} + +/* fp16 vector */ + +type ColumnFloat16Vector struct { + *vectorBase[entity.Float16Vector] +} + +func NewColumnFloat16Vector(fieldName string, dim int, data [][]byte) *ColumnFloat16Vector { + vectors := lo.Map(data, func(row []byte, _ int) entity.Float16Vector { return entity.Float16Vector(row) }) + return &ColumnFloat16Vector{ + vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeFloat16Vector), + } +} + +// AppendValue appends vector value into values. +// override default type constrains, add `[]byte` conversion +func (c *ColumnFloat16Vector) AppendValue(i interface{}) error { + switch vector := i.(type) { + case entity.Float16Vector: + c.values = append(c.values, vector) + case []byte: + c.values = append(c.values, vector) + default: + return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType) + } + return nil +} + +type ColumnBFloat16Vector struct { + *vectorBase[entity.BFloat16Vector] +} + +func NewColumnBFloat16Vector(fieldName string, dim int, data [][]byte) *ColumnBFloat16Vector { + vectors := lo.Map(data, func(row []byte, _ int) entity.BFloat16Vector { return entity.BFloat16Vector(row) }) + return &ColumnBFloat16Vector{ + vectorBase: newVectorBase(fieldName, dim, vectors, entity.FieldTypeBFloat16Vector), + } +} + +// AppendValue appends vector value into values. +// override default type constrains, add `[]byte` conversion +func (c *ColumnBFloat16Vector) AppendValue(i interface{}) error { + switch vector := i.(type) { + case entity.BFloat16Vector: + c.values = append(c.values, vector) + case []byte: + c.values = append(c.values, vector) + default: + return errors.Newf("unexpected append value type %T, field type %v", vector, c.fieldType) + } + return nil +} diff --git a/client/column/vector_gen.go b/client/column/vector_gen.go deleted file mode 100644 index e72a78ee0906e..0000000000000 --- a/client/column/vector_gen.go +++ /dev/null @@ -1,434 +0,0 @@ -// Code generated by go generate; DO NOT EDIT -// This file is generated by go generated -package column - -import ( - "fmt" - - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" - - "github.com/cockroachdb/errors" -) - -// ColumnBinaryVector generated columns type for BinaryVector -type ColumnBinaryVector struct { - ColumnBase - name string - dim int - values [][]byte -} - -// Name returns column name -func (c *ColumnBinaryVector) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnBinaryVector) Type() entity.FieldType { - return entity.FieldTypeBinaryVector -} - -// Len returns column data length -func (c *ColumnBinaryVector) Len() int { - return len(c.values) -} - -func (c *ColumnBinaryVector) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - if end == -1 || end > l { - end = l - } - return &ColumnBinaryVector{ - ColumnBase: c.ColumnBase, - name: c.name, - dim: c.dim, - values: c.values[start:end], - } -} - -// Dim returns vector dimension -func (c *ColumnBinaryVector) Dim() int { - return c.dim -} - -// Get returns values at index as interface{}. -func (c *ColumnBinaryVector) Get(idx int) (interface{}, error) { - if idx < 0 || idx >= c.Len() { - return nil, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnBinaryVector) AppendValue(i interface{}) error { - v, ok := i.([]byte) - if !ok { - return fmt.Errorf("invalid type, expected []byte, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnBinaryVector) Data() [][]byte { - return c.values -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnBinaryVector) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_BinaryVector, - FieldName: c.name, - } - - data := make([]byte, 0, len(c.values)*c.dim) - - for _, vector := range c.values { - data = append(data, vector...) - } - - fd.Field = &schemapb.FieldData_Vectors{ - Vectors: &schemapb.VectorField{ - Dim: int64(c.dim), - - Data: &schemapb.VectorField_BinaryVector{ - BinaryVector: data, - }, - }, - } - return fd -} - -// NewColumnBinaryVector auto generated constructor -func NewColumnBinaryVector(name string, dim int, values [][]byte) *ColumnBinaryVector { - return &ColumnBinaryVector{ - name: name, - dim: dim, - values: values, - } -} - -// ColumnFloatVector generated columns type for FloatVector -type ColumnFloatVector struct { - ColumnBase - name string - dim int - values [][]float32 -} - -// Name returns column name -func (c *ColumnFloatVector) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnFloatVector) Type() entity.FieldType { - return entity.FieldTypeFloatVector -} - -// Len returns column data length -func (c *ColumnFloatVector) Len() int { - return len(c.values) -} - -func (c *ColumnFloatVector) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - if end == -1 || end > l { - end = l - } - return &ColumnFloatVector{ - ColumnBase: c.ColumnBase, - name: c.name, - dim: c.dim, - values: c.values[start:end], - } -} - -// Dim returns vector dimension -func (c *ColumnFloatVector) Dim() int { - return c.dim -} - -// Get returns values at index as interface{}. -func (c *ColumnFloatVector) Get(idx int) (interface{}, error) { - if idx < 0 || idx >= c.Len() { - return nil, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnFloatVector) AppendValue(i interface{}) error { - v, ok := i.([]float32) - if !ok { - return fmt.Errorf("invalid type, expected []float32, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnFloatVector) Data() [][]float32 { - return c.values -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnFloatVector) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_FloatVector, - FieldName: c.name, - } - - data := make([]float32, 0, len(c.values)*c.dim) - - for _, vector := range c.values { - data = append(data, vector...) - } - - fd.Field = &schemapb.FieldData_Vectors{ - Vectors: &schemapb.VectorField{ - Dim: int64(c.dim), - - Data: &schemapb.VectorField_FloatVector{ - FloatVector: &schemapb.FloatArray{ - Data: data, - }, - }, - }, - } - return fd -} - -// NewColumnFloatVector auto generated constructor -func NewColumnFloatVector(name string, dim int, values [][]float32) *ColumnFloatVector { - return &ColumnFloatVector{ - name: name, - dim: dim, - values: values, - } -} - -// ColumnFloat16Vector generated columns type for Float16Vector -type ColumnFloat16Vector struct { - ColumnBase - name string - dim int - values [][]byte -} - -// Name returns column name -func (c *ColumnFloat16Vector) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnFloat16Vector) Type() entity.FieldType { - return entity.FieldTypeFloat16Vector -} - -// Len returns column data length -func (c *ColumnFloat16Vector) Len() int { - return len(c.values) -} - -func (c *ColumnFloat16Vector) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - if end == -1 || end > l { - end = l - } - return &ColumnFloat16Vector{ - ColumnBase: c.ColumnBase, - name: c.name, - dim: c.dim, - values: c.values[start:end], - } -} - -// Dim returns vector dimension -func (c *ColumnFloat16Vector) Dim() int { - return c.dim -} - -// Get returns values at index as interface{}. -func (c *ColumnFloat16Vector) Get(idx int) (interface{}, error) { - if idx < 0 || idx >= c.Len() { - return nil, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnFloat16Vector) AppendValue(i interface{}) error { - v, ok := i.([]byte) - if !ok { - return fmt.Errorf("invalid type, expected []byte, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnFloat16Vector) Data() [][]byte { - return c.values -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnFloat16Vector) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Float16Vector, - FieldName: c.name, - } - - data := make([]byte, 0, len(c.values)*c.dim*2) - - for _, vector := range c.values { - data = append(data, vector...) - } - - fd.Field = &schemapb.FieldData_Vectors{ - Vectors: &schemapb.VectorField{ - Dim: int64(c.dim), - - Data: &schemapb.VectorField_Float16Vector{ - Float16Vector: data, - }, - }, - } - return fd -} - -// NewColumnFloat16Vector auto generated constructor -func NewColumnFloat16Vector(name string, dim int, values [][]byte) *ColumnFloat16Vector { - return &ColumnFloat16Vector{ - name: name, - dim: dim, - values: values, - } -} - -// ColumnBFloat16Vector generated columns type for BFloat16Vector -type ColumnBFloat16Vector struct { - ColumnBase - name string - dim int - values [][]byte -} - -// Name returns column name -func (c *ColumnBFloat16Vector) Name() string { - return c.name -} - -// Type returns column entity.FieldType -func (c *ColumnBFloat16Vector) Type() entity.FieldType { - return entity.FieldTypeBFloat16Vector -} - -// Len returns column data length -func (c *ColumnBFloat16Vector) Len() int { - return len(c.values) -} - -func (c *ColumnBFloat16Vector) Slice(start, end int) Column { - l := c.Len() - if start > l { - start = l - } - if end == -1 || end > l { - end = l - } - if end == -1 || end > l { - end = l - } - return &ColumnBFloat16Vector{ - ColumnBase: c.ColumnBase, - name: c.name, - dim: c.dim, - values: c.values[start:end], - } -} - -// Dim returns vector dimension -func (c *ColumnBFloat16Vector) Dim() int { - return c.dim -} - -// Get returns values at index as interface{}. -func (c *ColumnBFloat16Vector) Get(idx int) (interface{}, error) { - if idx < 0 || idx >= c.Len() { - return nil, errors.New("index out of range") - } - return c.values[idx], nil -} - -// AppendValue append value into column -func (c *ColumnBFloat16Vector) AppendValue(i interface{}) error { - v, ok := i.([]byte) - if !ok { - return fmt.Errorf("invalid type, expected []byte, got %T", i) - } - c.values = append(c.values, v) - - return nil -} - -// Data returns column data -func (c *ColumnBFloat16Vector) Data() [][]byte { - return c.values -} - -// FieldData return column data mapped to schemapb.FieldData -func (c *ColumnBFloat16Vector) FieldData() *schemapb.FieldData { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_BFloat16Vector, - FieldName: c.name, - } - - data := make([]byte, 0, len(c.values)*c.dim*2) - - for _, vector := range c.values { - data = append(data, vector...) - } - - fd.Field = &schemapb.FieldData_Vectors{ - Vectors: &schemapb.VectorField{ - Dim: int64(c.dim), - - Data: &schemapb.VectorField_Bfloat16Vector{ - Bfloat16Vector: data, - }, - }, - } - return fd -} - -// NewColumnBFloat16Vector auto generated constructor -func NewColumnBFloat16Vector(name string, dim int, values [][]byte) *ColumnBFloat16Vector { - return &ColumnBFloat16Vector{ - name: name, - dim: dim, - values: values, - } -} diff --git a/client/column/vector_gen_test.go b/client/column/vector_gen_test.go deleted file mode 100644 index b2fdf9caa733b..0000000000000 --- a/client/column/vector_gen_test.go +++ /dev/null @@ -1,264 +0,0 @@ -// Code generated by go generate; DO NOT EDIT -// This file is generated by go generated - -package column - -import ( - "fmt" - "math/rand" - "testing" - "time" - - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/client/v2/entity" - "github.com/stretchr/testify/assert" -) - -func TestColumnBinaryVector(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_BinaryVector_%d", rand.Int()) - columnLen := 12 + rand.Intn(10) - dim := ([]int{64, 128, 256, 512})[rand.Intn(4)] - - v := make([][]byte, 0, columnLen) - dlen := dim - dlen /= 8 - - for i := 0; i < columnLen; i++ { - entry := make([]byte, dlen) - v = append(v, entry) - } - column := NewColumnBinaryVector(columnName, dim, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeBinaryVector - assert.Equal(t, "BinaryVector", ft.Name()) - assert.Equal(t, "[]byte", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "[]byte", pbName) - assert.Equal(t, "", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeBinaryVector, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.Equal(t, dim, column.Dim()) - assert.Equal(t, v, column.Data()) - - var ev []byte - err := column.AppendValue(ev) - assert.Equal(t, columnLen+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, columnLen+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - - c, err := FieldDataVector(fd) - assert.NotNil(t, c) - assert.NoError(t, err) - }) - - t.Run("test column field data error", func(t *testing.T) { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_BinaryVector, - FieldName: columnName, - } - _, err := FieldDataVector(fd) - assert.Error(t, err) - }) -} - -func TestColumnFloatVector(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_FloatVector_%d", rand.Int()) - columnLen := 12 + rand.Intn(10) - dim := ([]int{64, 128, 256, 512})[rand.Intn(4)] - - v := make([][]float32, 0, columnLen) - dlen := dim - - for i := 0; i < columnLen; i++ { - entry := make([]float32, dlen) - v = append(v, entry) - } - column := NewColumnFloatVector(columnName, dim, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeFloatVector - assert.Equal(t, "FloatVector", ft.Name()) - assert.Equal(t, "[]float32", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "[]float32", pbName) - assert.Equal(t, "", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeFloatVector, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.Equal(t, dim, column.Dim()) - assert.Equal(t, v, column.Data()) - - var ev []float32 - err := column.AppendValue(ev) - assert.Equal(t, columnLen+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, columnLen+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - - c, err := FieldDataVector(fd) - assert.NotNil(t, c) - assert.NoError(t, err) - }) - - t.Run("test column field data error", func(t *testing.T) { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_FloatVector, - FieldName: columnName, - } - _, err := FieldDataVector(fd) - assert.Error(t, err) - }) -} - -func TestColumnFloat16Vector(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_Float16Vector_%d", rand.Int()) - columnLen := 12 + rand.Intn(10) - dim := ([]int{64, 128, 256, 512})[rand.Intn(4)] - - v := make([][]byte, 0, columnLen) - dlen := dim - - dlen *= 2 - - for i := 0; i < columnLen; i++ { - entry := make([]byte, dlen) - v = append(v, entry) - } - column := NewColumnFloat16Vector(columnName, dim, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeFloat16Vector - assert.Equal(t, "Float16Vector", ft.Name()) - assert.Equal(t, "[]byte", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "[]byte", pbName) - assert.Equal(t, "", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeFloat16Vector, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.Equal(t, dim, column.Dim()) - assert.Equal(t, v, column.Data()) - - var ev []byte - err := column.AppendValue(ev) - assert.Equal(t, columnLen+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, columnLen+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - - c, err := FieldDataVector(fd) - assert.NotNil(t, c) - assert.NoError(t, err) - }) - - t.Run("test column field data error", func(t *testing.T) { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_Float16Vector, - FieldName: columnName, - } - _, err := FieldDataVector(fd) - assert.Error(t, err) - }) -} - -func TestColumnBFloat16Vector(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - columnName := fmt.Sprintf("column_BFloat16Vector_%d", rand.Int()) - columnLen := 12 + rand.Intn(10) - dim := ([]int{64, 128, 256, 512})[rand.Intn(4)] - - v := make([][]byte, 0, columnLen) - dlen := dim - - dlen *= 2 - - for i := 0; i < columnLen; i++ { - entry := make([]byte, dlen) - v = append(v, entry) - } - column := NewColumnBFloat16Vector(columnName, dim, v) - - t.Run("test meta", func(t *testing.T) { - ft := entity.FieldTypeBFloat16Vector - assert.Equal(t, "BFloat16Vector", ft.Name()) - assert.Equal(t, "[]byte", ft.String()) - pbName, pbType := ft.PbFieldType() - assert.Equal(t, "[]byte", pbName) - assert.Equal(t, "", pbType) - }) - - t.Run("test column attribute", func(t *testing.T) { - assert.Equal(t, columnName, column.Name()) - assert.Equal(t, entity.FieldTypeBFloat16Vector, column.Type()) - assert.Equal(t, columnLen, column.Len()) - assert.Equal(t, dim, column.Dim()) - assert.Equal(t, v, column.Data()) - - var ev []byte - err := column.AppendValue(ev) - assert.Equal(t, columnLen+1, column.Len()) - assert.Nil(t, err) - - err = column.AppendValue(struct{}{}) - assert.Equal(t, columnLen+1, column.Len()) - assert.NotNil(t, err) - }) - - t.Run("test column field data", func(t *testing.T) { - fd := column.FieldData() - assert.NotNil(t, fd) - assert.Equal(t, fd.GetFieldName(), columnName) - - c, err := FieldDataVector(fd) - assert.NotNil(t, c) - assert.NoError(t, err) - }) - - t.Run("test column field data error", func(t *testing.T) { - fd := &schemapb.FieldData{ - Type: schemapb.DataType_BFloat16Vector, - FieldName: columnName, - } - _, err := FieldDataVector(fd) - assert.Error(t, err) - }) -} diff --git a/client/column/vector_test.go b/client/column/vector_test.go new file mode 100644 index 0000000000000..6dfcae77b0e26 --- /dev/null +++ b/client/column/vector_test.go @@ -0,0 +1,167 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package column + +import ( + "fmt" + "math" + "math/rand" + "testing" + + "github.com/samber/lo" + "github.com/stretchr/testify/suite" + + "github.com/milvus-io/milvus/client/v2/entity" +) + +type VectorSuite struct { + suite.Suite +} + +func (s *VectorSuite) TestBasic() { + s.Run("float_vector", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + n := 3 + dim := rand.Intn(10) + 2 + data := make([][]float32, 0, n) + for i := 0; i < n; i++ { + row := lo.RepeatBy(dim, func(i int) float32 { + return rand.Float32() + }) + data = append(data, row) + } + column := NewColumnFloatVector(name, dim, data) + s.Equal(entity.FieldTypeFloatVector, column.Type()) + s.Equal(name, column.Name()) + s.Equal(lo.Map(data, func(row []float32, _ int) entity.FloatVector { return entity.FloatVector(row) }), column.Data()) + s.Equal(dim, column.Dim()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(lo.Flatten(data), fd.GetVectors().GetFloatVector().GetData()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnFloatVector) + if s.True(ok) { + s.Equal(entity.FieldTypeFloatVector, parsed.Type()) + s.Equal(name, parsed.Name()) + s.Equal(lo.Map(data, func(row []float32, _ int) entity.FloatVector { return entity.FloatVector(row) }), parsed.Data()) + s.Equal(dim, parsed.Dim()) + } + }) + + s.Run("binary_vector", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + n := 3 + dim := (rand.Intn(10) + 1) * 8 + data := make([][]byte, 0, n) + for i := 0; i < n; i++ { + row := lo.RepeatBy(dim/8, func(i int) byte { + return byte(rand.Intn(math.MaxUint8)) + }) + data = append(data, row) + } + column := NewColumnBinaryVector(name, dim, data) + s.Equal(entity.FieldTypeBinaryVector, column.Type()) + s.Equal(name, column.Name()) + s.Equal(lo.Map(data, func(row []byte, _ int) entity.BinaryVector { return entity.BinaryVector(row) }), column.Data()) + s.Equal(dim, column.Dim()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(lo.Flatten(data), fd.GetVectors().GetBinaryVector()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnBinaryVector) + if s.True(ok) { + s.Equal(entity.FieldTypeBinaryVector, parsed.Type()) + s.Equal(name, parsed.Name()) + s.Equal(lo.Map(data, func(row []byte, _ int) entity.BinaryVector { return entity.BinaryVector(row) }), parsed.Data()) + s.Equal(dim, parsed.Dim()) + } + }) + + s.Run("fp16_vector", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + n := 3 + dim := rand.Intn(10) + 1 + data := make([][]byte, 0, n) + for i := 0; i < n; i++ { + row := lo.RepeatBy(dim*2, func(i int) byte { + return byte(rand.Intn(math.MaxUint8)) + }) + data = append(data, row) + } + column := NewColumnFloat16Vector(name, dim, data) + s.Equal(entity.FieldTypeFloat16Vector, column.Type()) + s.Equal(name, column.Name()) + s.Equal(lo.Map(data, func(row []byte, _ int) entity.Float16Vector { return entity.Float16Vector(row) }), column.Data()) + s.Equal(dim, column.Dim()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(lo.Flatten(data), fd.GetVectors().GetFloat16Vector()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnFloat16Vector) + if s.True(ok) { + s.Equal(entity.FieldTypeFloat16Vector, parsed.Type()) + s.Equal(name, parsed.Name()) + s.Equal(lo.Map(data, func(row []byte, _ int) entity.Float16Vector { return entity.Float16Vector(row) }), parsed.Data()) + s.Equal(dim, parsed.Dim()) + } + }) + + s.Run("bf16_vector", func() { + name := fmt.Sprintf("field_%d", rand.Intn(1000)) + n := 3 + dim := rand.Intn(10) + 1 + data := make([][]byte, 0, n) + for i := 0; i < n; i++ { + row := lo.RepeatBy(dim*2, func(i int) byte { + return byte(rand.Intn(math.MaxUint8)) + }) + data = append(data, row) + } + column := NewColumnBFloat16Vector(name, dim, data) + s.Equal(entity.FieldTypeBFloat16Vector, column.Type()) + s.Equal(name, column.Name()) + s.Equal(lo.Map(data, func(row []byte, _ int) entity.BFloat16Vector { return entity.BFloat16Vector(row) }), column.Data()) + s.Equal(dim, column.Dim()) + + fd := column.FieldData() + s.Equal(name, fd.GetFieldName()) + s.Equal(lo.Flatten(data), fd.GetVectors().GetBfloat16Vector()) + + result, err := FieldDataColumn(fd, 0, -1) + s.NoError(err) + parsed, ok := result.(*ColumnBFloat16Vector) + if s.True(ok) { + s.Equal(entity.FieldTypeBFloat16Vector, parsed.Type()) + s.Equal(name, parsed.Name()) + s.Equal(lo.Map(data, func(row []byte, _ int) entity.BFloat16Vector { return entity.BFloat16Vector(row) }), parsed.Data()) + s.Equal(dim, parsed.Dim()) + } + }) +} + +func TestVectors(t *testing.T) { + suite.Run(t, new(VectorSuite)) +} diff --git a/tests/go_client/testcases/helper/data_helper.go b/tests/go_client/testcases/helper/data_helper.go index c6ed87ba383a6..aa3a0daf41073 100644 --- a/tests/go_client/testcases/helper/data_helper.go +++ b/tests/go_client/testcases/helper/data_helper.go @@ -179,13 +179,13 @@ func GenArrayColumnData(nb int, eleType entity.FieldType, option GenDataOption) } return column.NewColumnDoubleArray(fieldName, doubleValues) case entity.FieldTypeVarChar: - varcharValues := make([][][]byte, 0, nb) + varcharValues := make([][]string, 0, nb) for i := start; i < start+nb; i++ { - varcharArray := make([][]byte, 0, capacity) + varcharArray := make([]string, 0, capacity) for j := 0; j < capacity; j++ { var buf bytes.Buffer buf.WriteString(strconv.Itoa(i + j)) - varcharArray = append(varcharArray, buf.Bytes()) + varcharArray = append(varcharArray, buf.String()) } varcharValues = append(varcharValues, varcharArray) }