Skip to content

Commit

Permalink
fix: pass static-check
Browse files Browse the repository at this point in the history
Signed-off-by: OxalisCu <[email protected]>
  • Loading branch information
OxalisCu committed Aug 20, 2024
1 parent 71f0341 commit 7ce54a2
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 19 deletions.
5 changes: 3 additions & 2 deletions internal/util/importutilv2/csv/reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ import (
"fmt"
"io"

"go.uber.org/atomic"
"go.uber.org/zap"

"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/merr"
"go.uber.org/atomic"
"go.uber.org/zap"
)

type Row = map[storage.FieldID]any
Expand Down
7 changes: 4 additions & 3 deletions internal/util/importutilv2/csv/reader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@ import (
"os"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/testutil"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)

type ReaderSuite struct {
Expand Down Expand Up @@ -100,7 +101,7 @@ func (suite *ReaderSuite) run(dataType schemapb.DataType, elemType schemapb.Data
cm, err := f.NewPersistentStorageChunkManager(ctx)
suite.NoError(err)

// check reader seperate fields by '\t'
// check reader separate fields by '\t'
wrongSep := ','
_, err = NewReader(ctx, cm, schema, filePath, 64*1024*1024, wrongSep)
suite.Error(err)
Expand Down
17 changes: 6 additions & 11 deletions internal/util/importutilv2/csv/row_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ import (
"strconv"
"strings"

"github.com/samber/lo"

"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/samber/lo"
)

type RowParser interface {
Expand Down Expand Up @@ -186,7 +187,7 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string) (any, e
if err != nil {
return 0, r.wrapTypeError(obj, field)
}
return int64(num), nil
return num, nil
case schemapb.DataType_Float:
num, err := strconv.ParseFloat(obj, 32)
if err != nil {
Expand All @@ -198,7 +199,7 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string) (any, e
if err != nil {
return 0, r.wrapTypeError(obj, field)
}
return float64(num), nil
return num, nil
case schemapb.DataType_VarChar, schemapb.DataType_String:
return obj, nil
case schemapb.DataType_BinaryVector:
Expand All @@ -210,9 +211,6 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string) (any, e
if len(vec) != r.name2Dim[field.GetName()]/8 {
return nil, r.wrapDimError(len(vec)*8, field)
}
for i := 0; i < len(vec); i++ {
vec[i] = byte(vec[i])
}
return vec, nil
case schemapb.DataType_JSON:
var data interface{}
Expand All @@ -230,9 +228,6 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string) (any, e
if len(vec) != r.name2Dim[field.GetName()] {
return nil, r.wrapDimError(len(vec), field)
}
for i := 0; i < len(vec); i++ {
vec[i] = float32(vec[i])
}
return vec, nil
case schemapb.DataType_Float16Vector:
var vec []float32
Expand All @@ -245,7 +240,7 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string) (any, e
}
vec2 := make([]byte, len(vec)*2)
for i := 0; i < len(vec); i++ {
copy(vec2[i*2:], typeutil.Float32ToFloat16Bytes(float32(vec[i])))
copy(vec2[i*2:], typeutil.Float32ToFloat16Bytes(vec[i]))
}
return vec2, nil
case schemapb.DataType_BFloat16Vector:
Expand All @@ -259,7 +254,7 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string) (any, e
}
vec2 := make([]byte, len(vec)*2)
for i := 0; i < len(vec); i++ {
copy(vec2[i*2:], typeutil.Float32ToBFloat16Bytes(float32(vec[i])))
copy(vec2[i*2:], typeutil.Float32ToBFloat16Bytes(vec[i]))
}
return vec2, nil
case schemapb.DataType_SparseFloatVector:
Expand Down
3 changes: 2 additions & 1 deletion internal/util/importutilv2/csv/row_parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ import (
"strings"
"testing"

"github.com/stretchr/testify/assert"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/stretchr/testify/assert"
)

func TestNewRowParser_Invalid(t *testing.T) {
Expand Down
5 changes: 3 additions & 2 deletions internal/util/importutilv2/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ import (
"strconv"
"strings"

"github.com/samber/lo"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/samber/lo"
)

const (
Expand Down Expand Up @@ -108,7 +109,7 @@ func GetCSVSep(options Options) (rune, error) {
if err != nil || len(sep) == 0 {
return defaultSep, nil
} else if lo.Contains(unsupportedSep, []rune(sep)[0]) {
return 0, merr.WrapErrImportFailed(fmt.Sprintf("unsupported csv seperator: %s", sep))
return 0, merr.WrapErrImportFailed(fmt.Sprintf("unsupported csv separator: %s", sep))
}
return []rune(sep)[0], nil
}

0 comments on commit 7ce54a2

Please sign in to comment.