From f813fb45637633f61ba73ace5d1a0acabd815897 Mon Sep 17 00:00:00 2001 From: congqixia Date: Thu, 7 Nov 2024 10:50:25 +0800 Subject: [PATCH] enhance: [GoSDK] Remove example in main pkg to make ut script work (#37472) Related to #31293 Example with main func/package causes `go list` command failed with vcs error. This PR removes example files with main package. The example snippet shall be added back in runnable test format. --------- Signed-off-by: Congqi Xia --- client/column/columns_test.go | 4 +- client/example/database/main.go | 89 -------- client/example/playground/main.go | 326 ------------------------------ client/example/rowbase/main.go | 87 -------- scripts/run_go_codecov.sh | 4 +- 5 files changed, 4 insertions(+), 506 deletions(-) delete mode 100644 client/example/database/main.go delete mode 100644 client/example/playground/main.go delete mode 100644 client/example/rowbase/main.go diff --git a/client/column/columns_test.go b/client/column/columns_test.go index 1a4b3f1605bf5..bba30794bd446 100644 --- a/client/column/columns_test.go +++ b/client/column/columns_test.go @@ -40,9 +40,9 @@ func TestIDColumns(t *testing.T) { t.Run("nil id", func(t *testing.T) { _, err := IDColumns(intPKCol, nil, 0, -1) - assert.Error(t, err) + assert.NoError(t, err) _, err = IDColumns(strPKCol, nil, 0, -1) - assert.Error(t, err) + assert.NoError(t, err) idField := &schemapb.IDs{} col, err := IDColumns(intPKCol, idField, 0, -1) diff --git a/client/example/database/main.go b/client/example/database/main.go deleted file mode 100644 index 0069923d9a2c6..0000000000000 --- a/client/example/database/main.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "context" - "log" - - milvusclient "github.com/milvus-io/milvus/client/v2" - "github.com/milvus-io/milvus/client/v2/entity" -) - -const ( - milvusAddr = `localhost:19530` - nEntities, dim = 3000, 128 - collectionName = "hello_milvus" - - msgFmt = "==== %s ====\n" - idCol, randomCol, embeddingCol = "ID", "random", "embeddings" - topK = 3 -) - -func main() { - ctx := context.Background() - - log.Printf(msgFmt, "start connecting to Milvus") - c, err := milvusclient.New(ctx, &milvusclient.ClientConfig{ - Address: milvusAddr, - }) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - defer c.Close(ctx) - - dbNames, err := c.ListDatabase(ctx, milvusclient.NewListDatabaseOption()) - if err != nil { - log.Fatal("failed to list databases", err.Error()) - } - log.Println("=== Databases: ", dbNames) - - schema := entity.NewSchema().WithName("hello_milvus"). - WithField(entity.NewField().WithName("ID").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)). - WithField(entity.NewField().WithName("Vector").WithDataType(entity.FieldTypeFloatVector).WithDim(128)) - - if err := c.CreateCollection(ctx, milvusclient.NewCreateCollectionOption("hello_milvus", schema)); err != nil { - log.Fatal("failed to create collection:", err.Error()) - } - - collections, err := c.ListCollections(ctx, milvusclient.NewListCollectionOption()) - if err != nil { - log.Fatal("failed to list collections,", err.Error()) - } - - for _, collectionName := range collections { - collection, err := c.DescribeCollection(ctx, milvusclient.NewDescribeCollectionOption(collectionName)) - if err != nil { - log.Fatal(err.Error()) - } - log.Println(collection.Name) - for _, field := range collection.Schema.Fields { - log.Println("=== Field: ", field.Name, field.DataType, field.AutoID) - } - } - - c.CreateDatabase(ctx, milvusclient.NewCreateDatabaseOption("test")) - c.UsingDatabase(ctx, milvusclient.NewUsingDatabaseOption("test")) - - schema = entity.NewSchema().WithName("hello_milvus"). - WithField(entity.NewField().WithName("ID").WithDataType(entity.FieldTypeVarChar).WithMaxLength(64).WithIsPrimaryKey(true)). - WithField(entity.NewField().WithName("Vector").WithDataType(entity.FieldTypeFloatVector).WithDim(128)) - - if err := c.CreateCollection(ctx, milvusclient.NewCreateCollectionOption("hello_milvus", schema)); err != nil { - log.Fatal("failed to create collection:", err.Error()) - } - - collections, err = c.ListCollections(ctx, milvusclient.NewListCollectionOption()) - if err != nil { - log.Fatal("failed to list collections,", err.Error()) - } - - for _, collectionName := range collections { - collection, err := c.DescribeCollection(ctx, milvusclient.NewDescribeCollectionOption(collectionName)) - if err != nil { - log.Fatal(err.Error()) - } - log.Println(collection.Name) - for _, field := range collection.Schema.Fields { - log.Println("=== Field: ", field.Name, field.DataType, field.AutoID) - } - } -} diff --git a/client/example/playground/main.go b/client/example/playground/main.go deleted file mode 100644 index 10b3256bcdfe5..0000000000000 --- a/client/example/playground/main.go +++ /dev/null @@ -1,326 +0,0 @@ -package main - -import ( - "context" - "flag" - "log" - "math/rand" - "time" - - milvusclient "github.com/milvus-io/milvus/client/v2" - "github.com/milvus-io/milvus/client/v2/entity" - "github.com/milvus-io/milvus/client/v2/index" -) - -var cmd = flag.String("cmd", helloMilvusCmd, "command to run") - -const ( - helloMilvusCmd = `hello_milvus` - partitionsCmd = `partitions` - indexCmd = `indexes` - countCmd = `count` - - milvusAddr = `localhost:19530` - nEntities, dim = 3000, 128 - collectionName = "hello_milvus" - - msgFmt = "==== %s ====\n" - idCol, randomCol, embeddingCol = "ID", "random", "embeddings" - topK = 3 -) - -func main() { - flag.Parse() - - switch *cmd { - case helloMilvusCmd: - HelloMilvus() - case partitionsCmd: - Partitions() - case indexCmd: - Indexes() - case countCmd: - Count() - } -} - -func Count() { - ctx := context.Background() - - collectionName := "hello_count_inverted" - - c, err := milvusclient.New(ctx, &milvusclient.ClientConfig{ - Address: "127.0.0.1:19530", - }) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - - schema := entity.NewSchema().WithName(collectionName). - WithField(entity.NewField().WithName("id").WithDataType(entity.FieldTypeInt64).WithIsAutoID(true).WithIsPrimaryKey(true)). - WithField(entity.NewField().WithName("vector").WithDataType(entity.FieldTypeFloatVector).WithDim(128)) - - err = c.CreateCollection(ctx, milvusclient.NewCreateCollectionOption(collectionName, schema)) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - - indexTask, err := c.CreateIndex(ctx, milvusclient.NewCreateIndexOption(collectionName, "id", index.NewGenericIndex("inverted", map[string]string{}))) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - - indexTask.Await(ctx) - - indexTask, err = c.CreateIndex(ctx, milvusclient.NewCreateIndexOption(collectionName, "vector", index.NewHNSWIndex(entity.L2, 16, 32))) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - - indexTask.Await(ctx) - - loadTask, err := c.LoadCollection(ctx, milvusclient.NewLoadCollectionOption(collectionName)) - if err != nil { - log.Fatal("faied to load collection, err: ", err.Error()) - } - loadTask.Await(ctx) - - for i := 0; i < 100; i++ { - // randomData := make([]int64, 0, nEntities) - vectorData := make([][]float32, 0, nEntities) - // generate data - for i := 0; i < nEntities; i++ { - // randomData = append(randomData, rand.Int63n(1000)) - vec := make([]float32, 0, dim) - for j := 0; j < dim; j++ { - vec = append(vec, rand.Float32()) - } - vectorData = append(vectorData, vec) - } - - _, err = c.Insert(ctx, milvusclient.NewColumnBasedInsertOption(collectionName).WithFloatVectorColumn("vector", dim, vectorData)) - if err != nil { - log.Fatal("failed to insert data") - } - - log.Println("start flush collection") - flushTask, err := c.Flush(ctx, milvusclient.NewFlushOption(collectionName)) - if err != nil { - log.Fatal("failed to flush", err.Error()) - } - start := time.Now() - err = flushTask.Await(ctx) - if err != nil { - log.Fatal("failed to flush", err.Error()) - } - log.Println("flush done, elapsed", time.Since(start)) - - result, err := c.Query(ctx, milvusclient.NewQueryOption(collectionName). - WithOutputFields("count(*)"). - WithConsistencyLevel(entity.ClStrong)) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - for _, rs := range result.Fields { - log.Println(rs) - } - result, err = c.Query(ctx, milvusclient.NewQueryOption(collectionName). - WithOutputFields("count(*)"). - WithFilter("id > 0"). - WithConsistencyLevel(entity.ClStrong)) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - for _, rs := range result.Fields { - log.Println(rs) - } - } - - err = c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - if err != nil { - log.Fatal("=== Failed to drop collection", err.Error()) - } -} - -func HelloMilvus() { - ctx := context.Background() - - log.Printf(msgFmt, "start connecting to Milvus") - c, err := milvusclient.New(ctx, &milvusclient.ClientConfig{ - Address: milvusAddr, - }) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - defer c.Close(ctx) - - if has, err := c.HasCollection(ctx, milvusclient.NewHasCollectionOption(collectionName)); err != nil { - log.Fatal("failed to check collection exists or not", err.Error()) - } else if has { - c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - } - - err = c.CreateCollection(ctx, milvusclient.SimpleCreateCollectionOptions(collectionName, dim).WithVarcharPK(true, 128)) - if err != nil { - log.Fatal("failed to create collection", err.Error()) - } - - collections, err := c.ListCollections(ctx, milvusclient.NewListCollectionOption()) - if err != nil { - log.Fatal("failed to list collections,", err.Error()) - } - - for _, collectionName := range collections { - collection, err := c.DescribeCollection(ctx, milvusclient.NewDescribeCollectionOption(collectionName)) - if err != nil { - log.Fatal(err.Error()) - } - log.Println(collection.Name) - for _, field := range collection.Schema.Fields { - log.Println("=== Field: ", field.Name, field.DataType, field.AutoID) - } - } - - // randomData := make([]int64, 0, nEntities) - vectorData := make([][]float32, 0, nEntities) - // generate data - for i := 0; i < nEntities; i++ { - // randomData = append(randomData, rand.Int63n(1000)) - vec := make([]float32, 0, dim) - for j := 0; j < dim; j++ { - vec = append(vec, rand.Float32()) - } - vectorData = append(vectorData, vec) - } - - _, err = c.Insert(ctx, milvusclient.NewColumnBasedInsertOption(collectionName).WithFloatVectorColumn("vector", dim, vectorData)) - if err != nil { - log.Fatal("failed to insert data") - } - - log.Println("start flush collection") - flushTask, err := c.Flush(ctx, milvusclient.NewFlushOption(collectionName)) - if err != nil { - log.Fatal("failed to flush", err.Error()) - } - start := time.Now() - err = flushTask.Await(ctx) - if err != nil { - log.Fatal("failed to flush", err.Error()) - } - log.Println("flush done, elapsed", time.Since(start)) - - vec2search := []entity.Vector{ - entity.FloatVector(vectorData[len(vectorData)-2]), - entity.FloatVector(vectorData[len(vectorData)-1]), - } - - resultSets, err := c.Search(ctx, milvusclient.NewSearchOption(collectionName, 3, vec2search).WithConsistencyLevel(entity.ClEventually)) - if err != nil { - log.Fatal("failed to search collection", err.Error()) - } - for _, resultSet := range resultSets { - for i := 0; i < resultSet.ResultCount; i++ { - log.Print(resultSet.IDs.Get(i)) - } - log.Println() - } - - err = c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - if err != nil { - log.Fatal("=== Failed to drop collection", err.Error()) - } -} - -func Partitions() { - ctx := context.Background() - - log.Printf(msgFmt, "start connecting to Milvus") - c, err := milvusclient.New(ctx, &milvusclient.ClientConfig{ - Address: milvusAddr, - }) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - defer c.Close(ctx) - - has, err := c.HasCollection(ctx, milvusclient.NewHasCollectionOption(collectionName)) - if err != nil { - log.Fatal(err) - } - if has { - c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - } - - err = c.CreateCollection(ctx, milvusclient.SimpleCreateCollectionOptions(collectionName, dim)) - if err != nil { - log.Fatal("failed to create collection", err.Error()) - } - - partitions, err := c.ListPartitions(ctx, milvusclient.NewListPartitionOption(collectionName)) - if err != nil { - log.Fatal("failed to create collection", err.Error()) - } - - for _, partitionName := range partitions { - err := c.DropPartition(ctx, milvusclient.NewDropPartitionOption(collectionName, partitionName)) - if err != nil { - log.Println(err.Error()) - } - } - - c.CreatePartition(ctx, milvusclient.NewCreatePartitionOption(collectionName, "new_partition")) - partitions, err = c.ListPartitions(ctx, milvusclient.NewListPartitionOption(collectionName)) - if err != nil { - log.Fatal("failed to create collection", err.Error()) - } - log.Println(partitions) - - err = c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - if err != nil { - log.Fatal("=== Failed to drop collection", err.Error()) - } -} - -func Indexes() { - ctx := context.Background() - - log.Printf(msgFmt, "start connecting to Milvus") - c, err := milvusclient.New(ctx, &milvusclient.ClientConfig{ - Address: milvusAddr, - }) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - defer c.Close(ctx) - - has, err := c.HasCollection(ctx, milvusclient.NewHasCollectionOption(collectionName)) - if err != nil { - log.Fatal(err) - } - if has { - c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - } - - err = c.CreateCollection(ctx, milvusclient.SimpleCreateCollectionOptions(collectionName, dim)) - if err != nil { - log.Fatal("failed to create collection", err.Error()) - } - - index := index.NewHNSWIndex(entity.COSINE, 16, 64) - - createIdxOpt := milvusclient.NewCreateIndexOption(collectionName, "vector", index) - task, err := c.CreateIndex(ctx, createIdxOpt) - if err != nil { - log.Fatal("failed to create index", err.Error()) - } - task.Await(ctx) - - indexes, err := c.ListIndexes(ctx, milvusclient.NewListIndexOption(collectionName)) - if err != nil { - log.Fatal("failed to list indexes", err.Error()) - } - for _, indexName := range indexes { - log.Println(indexName) - } -} diff --git a/client/example/rowbase/main.go b/client/example/rowbase/main.go deleted file mode 100644 index 2b43378bb8071..0000000000000 --- a/client/example/rowbase/main.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "context" - "log" - "math/rand" - - "github.com/samber/lo" - - milvusclient "github.com/milvus-io/milvus/client/v2" - "github.com/milvus-io/milvus/client/v2/row" -) - -type Data struct { - ID int64 `milvus:"name:id;primary_key;auto_id"` - Vector []float32 `milvus:"name:vector;dim:128"` -} - -const ( - milvusAddr = `localhost:19530` - nEntities, dim = 10, 128 - collectionName = "hello_row_base" - - msgFmt = "==== %s ====\n" - idCol, randomCol, embeddingCol = "id", "random", "vector" - topK = 3 -) - -func main() { - schema, err := row.ParseSchema(&Data{}) - if err != nil { - log.Fatal("failed to parse schema from struct", err.Error()) - } - - for _, field := range schema.Fields { - log.Printf("Field name: %s, FieldType %s, IsPrimaryKey: %t", field.Name, field.DataType, field.PrimaryKey) - } - schema.WithName(collectionName) - - ctx := context.Background() - - log.Printf(msgFmt, "start connecting to Milvus") - c, err := milvusclient.New(ctx, &milvusclient.ClientConfig{ - Address: milvusAddr, - }) - if err != nil { - log.Fatal("failed to connect to milvus, err: ", err.Error()) - } - defer c.Close(ctx) - - if has, err := c.HasCollection(ctx, milvusclient.NewHasCollectionOption(collectionName)); err != nil { - log.Fatal("failed to check collection exists or not", err.Error()) - } else if has { - log.Printf("collection %s alread exists, dropping it now\n", collectionName) - c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) - } - - err = c.CreateCollection(ctx, milvusclient.NewCreateCollectionOption(collectionName, schema)) - if err != nil { - log.Fatal("failed to create collection", err.Error()) - } - - var rows []*Data - for i := 0; i < nEntities; i++ { - vec := make([]float32, 0, dim) - for j := 0; j < dim; j++ { - vec = append(vec, rand.Float32()) - } - rows = append(rows, &Data{ - Vector: vec, - }) - } - - insertResult, err := c.Insert(ctx, milvusclient.NewRowBasedInsertOption(collectionName, lo.Map(rows, func(data *Data, _ int) any { - return data - })...)) - if err != nil { - log.Fatal("failed to insert data: ", err.Error()) - } - log.Println(insertResult.IDs) - for _, row := range rows { - // id shall be written back - log.Println(row.ID) - } - - c.DropCollection(ctx, milvusclient.NewDropCollectionOption(collectionName)) -} diff --git a/scripts/run_go_codecov.sh b/scripts/run_go_codecov.sh index 1e8ca87e5f928..57231534341f3 100755 --- a/scripts/run_go_codecov.sh +++ b/scripts/run_go_codecov.sh @@ -73,5 +73,5 @@ endTime=`date +%s` echo "Total time for go unittest:" $(($endTime-$beginTime)) "s" # generate html report -go tool cover -html=./${FILE_COVERAGE_INFO} -o ./${FILE_COVERAGE_HTML} -echo "Generate go coverage report to ${FILE_COVERAGE_HTML}" +# go tool cover -html=./${FILE_COVERAGE_INFO} -o ./${FILE_COVERAGE_HTML} +# echo "Generate go coverage report to ${FILE_COVERAGE_HTML}"