Implement agent handler indexInfo test case (#1708)

* draft index info test cases Signed-off-by: kevindiu <[email protected]> * imple info test Signed-off-by: kevindiu <[email protected]> * update info test Signed-off-by: kevindiu <[email protected]> * add generate remove request impl Signed-off-by: kevindiu <[email protected]> * impl index info test Signed-off-by: kevindiu <[email protected]> * try impl ngt indexing from backup files Signed-off-by: kevindiu <[email protected]> * fix info test Signed-off-by: kevindiu <[email protected]> * remove index file Signed-off-by: kevindiu <[email protected]> * fix comment and implement gen remove req test Signed-off-by: kevindiu <[email protected]> * Apply suggestions from code review * restore backup file in test Signed-off-by: kevindiu <[email protected]> * Update internal/test/data/request/remove_test.go * fix comments Signed-off-by: kevindiu <[email protected]> * refactor Signed-off-by: kevindiu <[email protected]> * refactor Signed-off-by: kevindiu <[email protected]> * fix comment Signed-off-by: kevindiu <[email protected]> * fix comment Signed-off-by: kevindiu <[email protected]>
vdaas · Jul 21, 2022 · ed23300 · ed23300
1 parent 003f80d
commit ed23300
Show file tree

Hide file tree

Showing 12 changed files with 692 additions and 70 deletions.
diff --git a/internal/test/data/backup/100index/grp b/internal/test/data/backup/100index/grp
diff --git a/internal/test/data/backup/100index/metadata.json b/internal/test/data/backup/100index/metadata.json
@@ -0,0 +1 @@
+{"is_invalid":false,"ngt":{"index_count":100}}
diff --git a/internal/test/data/backup/100index/ngt-meta.kvsdb b/internal/test/data/backup/100index/ngt-meta.kvsdb
diff --git a/internal/test/data/backup/100index/obj b/internal/test/data/backup/100index/obj
diff --git a/internal/test/data/backup/100index/prf b/internal/test/data/backup/100index/prf
@@ -0,0 +1,26 @@
+AccuracyTable	
+BatchSizeForCreation	200
+BuildTimeLimit	0
+DatabaseType	Memory
+Dimension	784
+DistanceType	L2
+DynamicEdgeSizeBase	30
+DynamicEdgeSizeRate	20
+EdgeSizeForCreation	20
+EdgeSizeForSearch	10
+EdgeSizeLimitForCreation	5
+EpsilonForCreation	0.1
+GraphType	ANNG
+IncomingEdge	80
+IncrimentalEdgeSizeLimitForTruncation	0
+IndexType	GraphAndTree
+ObjectAlignment	False
+ObjectType	Float-4
+OutgoingEdge	10
+PathAdjustmentInterval	0
+PrefetchOffset	1
+PrefetchSize	3136
+SeedSize	10
+SeedType	None
+ThreadPoolSize	32
+TruncationThreadPoolSize	8
diff --git a/internal/test/data/backup/100index/tre b/internal/test/data/backup/100index/tre
diff --git a/internal/test/data/request/insert.go b/internal/test/data/request/insert.go
@@ -58,7 +58,7 @@ func GenMultiInsertReq(t ObjectType, dist vector.Distribution, num int, dim int,
 	return req, nil
 }
 
-// generate MultiInsert request with the same vector
+// GenSameVecMultiInsertReq generates Insert_MultiRequest with the same vector.
 func GenSameVecMultiInsertReq(num int, vec []float32, cfg *payload.Insert_Config) *payload.Insert_MultiRequest {
 	req := &payload.Insert_MultiRequest{
 		Requests: make([]*payload.Insert_Request, num),

diff --git a/internal/test/data/request/remove.go b/internal/test/data/request/remove.go
@@ -0,0 +1,39 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package request
+
+import (
+	"strconv"
+
+	"github.com/vdaas/vald/apis/grpc/v1/payload"
+)
+
+// GenMultiRemoveReq generates Remove_MultiRequest request.
+func GenMultiRemoveReq(num int, cfg *payload.Remove_Config) *payload.Remove_MultiRequest {
+	req := &payload.Remove_MultiRequest{
+		Requests: make([]*payload.Remove_Request, num),
+	}
+	for i := 0; i < num; i++ {
+		req.Requests[i] = &payload.Remove_Request{
+			Id: &payload.Object_ID{
+				Id: "uuid-" + strconv.Itoa(i+1),
+			},
+			Config: cfg,
+		}
+	}
+
+	return req
+}
diff --git a/internal/test/data/request/remove_test.go b/internal/test/data/request/remove_test.go
@@ -0,0 +1,170 @@
+//
+// Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package request
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/vdaas/vald/apis/grpc/v1/payload"
+	"github.com/vdaas/vald/internal/errors"
+	"github.com/vdaas/vald/internal/test/goleak"
+)
+
+func TestGenMultiRemoveReq(t *testing.T) {
+	type args struct {
+		num int
+		cfg *payload.Remove_Config
+	}
+	type want struct {
+		want *payload.Remove_MultiRequest
+	}
+	type test struct {
+		name       string
+		args       args
+		want       want
+		checkFunc  func(want, *payload.Remove_MultiRequest) error
+		beforeFunc func(args)
+		afterFunc  func(args)
+	}
+	defaultCheckFunc := func(w want, got *payload.Remove_MultiRequest) error {
+		if !reflect.DeepEqual(got, w.want) {
+			return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want)
+		}
+		return nil
+	}
+	defaultRemoveCfg := &payload.Remove_Config{
+		SkipStrictExistCheck: true,
+	}
+	tests := []test{
+		{
+			name: "success to generate 1 remove request",
+			args: args{
+				num: 1,
+				cfg: defaultRemoveCfg,
+			},
+			want: want{
+				want: &payload.Remove_MultiRequest{
+					Requests: []*payload.Remove_Request{
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-1",
+							},
+							Config: defaultRemoveCfg,
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "success to generate 5 remove request",
+			args: args{
+				num: 5,
+				cfg: defaultRemoveCfg,
+			},
+			want: want{
+				want: &payload.Remove_MultiRequest{
+					Requests: []*payload.Remove_Request{
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-1",
+							},
+							Config: defaultRemoveCfg,
+						},
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-2",
+							},
+							Config: defaultRemoveCfg,
+						},
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-3",
+							},
+							Config: defaultRemoveCfg,
+						},
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-4",
+							},
+							Config: defaultRemoveCfg,
+						},
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-5",
+							},
+							Config: defaultRemoveCfg,
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "success to generate 1 remove request with cfg is nil",
+			args: args{
+				num: 1,
+				cfg: nil,
+			},
+			want: want{
+				want: &payload.Remove_MultiRequest{
+					Requests: []*payload.Remove_Request{
+						{
+							Id: &payload.Object_ID{
+								Id: "uuid-1",
+							},
+							Config: nil,
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "success to generate 0 remove request",
+			args: args{
+				num: 0,
+				cfg: nil,
+			},
+			want: want{
+				want: &payload.Remove_MultiRequest{
+					Requests: []*payload.Remove_Request{},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		test := tc
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
+			if test.beforeFunc != nil {
+				test.beforeFunc(test.args)
+			}
+			if test.afterFunc != nil {
+				defer test.afterFunc(test.args)
+			}
+			checkFunc := test.checkFunc
+			if test.checkFunc == nil {
+				checkFunc = defaultCheckFunc
+			}
+
+			got := GenMultiRemoveReq(test.args.num, test.args.cfg)
+			if err := checkFunc(test.want, got); err != nil {
+				tt.Errorf("error = %v", err)
+			}
+		})
+	}
+}
diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go
@@ -19,18 +19,27 @@ package grpc
 
 import (
 	"context"
+	"os"
 	"reflect"
 	"testing"
 
 	"github.com/vdaas/vald/apis/grpc/v1/payload"
 	"github.com/vdaas/vald/internal/config"
 	"github.com/vdaas/vald/internal/errgroup"
 	"github.com/vdaas/vald/internal/errors"
+	"github.com/vdaas/vald/internal/info"
+	"github.com/vdaas/vald/internal/log"
+	"github.com/vdaas/vald/internal/log/logger"
 	"github.com/vdaas/vald/internal/test/data/request"
 	"github.com/vdaas/vald/internal/test/data/vector"
 	"github.com/vdaas/vald/internal/test/goleak"
 	"github.com/vdaas/vald/pkg/agent/core/ngt/service"
 )
+func TestMain(m *testing.M) {
+	log.Init(log.WithLoggerType(logger.NOP.String()))
+	info.Init("")
+	os.Exit(m.Run())
+}
 
 func buildIndex(ctx context.Context, t request.ObjectType, dist vector.Distribution, num int, insertCfg *payload.Insert_Config,
 	ngtCfg *config.NGT, ngtOpts []service.Option, overwriteIDs []string, overwriteVectors [][]float32,