From 130f7310d1fb7e67e4b5de050efefd2552449054 Mon Sep 17 00:00:00 2001 From: kpango Date: Wed, 18 Oct 2023 15:12:16 +0900 Subject: [PATCH 01/10] add core ngt benchmark Signed-off-by: kpango --- .../vald-helm-operator/crds/valdrelease.yaml | 3 + charts/vald/README.md | 1 + charts/vald/values.schema.json | 5 + charts/vald/values.yaml | 3 + cmd/tools/cli/benchmark/core/main.go | 513 ++++++ cmd/tools/cli/benchmark/core/main_test.go | 265 +++ go.mod | 12 +- go.sum | 14 +- internal/client/v1/client/vald/vald_test.go | 222 +++ internal/config/ngt.go | 3 + internal/conv/conv.go | 2 +- internal/conv/conv_test.go | 2 +- internal/core/algorithm/ngt/ngt.go | 59 +- internal/core/algorithm/ngt/ngt_bench_test.go | 164 ++ internal/core/algorithm/ngt/ngt_test.go | 2 +- internal/core/algorithm/ngt/option.go | 12 + internal/db/kvs/bbolt/bbolt_test.go | 332 ++++ internal/errors/errors.go | 2 +- internal/errors/file.go | 2 +- internal/io/copy_test.go | 320 ++++ internal/net/grpc/client.go | 26 +- internal/net/grpc/context.go | 3 +- internal/net/grpc/grpcconns.go | 251 --- internal/net/grpc/grpcconns_test.go | 1458 ----------------- internal/net/grpc/pool/pool.go | 63 +- internal/net/grpc/pool/pool_bench_test.go | 3 +- internal/net/grpc/pool/pool_test.go | 106 +- internal/net/grpc/stream_test.go | 89 + internal/strings/strings.go | 33 +- internal/sync/semaphore/semaphore_test.go | 40 +- .../sync/singleflight/singleflight_test.go | 2 +- k8s/agent/configmap.yaml | 1 + k8s/operator/helm/crds/valdrelease.yaml | 3 + pkg/agent/core/ngt/service/ngt.go | 1 + pkg/agent/core/ngt/service/ngt_test.go | 36 +- .../core/ngt/service/vqueue/queue_test.go | 109 ++ .../filter/handler/grpc/handler_test.go | 174 ++ pkg/gateway/lb/handler/grpc/handler_test.go | 138 ++ versions/PROMETHEUS_STACK_VERSION | 2 +- 39 files changed, 2548 insertions(+), 1928 deletions(-) create mode 100644 cmd/tools/cli/benchmark/core/main.go create mode 100644 cmd/tools/cli/benchmark/core/main_test.go create mode 100644 internal/core/algorithm/ngt/ngt_bench_test.go delete mode 100644 internal/net/grpc/grpcconns.go delete mode 100644 internal/net/grpc/grpcconns_test.go diff --git a/charts/vald-helm-operator/crds/valdrelease.yaml b/charts/vald-helm-operator/crds/valdrelease.yaml index 3854881577..9f36ef7f5d 100644 --- a/charts/vald-helm-operator/crds/valdrelease.yaml +++ b/charts/vald-helm-operator/crds/valdrelease.yaml @@ -250,6 +250,9 @@ spec: type: boolean enable_proactive_gc: type: boolean + error_buffer_limit: + type: integer + minimum: 1 index_path: type: string initial_delay_max_duration: diff --git a/charts/vald/README.md b/charts/vald/README.md index 9cadef374f..a6f0cb53e9 100644 --- a/charts/vald/README.md +++ b/charts/vald/README.md @@ -84,6 +84,7 @@ Run the following command to install the chart, | agent.ngt.enable_copy_on_write | bool | `false` | enable copy on write saving for more stable backup | | agent.ngt.enable_in_memory_mode | bool | `true` | in-memory mode enabled | | agent.ngt.enable_proactive_gc | bool | `false` | enable proactive GC call for reducing heap memory allocation | +| agent.ngt.error_buffer_limit | int | `10` | maximum number of core ngt error buffer pool size limit | | agent.ngt.index_path | string | `""` | path to index data | | agent.ngt.initial_delay_max_duration | string | `"3m"` | maximum duration for initial delay | | agent.ngt.kvsdb.concurrency | int | `6` | kvsdb processing concurrency | diff --git a/charts/vald/values.schema.json b/charts/vald/values.schema.json index 6634a1f68e..46ce24e324 100644 --- a/charts/vald/values.schema.json +++ b/charts/vald/values.schema.json @@ -241,6 +241,11 @@ "type": "boolean", "description": "enable proactive GC call for reducing heap memory allocation" }, + "error_buffer_limit": { + "type": "integer", + "description": "maximum number of core ngt error buffer pool size limit", + "minimum": 1 + }, "index_path": { "type": "string", "description": "path to index data" diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 6d302054fc..9ccabd591b 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -1723,6 +1723,9 @@ agent: # @schema {"name": "agent.ngt.broken_index_history_limit", "type": "integer", "minimum": 0} # agent.ngt.broken_index_history_limit -- maximum number of broken index generations to backup broken_index_history_limit: 0 + # @schema {"name": "agent.ngt.error_buffer_limit", "type": "integer", "minimum": 1} + # agent.ngt.error_buffer_limit -- maximum number of core ngt error buffer pool size limit + error_buffer_limit: 10 # @schema {"name": "agent.sidecar", "type": "object"} sidecar: # @schema {"name": "agent.sidecar.enabled", "type": "boolean"} diff --git a/cmd/tools/cli/benchmark/core/main.go b/cmd/tools/cli/benchmark/core/main.go new file mode 100644 index 0000000000..9a97418b2e --- /dev/null +++ b/cmd/tools/cli/benchmark/core/main.go @@ -0,0 +1,513 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package ngt provides implementation of Go API for https://github.com/yahoojapan/NGT +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "runtime" + "strconv" + "sync" + "time" + + "github.com/vdaas/vald/internal/conv" + "github.com/vdaas/vald/internal/core/algorithm/ngt" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/http/metrics" + "github.com/vdaas/vald/internal/strings" + "gonum.org/v1/hdf5" +) + +func main() { + var ( + buf []byte + err error + lines, fields []string + line string + vmpeak, + vmsize, + vmdata, + vmrss, + vmhwm, + vmstack, + vmswap, + vmexe, + vmlib, + vmlock, + vmpin, + vmpte, + gc, + save, + cls float64 + + pfile = fmt.Sprintf("/proc/%d/status", os.Getpid()) + zero = float64(0.0) + format = "%s\t" + strings.TrimSuffix(strings.Repeat("%.2f\t", 42), "\t") + ) + output := func(header string) { + buf, err = os.ReadFile(pfile) + if err != nil { + log.Fatal(err) + } + lines = strings.Split(conv.Btoa(buf), "\n") + for _, line = range lines { + fields = strings.Fields(line) + + switch { + case strings.HasPrefix(line, "VmPeak"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmpeak = f + } + case strings.HasPrefix(line, "VmSize"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmsize = f + } + case strings.HasPrefix(line, "VmHWM"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmhwm = f + } + case strings.HasPrefix(line, "VmRSS"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmrss = f + } + case strings.HasPrefix(line, "VmData"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmdata = f + } + case strings.HasPrefix(line, "VmStk"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmstack = f + } + case strings.HasPrefix(line, "VmExe"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmexe = f + } + case strings.HasPrefix(line, "VmLck"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmlock = f + } + case strings.HasPrefix(line, "VmLib"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmlib = f + } + case strings.HasPrefix(line, "VmPTE"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmpte = f + } + case strings.HasPrefix(line, "VmSwap"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmswap = f + } + case strings.HasPrefix(line, "VmPin"): + f, err := strconv.ParseFloat(fields[1], 64) + if err == nil { + vmpin = f + } + } + fields = fields[:0:0] + fields = nil + } + buf = buf[:0:0] + buf = nil + lines = lines[:0:0] + lines = nil + switch { + case strings.Contains(header, "gc"): + gc = vmpeak + save = zero + cls = zero + case strings.Contains(header, "save"): + save = vmpeak + gc = zero + cls = zero + case strings.Contains(header, "close"): + cls = vmpeak + gc = zero + save = zero + default: + gc = zero + save = zero + cls = zero + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + metrics := []interface{}{ + header, + vmpeak, + vmsize, + vmdata, + vmrss, + vmhwm, + vmstack, + vmswap, + vmexe, + vmlib, + vmlock, + vmpin, + vmpte, + float64(m.Alloc) / 1024.0, + float64(m.BuckHashSys), + float64(m.Frees), + float64(m.GCSys) / 1024.0, + float64(m.HeapAlloc) / 1024.0, + float64(m.HeapIdle) / 1024.0, + float64(m.HeapInuse) / 1024.0, + float64(m.HeapObjects), + float64(m.HeapReleased) / 1024.0, + float64(m.HeapSys) / 1024.0, + float64(m.HeapIdle-m.HeapReleased) / 1024.0, + float64(m.Lookups), + float64(m.MCacheInuse), + float64(m.MCacheSys), + float64(m.MSpanInuse) / 1024.0, + float64(m.MSpanSys), + float64(m.Mallocs), + float64(m.Mallocs - m.Frees), + float64(m.NextGC), + float64(m.NumForcedGC), + float64(m.NumGC), + float64(m.OtherSys), + float64(m.PauseTotalNs) / 1024.0, + float64(m.StackInuse), + float64(m.StackSys), + float64(m.Sys) / 1024.0, + float64(m.TotalAlloc) / 1024.0, + gc, + save, + cls, + } + log.Infof(format, metrics...) + switch { + case strings.Contains(header, "gc"), + strings.Contains(header, "save"), + strings.Contains(header, "close"): + log.Infof(format, metrics...) + } + } + defer output("end") + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + srv := &http.Server{ + Addr: "0.0.0.0:6060", + Handler: metrics.NewPProfHandler(), + } + go srv.ListenAndServe() + <-ctx.Done() + srv.Shutdown(context.Background()) + }() + + vectors, _, _ := load(os.Getenv("DATA_PATH")) + log.Infof("# of vectors: %v", len(vectors)) + log.Info(strings.Join([]string{ + "Operation", + "VmPeak", + "VmSize", + "VmData", + "VmRSS", + "VmHWM", + "VmStack", + "VmSwap", + "VmEXE", + "VmLib", + "VmLock", + "VmPin", + "VmPTE", + "Alloc", + "BuckHashSys", + "Frees", + "GCSys", + "HeapAlloc", + "HeapIdle", + "HeapInuse", + "HeapObjects", + "HeapReleased", + "HeapSys", + "HeapWillReturn", + "Lookups", + "MCacheInuse", + "MCacheSys", + "MSpanInuse", + "MSpanSys", + "Mallocs", + "LiveObjects", + "NextGC", + "NumForcedGC", + "NumGC", + "OtherSys", + "PauseTotalNs", + "StackInuse", + "StackSys", + "Sys", + "TotalAlloc", + "GC", + "Save", + "Close", + }, "\t")) + output("start") + path, _ := file.MkdirTemp("") + sleep(ctx, time.Second*5, time.Minute*4, func() { + output("waiting for start") + }, func() { + runtime.GC() + output("gc") + time.Sleep(time.Minute) + output("starting") + }) + + ids := make([]uint, len(vectors)) + run(ctx, false, path, len(vectors[0]), vectors, ids, time.Hour*2, output) + sleep(ctx, time.Second*5, time.Minute*4, func() { + output("waiting for next") + }, func() { + runtime.GC() + output("gc") + time.Sleep(time.Minute) + output("starting") + }) + run(ctx, true, path, len(vectors[0]), nil, nil, 0, output) + sleep(ctx, time.Second*5, time.Minute*4, func() { + output("waiting for next") + }, func() { + runtime.GC() + output("gc") + time.Sleep(time.Minute) + output("starting") + }) + run(ctx, true, path, len(vectors[0]), vectors, ids, time.Hour*2, output) + + ids = ids[:0:0] + ids = nil + vectors = vectors[:0:0] + vectors = nil + sleep(ctx, time.Second*5, time.Minute*5, func() { + output("waiting for gc") + }, func() { + runtime.GC() + output("gc") + }) + sleep(ctx, time.Second*5, time.Minute*5, func() { + output("waiting for gc") + }, func() { + runtime.GC() + output("gc") + }) + sleep(ctx, time.Second*5, time.Minute*5, func() { + output("finalizing") + }, func() { + cancel() + wg.Wait() + }) +} + +func run(ctx context.Context, load bool, path string, dim int, vectors [][]float32, ids []uint, dur time.Duration, output func(header string)) { + var n ngt.NGT + if load { + n, _ = ngt.Load( + ngt.WithDimension(dim), + ngt.WithDefaultPoolSize(8), + ngt.WithObjectType(ngt.Float), + ngt.WithDistanceType(ngt.L2), + ) + } else { + n, _ = ngt.New( + ngt.WithDimension(dim), + ngt.WithDefaultPoolSize(8), + ngt.WithObjectType(ngt.Float), + ngt.WithDistanceType(ngt.L2), + ) + } + + if vectors != nil { + var ( + i int + vector []float32 + id uint + err error + ) + if ids == nil { + ids = make([]uint, len(vectors)) + } else if load { + for _, id = range ids { + _ = n.Remove(id) + } + output("remove") + } + sleep(ctx, 0, dur, func() { + for i, vector = range vectors { + id, err = n.Insert(vector) + if err != nil { + log.Fatal(err) + } + ids[i] = id + } + output("insert") + if err = n.CreateIndex(8); err != nil { + log.Fatal(err) + } + output("create index") + for _, id = range ids { + if err = n.Remove(id); err != nil { + log.Fatal(err) + } + } + output("remove") + }, func() { + for _, vector = range vectors { + _, err = n.Insert(vector) + if err != nil { + log.Fatal(err) + } + } + output("insert") + if err = n.CreateIndex(8); err != nil { + log.Fatal(err) + } + output("create index") + if err = n.SaveIndex(); err != nil { + log.Fatal(err) + } + output("save index") + }) + } + sleep(ctx, time.Second*5, time.Minute*10, func() { + output("finalizing") + }, func() { + n.Close() + n = nil + output("close") + }) +} + +func sleep(ctx context.Context, duration, limit time.Duration, fn, efn func()) { + if limit == 0 { + fn() + efn() + return + } + defer efn() + end := time.NewTimer(limit) + defer end.Stop() + if duration == 0 { + for { + select { + case <-ctx.Done(): + return + case <-end.C: + return + default: + fn() + } + } + return + } + ticker := time.NewTicker(duration) + defer ticker.Stop() + for range ticker.C { + select { + case <-ctx.Done(): + return + case <-end.C: + return + default: + fn() + } + } + return +} + +// load function loads training and test vector from hdf file. The size of ids is same to the number of training data. +// Each id, which is an element of ids, will be set a random number. +func load(path string) (train, test [][]float32, err error) { + var f *hdf5.File + f, err = hdf5.OpenFile(path, hdf5.F_ACC_RDONLY) + if err != nil { + return nil, nil, err + } + defer f.Close() + + // readFn function reads vectors of the hierarchy with the given the name. + readFn := func(name string) ([][]float32, error) { + // Opens and returns a named Dataset. + // The returned dataset must be closed by the user when it is no longer needed. + d, err := f.OpenDataset(name) + if err != nil { + return nil, err + } + defer d.Close() + + // Space returns an identifier for a copy of the dataspace for a dataset. + sp := d.Space() + defer sp.Close() + + // SimpleExtentDims returns dataspace dimension size and maximum size. + dims, _, _ := sp.SimpleExtentDims() + row, dim := int(dims[0]), int(dims[1]) + + // Gets the stored vector. All are represented as one-dimensional arrays. + // The type of the slice depends on your dataset. + // For fashion-mnist-784-euclidean.hdf5, the datatype is float32. + vec := make([]float32, sp.SimpleExtentNPoints()) + if err := d.Read(&vec); err != nil { + return nil, err + } + + // Converts a one-dimensional array to a two-dimensional array. + // Use the `dim` variable as a separator. + vecs := make([][]float32, row) + for i := 0; i < row; i++ { + vecs[i] = make([]float32, dim) + for j := 0; j < dim; j++ { + vecs[i][j] = float32(vec[i*dim+j]) + } + } + + return vecs, nil + } + + // Gets vector of `train` hierarchy. + train, err = readFn("train") + if err != nil { + return nil, nil, err + } + + // Gets vector of `test` hierarchy. + test, err = readFn("test") + if err != nil { + return nil, nil, err + } + + return +} diff --git a/cmd/tools/cli/benchmark/core/main_test.go b/cmd/tools/cli/benchmark/core/main_test.go new file mode 100644 index 0000000000..3748eb8a90 --- /dev/null +++ b/cmd/tools/cli/benchmark/core/main_test.go @@ -0,0 +1,265 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package core + +// NOT IMPLEMENTED BELOW +// +// func Test_main(t *testing.T) { +// type want struct { +// } +// type test struct { +// name string +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// main() +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func Test_run(t *testing.T) { +// type args struct { +// dur time.Duration +// output func(header string) +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:nil, +// output:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:nil, +// output:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// run(test.args.dur, test.args.output) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func Test_load(t *testing.T) { +// type args struct { +// path string +// } +// type want struct { +// wantTrain [][]float32 +// wantTest [][]float32 +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, [][]float32, [][]float32, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotTrain [][]float32, gotTest [][]float32, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotTrain, w.wantTrain) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotTrain, w.wantTrain) +// } +// if !reflect.DeepEqual(gotTest, w.wantTest) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotTest, w.wantTest) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotTrain, gotTest, err := load(test.args.path) +// if err := checkFunc(test.want, gotTrain, gotTest, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/go.mod b/go.mod index 8dee173bf2..489dc919ad 100755 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ replace ( cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.11.2 cloud.google.com/go/storage => cloud.google.com/go/storage v1.33.0 cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.2 - code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4 + code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20231017140541-3b893ed0421b contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql v0.1.7 @@ -24,7 +24,7 @@ replace ( github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 github.com/Azure/go-amqp => github.com/Azure/go-amqp v1.0.2 github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.1-0.20230905222633-df94ce56f001+incompatible github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/autorest v0.11.30-0.20230905222633-df94ce56f001 @@ -36,14 +36,14 @@ replace ( github.com/Azure/go-autorest/tracing => github.com/Azure/go-autorest/tracing v0.6.1-0.20230905222633-df94ce56f001 github.com/BurntSushi/toml => github.com/BurntSushi/toml v1.3.2 github.com/DATA-DOG/go-sqlmock => github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/GoogleCloudPlatform/cloudsql-proxy => github.com/GoogleCloudPlatform/cloudsql-proxy v1.33.11 + github.com/GoogleCloudPlatform/cloudsql-proxy => github.com/GoogleCloudPlatform/cloudsql-proxy v1.33.12 github.com/Masterminds/semver/v3 => github.com/Masterminds/semver/v3 v3.2.1 github.com/ajstarks/deck => github.com/ajstarks/deck v0.0.0-20231012031509-f833e437b68a github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate v0.0.0-20231012031509-f833e437b68a github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.26 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.27 github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.2 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.19.0 @@ -207,7 +207,7 @@ replace ( github.com/lucasb-eyer/go-colorful => github.com/lucasb-eyer/go-colorful v1.2.0 github.com/mailru/easyjson => github.com/mailru/easyjson v0.7.7 github.com/mattn/go-colorable => github.com/mattn/go-colorable v0.1.13 - github.com/mattn/go-isatty => github.com/mattn/go-isatty v0.0.19 + github.com/mattn/go-isatty => github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 v1.14.17 github.com/matttproud/golang_protobuf_extensions => github.com/matttproud/golang_protobuf_extensions v1.0.4 github.com/mitchellh/colorstring => github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db @@ -316,7 +316,7 @@ replace ( google.golang.org/genproto => google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b - google.golang.org/grpc => google.golang.org/grpc v1.58.3 + google.golang.org/grpc => google.golang.org/grpc v1.59.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index de5cead015..9e3a360ee5 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ cloud.google.com/go/vpcaccess v1.7.2/go.mod h1:mmg/MnRHv+3e8FJUjeSibVFvQF1cCy2Ms cloud.google.com/go/webrisk v1.9.2/go.mod h1:pY9kfDgAqxUpDBOrG4w8deLfhvJmejKB0qd/5uQIPBc= cloud.google.com/go/websecurityscanner v1.6.2/go.mod h1:7YgjuU5tun7Eg2kpKgGnDuEOXWIrh8x8lWrJT4zfmas= cloud.google.com/go/workflows v1.12.1/go.mod h1:5A95OhD/edtOhQd/O741NSfIMezNTbCwLM1P1tBRGHM= -code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4 h1:9G5F8zgma5v0GdDvNz6iZwwJp3RS/z0SY/aHGfVwvTo= -code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4/go.mod h1:wYHCXH/gI19ujoFVuMkY48qPpPCoHLKBKXPkn67h/Yc= +code.cloudfoundry.org/bytefmt v0.0.0-20231017140541-3b893ed0421b h1:/2OEIBwZAaJ8n8iTXrM4v/+bdyLDTLwcW6RZtkO4+r0= +code.cloudfoundry.org/bytefmt v0.0.0-20231017140541-3b893ed0421b/go.mod h1:CKNYSQxmKcMCNIKoRG5rRR4AIgJMIoK65ya+Z5xHnk4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20221208032759-85de2813cf6b/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= eliasnaur.com/font v0.0.0-20230308162249-dd43949cb42d/go.mod h1:OYVuxibdk9OSLX8vAqydtRPP87PyTFcT9uH3MlEGBQA= @@ -177,8 +177,8 @@ github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/P github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.45.26 h1:PJ2NJNY5N/yeobLYe1Y+xLdavBi67ZI8gvph6ftwVCg= -github.com/aws/aws-sdk-go v1.45.26/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.27 h1:b+zOTPkAG4i2RvqPdHxkJZafmhhVaVHBp4r41Tu4I6U= +github.com/aws/aws-sdk-go v1.45.27/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs= @@ -464,7 +464,7 @@ github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLO github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mandolyte/mdtopdf v1.3.2/go.mod h1:c28Ldk+tVc/y7QQcEcILStS/OFlerdXGGdBUzJQBgEo= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -667,8 +667,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go. google.golang.org/genproto/googleapis/bytestream v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:itlFWGBbEyD32PUeJsTG8h8Wz7iJXfVK4gt1EJ+pAG0= google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/internal/client/v1/client/vald/vald_test.go b/internal/client/v1/client/vald/vald_test.go index 0a3bc6a440..98f17fc28a 100644 --- a/internal/client/v1/client/vald/vald_test.go +++ b/internal/client/v1/client/vald/vald_test.go @@ -3299,6 +3299,119 @@ package vald // } // } // +// func Test_client_RemoveByTimestamp(t *testing.T) { +// type args struct { +// ctx context.Context +// in *payload.Remove_TimestampRequest +// opts []grpc.CallOption +// } +// type fields struct { +// addrs []string +// c grpc.Client +// } +// type want struct { +// wantRes *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// addrs:nil, +// c:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// addrs:nil, +// c:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &client{ +// addrs: test.fields.addrs, +// c: test.fields.c, +// } +// +// gotRes, err := c.RemoveByTimestamp(test.args.ctx, test.args.in, test.args.opts...) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// // func Test_client_GetObject(t *testing.T) { // type args struct { // ctx context.Context @@ -6627,6 +6740,115 @@ package vald // } // } // +// func Test_singleClient_RemoveByTimestamp(t *testing.T) { +// type args struct { +// ctx context.Context +// in *payload.Remove_TimestampRequest +// opts []grpc.CallOption +// } +// type fields struct { +// vc vald.Client +// } +// type want struct { +// wantRes *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// vc:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// in:nil, +// opts:nil, +// }, +// fields: fields { +// vc:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &singleClient{ +// vc: test.fields.vc, +// } +// +// gotRes, err := c.RemoveByTimestamp(test.args.ctx, test.args.in, test.args.opts...) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// // func Test_singleClient_GetObject(t *testing.T) { // type args struct { // ctx context.Context diff --git a/internal/config/ngt.go b/internal/config/ngt.go index f7b777867c..4048d3c7ff 100644 --- a/internal/config/ngt.go +++ b/internal/config/ngt.go @@ -91,6 +91,9 @@ type NGT struct { // BrokenIndexHistoryLimit represents the maximum number of broken index generations that will be backed up BrokenIndexHistoryLimit int `yaml:"broken_index_history_limit" json:"broken_index_history_limit,omitempty"` + + // ErrorBufferLimit represents the maximum number of core ngt error buffer pool size limit + ErrorBufferLimit uint64 `yaml:"error_buffer_limit" json:"error_buffer_limit,omitempty"` } // KVSDB represent the ngt vector bidirectional kv store configuration diff --git a/internal/conv/conv.go b/internal/conv/conv.go index 8a4cc49ae7..6ac71a156c 100644 --- a/internal/conv/conv.go +++ b/internal/conv/conv.go @@ -15,9 +15,9 @@ package conv import ( "io" - "strings" "unsafe" + "github.com/vdaas/vald/internal/strings" "golang.org/x/text/encoding/japanese" "golang.org/x/text/transform" ) diff --git a/internal/conv/conv_test.go b/internal/conv/conv_test.go index 35330ce12b..6aab3fb3d1 100644 --- a/internal/conv/conv_test.go +++ b/internal/conv/conv_test.go @@ -16,11 +16,11 @@ package conv import ( "io" "reflect" - "strings" "testing" "testing/iotest" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/test/goleak" "golang.org/x/text/encoding/japanese" "golang.org/x/text/transform" diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go index f700adba93..557bbb49a7 100644 --- a/internal/core/algorithm/ngt/ngt.go +++ b/internal/core/algorithm/ngt/ngt.go @@ -97,9 +97,11 @@ type ( radius float32 epsilon float32 poolSize uint32 - cnt uint64 + cnt atomic.Uint64 prop C.NGTProperty - epool sync.Pool + epool sync.Pool // NGT error buffer pool + eps atomic.Uint64 // NGT error buffer pool size + epl uint64 // NGT error buffer pool size limit index C.NGTIndex ospace C.NGTObjectSpace mu *sync.RWMutex @@ -276,7 +278,7 @@ func (n *ngt) setup() error { }, } - for i := 0; i < 20; i++ { + for i := uint64(0); i < n.epl; i++ { n.PutErrorBuffer(C.ngt_create_error_object()) } @@ -411,7 +413,7 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi rsize := int(C.ngt_get_result_size(results, ebuf)) if rsize <= 0 { - if atomic.LoadUint64(&n.cnt) == 0 { + if n.cnt.Load() == 0 { n.PutErrorBuffer(ebuf) return nil, errors.ErrSearchResultEmptyButNoDataStored } @@ -476,7 +478,7 @@ func (n *ngt) LinearSearch(ctx context.Context, vec []float32, size int) (result rsize := int(C.ngt_get_result_size(results, ebuf)) if rsize <= 0 { - if atomic.LoadUint64(&n.cnt) == 0 { + if n.cnt.Load() == 0 { n.PutErrorBuffer(ebuf) return nil, errors.ErrSearchResultEmptyButNoDataStored } @@ -510,21 +512,24 @@ func (n *ngt) LinearSearch(ctx context.Context, vec []float32, size int) (result // Insert returns NGT object id. // This only stores not indexing, you must call CreateIndex and SaveIndex. func (n *ngt) Insert(vec []float32) (id uint, err error) { - dim := int(n.dimension) - if len(vec) != dim { - return 0, errors.ErrIncompatibleDimensionSize(len(vec), dim) + if len(vec) != int(n.dimension) { + return 0, errors.ErrIncompatibleDimensionSize(len(vec), int(n.dimension)) } - + dim := C.uint32_t(n.dimension) + cvec := (*C.float)(&vec[0]) ebuf := n.GetErrorBuffer() n.lock(true) - id = uint(C.ngt_insert_index_as_float(n.index, (*C.float)(&vec[0]), C.uint32_t(n.dimension), ebuf)) + oid := C.ngt_insert_index_as_float(n.index, cvec, dim, ebuf) n.unlock(true) + id = uint(oid) + cvec = nil + vec = vec[:0:0] vec = nil if id == 0 { return 0, n.newGoError(ebuf) } n.PutErrorBuffer(ebuf) - atomic.AddUint64(&n.cnt, 1) + n.cnt.Add(1) return id, nil } @@ -676,7 +681,7 @@ func (n *ngt) Remove(id uint) error { } n.PutErrorBuffer(ebuf) - atomic.AddUint64(&n.cnt, ^uint64(0)) + n.cnt.Add(^uint64(0)) return nil } @@ -742,32 +747,30 @@ func (n *ngt) newGoError(ebuf C.NGTError) (err error) { n.PutErrorBuffer(ebuf) return nil } - n.PutErrorBuffer(C.ngt_create_error_object()) + if n.epl == 0 || n.eps.Load() < n.epl { + n.PutErrorBuffer(C.ngt_create_error_object()) + } C.ngt_destroy_error_object(ebuf) return errors.NewNGTError(msg) } -// Close NGT index. -func (n *ngt) Close() { - if n.index != nil { - C.ngt_close_index(n.index) - n.index = nil - n.prop = nil - n.ospace = nil - } -} - func (n *ngt) GetErrorBuffer() (ebuf C.NGTError) { var ok bool ebuf, ok = n.epool.Get().(C.NGTError) if !ok { ebuf = C.ngt_create_error_object() } + n.eps.Add(^uint64(0)) return ebuf } func (n *ngt) PutErrorBuffer(ebuf C.NGTError) { + if n.epl != 0 && n.eps.Load() > n.epl { + C.ngt_destroy_error_object(ebuf) + return + } n.epool.Put(ebuf) + n.eps.Add(1) } func (n *ngt) lock(cLock bool) { @@ -797,3 +800,13 @@ func (n *ngt) rUnlock(cLock bool) { n.cmu.RUnlock() } } + +// Close NGT index. +func (n *ngt) Close() { + if n.index != nil { + C.ngt_close_index(n.index) + n.index = nil + n.prop = nil + n.ospace = nil + } +} diff --git a/internal/core/algorithm/ngt/ngt_bench_test.go b/internal/core/algorithm/ngt/ngt_bench_test.go new file mode 100644 index 0000000000..dc0b8404ba --- /dev/null +++ b/internal/core/algorithm/ngt/ngt_bench_test.go @@ -0,0 +1,164 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package ngt provides implementation of Go API for https://github.com/yahoojapan/NGT +package ngt + +import ( + "fmt" + "os" + "runtime" + "testing" + + "github.com/vdaas/vald/internal/strings" + "gonum.org/v1/hdf5" +) + +var ( + vectors [][]float32 + n NGT + pid int +) + +func init() { + vectors, _, _ = load("sift-128-euclidean.hdf5") + n, _ = New( + WithDimension(len(vectors[0])), + WithDefaultPoolSize(8), + WithObjectType(Float), + WithDistanceType(L2), + ) + pid = os.Getpid() +} + +func BenchmarkNGT(b *testing.B) { + b.Logf("# of vectors: %v", len(vectors)) + output := func(header string) { + status := fmt.Sprintf("/proc/%d/status", pid) + buf, err := os.ReadFile(status) + if err != nil { + b.Fatal(err) + } + var vmpeak, vmrss, vmhwm string + for _, line := range strings.Split(string(buf), "\n") { + switch { + case strings.HasPrefix(line, "VmPeak"): + vmpeak = strings.Fields(line)[1] + case strings.HasPrefix(line, "VmHWM"): + vmhwm = strings.Fields(line)[1] + case strings.HasPrefix(line, "VmRSS"): + vmrss = strings.Fields(line)[1] + } + } + + var m runtime.MemStats + runtime.ReadMemStats(&m) + b.Logf("%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v", header, vmpeak, vmhwm, vmrss, m.Alloc/1024, m.TotalAlloc/1024, m.HeapAlloc/1024, m.HeapSys/1024, m.HeapInuse/1024) + } + b.Logf(" operation\tVmPeak\tVmHWM\tVmRSS\tAlloc\tTotalAlloc\tHeapAlloc\tHeapSys\tHeapInuse") + b.ResetTimer() + output(" start") + defer output(" end") + for N := 0; N < b.N; N++ { + for i := 0; i < 3; i++ { + ids := make([]uint, len(vectors)) + for idx, vector := range vectors { + id, err := n.Insert(vector) + if err != nil { + b.Fatal(err) + } + ids[idx] = id + } + output(" insert") + + if err := n.CreateIndex(8); err != nil { + b.Fatal(err) + } + output("create index") + + for _, id := range ids { + if err := n.Remove(id); err != nil { + b.Fatal(err) + } + } + output(" remove") + } + } +} + +// load function loads training and test vector from hdf file. The size of ids is same to the number of training data. +// Each id, which is an element of ids, will be set a random number. +func load(path string) (train, test [][]float32, err error) { + var f *hdf5.File + f, err = hdf5.OpenFile(path, hdf5.F_ACC_RDONLY) + if err != nil { + return nil, nil, err + } + defer f.Close() + + // readFn function reads vectors of the hierarchy with the given the name. + readFn := func(name string) ([][]float32, error) { + // Opens and returns a named Dataset. + // The returned dataset must be closed by the user when it is no longer needed. + d, err := f.OpenDataset(name) + if err != nil { + return nil, err + } + defer d.Close() + + // Space returns an identifier for a copy of the dataspace for a dataset. + sp := d.Space() + defer sp.Close() + + // SimpleExtentDims returns dataspace dimension size and maximum size. + dims, _, _ := sp.SimpleExtentDims() + row, dim := int(dims[0]), int(dims[1]) + + // Gets the stored vector. All are represented as one-dimensional arrays. + // The type of the slice depends on your dataset. + // For fashion-mnist-784-euclidean.hdf5, the datatype is float32. + vec := make([]float32, sp.SimpleExtentNPoints()) + if err := d.Read(&vec); err != nil { + return nil, err + } + + // Converts a one-dimensional array to a two-dimensional array. + // Use the `dim` variable as a separator. + vecs := make([][]float32, row) + for i := 0; i < row; i++ { + vecs[i] = make([]float32, dim) + for j := 0; j < dim; j++ { + vecs[i][j] = float32(vec[i*dim+j]) + } + } + + return vecs, nil + } + + // Gets vector of `train` hierarchy. + train, err = readFn("train") + if err != nil { + return nil, nil, err + } + + // Gets vector of `test` hierarchy. + test, err = readFn("test") + if err != nil { + return nil, nil, err + } + + return +} diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go index 04c25ac8cc..a90992b5c0 100644 --- a/internal/core/algorithm/ngt/ngt_test.go +++ b/internal/core/algorithm/ngt/ngt_test.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" "reflect" - "strings" "testing" "github.com/vdaas/vald/internal/core/algorithm" @@ -32,6 +31,7 @@ import ( "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/log/logger" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/comparator" "github.com/vdaas/vald/internal/test/goleak" diff --git a/internal/core/algorithm/ngt/option.go b/internal/core/algorithm/ngt/option.go index 4ff074dd5a..29f9f74c10 100644 --- a/internal/core/algorithm/ngt/option.go +++ b/internal/core/algorithm/ngt/option.go @@ -51,6 +51,7 @@ var ( WithObjectType(Float), WithDistanceType(L2), WithBulkInsertChunkSize(100), + WithErrorBufferLimit(10), } ) @@ -315,3 +316,14 @@ func WithDefaultEpsilon(epsilon float32) Option { return nil } } + +// WithErrorBufferLimit represents the option to set the default error buffer pool size limit for NGT. +func WithErrorBufferLimit(limit uint64) Option { + return func(n *ngt) error { + if limit == 0 { + return errors.NewErrInvalidOption("errorBufferLimit", limit) + } + n.epl = limit + return nil + } +} diff --git a/internal/db/kvs/bbolt/bbolt_test.go b/internal/db/kvs/bbolt/bbolt_test.go index cb94504d36..24589ad2dc 100644 --- a/internal/db/kvs/bbolt/bbolt_test.go +++ b/internal/db/kvs/bbolt/bbolt_test.go @@ -211,3 +211,335 @@ func Test_bbolt_AsyncSet(t *testing.T) { } // NOT IMPLEMENTED BELOW +// +// func Test_bbolt_Set(t *testing.T) { +// type args struct { +// key []byte +// val []byte +// } +// type fields struct { +// db *bolt.DB +// file string +// bucket []byte +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// key:nil, +// val:nil, +// }, +// fields: fields { +// db:nil, +// file:"", +// bucket:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// key:nil, +// val:nil, +// }, +// fields: fields { +// db:nil, +// file:"", +// bucket:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// b := &bbolt{ +// db: test.fields.db, +// file: test.fields.file, +// bucket: test.fields.bucket, +// } +// +// err := b.Set(test.args.key, test.args.val) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_bbolt_Get(t *testing.T) { +// type args struct { +// key []byte +// } +// type fields struct { +// db *bolt.DB +// file string +// bucket []byte +// } +// type want struct { +// wantVal []byte +// wantOk bool +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, []byte, bool, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotVal []byte, gotOk bool, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotVal, w.wantVal) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotVal, w.wantVal) +// } +// if !reflect.DeepEqual(gotOk, w.wantOk) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotOk, w.wantOk) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// key:nil, +// }, +// fields: fields { +// db:nil, +// file:"", +// bucket:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// key:nil, +// }, +// fields: fields { +// db:nil, +// file:"", +// bucket:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// b := &bbolt{ +// db: test.fields.db, +// file: test.fields.file, +// bucket: test.fields.bucket, +// } +// +// gotVal, gotOk, err := b.Get(test.args.key) +// if err := checkFunc(test.want, gotVal, gotOk, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_bbolt_Close(t *testing.T) { +// type args struct { +// remove bool +// } +// type fields struct { +// db *bolt.DB +// file string +// bucket []byte +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// remove:false, +// }, +// fields: fields { +// db:nil, +// file:"", +// bucket:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// remove:false, +// }, +// fields: fields { +// db:nil, +// file:"", +// bucket:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// b := &bbolt{ +// db: test.fields.db, +// file: test.fields.file, +// bucket: test.fields.bucket, +// } +// +// err := b.Close(test.args.remove) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 3fd58dc0ec..28780cf817 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -22,8 +22,8 @@ import ( "fmt" "reflect" "runtime" - "strings" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" ) diff --git a/internal/errors/file.go b/internal/errors/file.go index 6517a23b3a..80ee047173 100644 --- a/internal/errors/file.go +++ b/internal/errors/file.go @@ -106,7 +106,7 @@ func fitos(path string, fi os.FileInfo) string { var err error fi, err = os.Stat(path) if err != nil || fi == nil { - return fmt.Sprintf("unknown file info: %v", fi) + return fmt.Sprintf("unknown file info: for %s\t%v", path, fi) } } if fi != nil { diff --git a/internal/io/copy_test.go b/internal/io/copy_test.go index f1cf6bba79..8faf58f649 100644 --- a/internal/io/copy_test.go +++ b/internal/io/copy_test.go @@ -343,3 +343,323 @@ func Test_copier_Copy(t *testing.T) { } // NOT IMPLEMENTED BELOW +// +// func TestCopyBuffer(t *testing.T) { +// type args struct { +// src io.Reader +// buf []byte +// } +// type want struct { +// wantWritten int64 +// wantDst string +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, int64, string, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotWritten int64, gotDst string, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotWritten, w.wantWritten) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotWritten, w.wantWritten) +// } +// if !reflect.DeepEqual(gotDst, w.wantDst) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotDst, w.wantDst) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// src:nil, +// buf:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// src:nil, +// buf:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// dst := &bytes.Buffer{} +// +// gotWritten, err := CopyBuffer(dst, test.args.src, test.args.buf) +// if err := checkFunc(test.want, gotWritten, err, dst.String()); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_copier_CopyBuffer(t *testing.T) { +// type args struct { +// src io.Reader +// buf []byte +// } +// type fields struct { +// bufSize int64 +// } +// type want struct { +// wantWritten int64 +// wantDst string +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int64, string, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotWritten int64, gotDst string, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotWritten, w.wantWritten) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotWritten, w.wantWritten) +// } +// if !reflect.DeepEqual(gotDst, w.wantDst) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotDst, w.wantDst) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// src:nil, +// buf:nil, +// }, +// fields: fields { +// bufSize:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// src:nil, +// buf:nil, +// }, +// fields: fields { +// bufSize:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &copier{ +// bufSize: test.fields.bufSize, +// } +// dst := &bytes.Buffer{} +// +// gotWritten, err := c.CopyBuffer(dst, test.args.src, test.args.buf) +// if err := checkFunc(test.want, gotWritten, err, dst.String()); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_copier_copyBuffer(t *testing.T) { +// type args struct { +// src io.Reader +// buf *bytes.Buffer +// } +// type fields struct { +// bufSize int64 +// } +// type want struct { +// wantWritten int64 +// wantDst string +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int64, string, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotWritten int64, gotDst string, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotWritten, w.wantWritten) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotWritten, w.wantWritten) +// } +// if !reflect.DeepEqual(gotDst, w.wantDst) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotDst, w.wantDst) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// src:nil, +// buf:bytes.Buffer{}, +// }, +// fields: fields { +// bufSize:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// src:nil, +// buf:bytes.Buffer{}, +// }, +// fields: fields { +// bufSize:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &copier{ +// bufSize: test.fields.bufSize, +// } +// dst := &bytes.Buffer{} +// +// gotWritten, err := c.copyBuffer(dst, test.args.src, test.args.buf) +// if err := checkFunc(test.want, gotWritten, err, dst.String()); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/net/grpc/client.go b/internal/net/grpc/client.go index c6c77b5ad9..29729a8516 100644 --- a/internal/net/grpc/client.go +++ b/internal/net/grpc/client.go @@ -95,7 +95,7 @@ type gRPCClient struct { addrs map[string]struct{} poolSize uint64 clientCount uint64 - conns grpcConns + conns sync.Map[string, pool.Conn] hcDur time.Duration prDur time.Duration dialer net.Dialer @@ -216,7 +216,7 @@ func (g *gRPCClient) StartConnectionMonitor(ctx context.Context) (<-chan error, return ctx.Err() case <-prTick.C: if g.enablePoolRebalance { - err = g.conns.Range(func(addr string, p pool.Conn) bool { + err = g.rangeConns(func(addr string, p pool.Conn) bool { // if addr or pool is nil or empty the registration of conns is invalid let's disconnect them if len(addr) == 0 || p == nil { disconnectTargets = append(disconnectTargets, addr) @@ -253,7 +253,7 @@ func (g *gRPCClient) StartConnectionMonitor(ctx context.Context) (<-chan error, }) } case <-hcTick.C: - err = g.conns.Range(func(addr string, p pool.Conn) bool { + err = g.rangeConns(func(addr string, p pool.Conn) bool { // if addr or pool is nil or empty the registration of conns is invalid let's disconnect them if len(addr) == 0 || p == nil { disconnectTargets = append(disconnectTargets, addr) @@ -367,7 +367,7 @@ func (g *gRPCClient) Range(ctx context.Context, if g.conns.Len() == 0 { return errors.ErrGRPCClientConnNotFound("*") } - err = g.conns.Range(func(addr string, p pool.Conn) bool { + err = g.rangeConns(func(addr string, p pool.Conn) bool { ssctx, sspan := trace.StartSpan(sctx, apiName+"/Client.Range/"+addr) defer func() { if sspan != nil { @@ -428,7 +428,7 @@ func (g *gRPCClient) RangeConcurrent(ctx context.Context, if g.conns.Len() == 0 { return errors.ErrGRPCClientConnNotFound("*") } - err = g.conns.Range(func(addr string, p pool.Conn) bool { + err = g.rangeConns(func(addr string, p pool.Conn) bool { eg.Go(safety.RecoverFunc(func() (err error) { ssctx, sspan := trace.StartSpan(egctx, apiName+"/Client.RangeConcurrent/"+addr) defer func() { @@ -644,7 +644,7 @@ func (g *gRPCClient) RoundRobin(ctx context.Context, f func(ctx context.Context, } do := func() (data interface{}, err error) { - cerr := g.conns.Range(func(addr string, p pool.Conn) bool { + cerr := g.rangeConns(func(addr string, p pool.Conn) bool { select { case <-ctx.Done(): err = ctx.Err() @@ -1010,7 +1010,7 @@ func (g *gRPCClient) Disconnect(ctx context.Context, addr string) error { func (g *gRPCClient) ConnectedAddrs() (addrs []string) { addrs = make([]string, 0, g.conns.Len()) - err := g.conns.Range(func(addr string, p pool.Conn) bool { + err := g.rangeConns(func(addr string, p pool.Conn) bool { if p != nil && p.IsHealthy(context.Background()) { addrs = append(addrs, addr) } @@ -1035,3 +1035,15 @@ func (g *gRPCClient) Close(ctx context.Context) (err error) { }) return err } + +func (g *gRPCClient) rangeConns(fn func(addr string, p pool.Conn) bool) error { + var cnt int + g.conns.Range(func(addr string, p pool.Conn) bool { + cnt++ + return fn(addr, p) + }) + if cnt == 0 { + return errors.ErrGRPCClientConnNotFound("*") + } + return nil +} diff --git a/internal/net/grpc/context.go b/internal/net/grpc/context.go index 90e5ee32a6..3ace275dd1 100644 --- a/internal/net/grpc/context.go +++ b/internal/net/grpc/context.go @@ -15,7 +15,8 @@ package grpc import ( "context" - "strings" + + "github.com/vdaas/vald/internal/strings" ) type contextKey string diff --git a/internal/net/grpc/grpcconns.go b/internal/net/grpc/grpcconns.go deleted file mode 100644 index c38be7181e..0000000000 --- a/internal/net/grpc/grpcconns.go +++ /dev/null @@ -1,251 +0,0 @@ -// -// Copyright (C) 2019-2023 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// Package grpc provides generic functionality for grpc -package grpc - -import ( - "sync/atomic" - "unsafe" - - "github.com/vdaas/vald/internal/errors" - "github.com/vdaas/vald/internal/net/grpc/pool" - "github.com/vdaas/vald/internal/sync" -) - -type grpcConns struct { - mu sync.Mutex - read atomic.Pointer[readOnlyGrpcConns] - dirty map[string]*entryGrpcConns - misses int -} - -type readOnlyGrpcConns struct { - m map[string]*entryGrpcConns - amended bool -} - -// skipcq: GSC-G103 -var expungedGrpcConns = unsafe.Pointer(new(pool.Conn)) - -type entryGrpcConns struct { - p unsafe.Pointer -} - -func newEntryGrpcConns(i pool.Conn) *entryGrpcConns { - // skipcq: GSC-G103 - return &entryGrpcConns{p: unsafe.Pointer(&i)} -} - -func (m *grpcConns) load() (read readOnlyGrpcConns) { - r := m.read.Load() - if r != nil { - return *r - } - read = readOnlyGrpcConns{} - old := m.read.Swap(&read) - if old != nil { - m.read.Store(old) - return *old - } - return read -} - -func (m *grpcConns) Load(key string) (value pool.Conn, ok bool) { - read := m.load() - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - read = m.load() - e, ok = read.m[key] - if !ok && read.amended { - e, ok = m.dirty[key] - m.missLocked() - } - m.mu.Unlock() - } - if !ok { - return value, false - } - return e.load() -} - -func (e *entryGrpcConns) load() (value pool.Conn, ok bool) { - p := atomic.LoadPointer(&e.p) - if p == nil || p == expungedGrpcConns { - return value, false - } - return *(*pool.Conn)(p), true -} - -func (m *grpcConns) Store(key string, value pool.Conn) { - read := m.load() - if e, ok := read.m[key]; ok && e.tryStore(&value) { - return - } - - m.mu.Lock() - read = m.load() - if e, ok := read.m[key]; ok { - if e.unexpungeLocked() { - m.dirty[key] = e - } - e.storeLocked(&value) - } else if e, ok := m.dirty[key]; ok { - e.storeLocked(&value) - } else { - if !read.amended { - m.dirtyLocked() - m.read.Store(&readOnlyGrpcConns{m: read.m, amended: true}) - } - m.dirty[key] = newEntryGrpcConns(value) - } - m.mu.Unlock() -} - -func (e *entryGrpcConns) tryStore(i *pool.Conn) bool { - for { - p := atomic.LoadPointer(&e.p) - if p == expungedGrpcConns { - return false - } - // skipcq: GSC-G103 - if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { - return true - } - } -} - -func (e *entryGrpcConns) unexpungeLocked() (wasExpunged bool) { - return atomic.CompareAndSwapPointer(&e.p, expungedGrpcConns, nil) -} - -func (e *entryGrpcConns) storeLocked(i *pool.Conn) { - // skipcq: GSC-G103 - atomic.StorePointer(&e.p, unsafe.Pointer(i)) -} - -func (m *grpcConns) Delete(key string) { - read := m.load() - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - read = m.load() - e, ok = read.m[key] - if !ok && read.amended { - delete(m.dirty, key) - } - m.mu.Unlock() - } - if ok { - e.delete() - } -} - -func (e *entryGrpcConns) delete() (hadValue bool) { - for { - p := atomic.LoadPointer(&e.p) - if p == nil || p == expungedGrpcConns { - return false - } - if atomic.CompareAndSwapPointer(&e.p, p, nil) { - return true - } - } -} - -func (m *grpcConns) Range(f func(key string, value pool.Conn) bool) (err error) { - read := m.load() - if read.amended { - m.mu.Lock() - read = m.load() - if read.amended { - read = readOnlyGrpcConns{m: m.dirty} - m.read.Store(&read) - m.dirty = nil - m.misses = 0 - } - m.mu.Unlock() - } - - var cnt int - for k, e := range read.m { - v, ok := e.load() - if !ok { - continue - } - cnt++ - if !f(k, v) { - return nil - } - } - if cnt == 0 { - return errors.ErrGRPCClientConnNotFound("*") - } - return nil -} - -func (m *grpcConns) Len() int { - read := m.load() - if read.amended { - m.mu.Lock() - read = m.load() - if read.amended { - read = readOnlyGrpcConns{m: m.dirty} - m.read.Store(&read) - m.dirty = nil - m.misses = 0 - } - m.mu.Unlock() - } - - return len(read.m) -} - -func (m *grpcConns) missLocked() { - m.misses++ - if m.misses < len(m.dirty) { - return - } - m.read.Store(&readOnlyGrpcConns{m: m.dirty}) - m.dirty = nil - m.misses = 0 -} - -func (m *grpcConns) dirtyLocked() { - if m.dirty != nil { - return - } - - read := m.load() - m.dirty = make(map[string]*entryGrpcConns, len(read.m)) - for k, e := range read.m { - if !e.tryExpungeLocked() { - m.dirty[k] = e - } - } -} - -func (e *entryGrpcConns) tryExpungeLocked() (isExpunged bool) { - p := atomic.LoadPointer(&e.p) - for p == nil { - if atomic.CompareAndSwapPointer(&e.p, nil, expungedGrpcConns) { - return true - } - p = atomic.LoadPointer(&e.p) - } - return p == expungedGrpcConns -} diff --git a/internal/net/grpc/grpcconns_test.go b/internal/net/grpc/grpcconns_test.go deleted file mode 100644 index 89c5ecc88a..0000000000 --- a/internal/net/grpc/grpcconns_test.go +++ /dev/null @@ -1,1458 +0,0 @@ -// Copyright (C) 2019-2023 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package grpc - -// NOT IMPLEMENTED BELOW -// -// func Test_newEntryGrpcConns(t *testing.T) { -// type args struct { -// i pool.Conn -// } -// type want struct { -// want *entryGrpcConns -// } -// type test struct { -// name string -// args args -// want want -// checkFunc func(want, *entryGrpcConns) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want, got *entryGrpcConns) error { -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// i:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// i:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// -// got := newEntryGrpcConns(test.args.i) -// if err := checkFunc(test.want, got); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_grpcConns_load(t *testing.T) { -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// wantRead readOnlyGrpcConns -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want, readOnlyGrpcConns) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want, gotRead readOnlyGrpcConns) error { -// if !reflect.DeepEqual(gotRead, w.wantRead) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRead, w.wantRead) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// gotRead := m.load() -// if err := checkFunc(test.want, gotRead); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_grpcConns_Load(t *testing.T) { -// type args struct { -// key string -// } -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// wantValue pool.Conn -// wantOk bool -// } -// type test struct { -// name string -// args args -// fields fields -// want want -// checkFunc func(want, pool.Conn, bool) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want, gotValue pool.Conn, gotOk bool) error { -// if !reflect.DeepEqual(gotValue, w.wantValue) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotValue, w.wantValue) -// } -// if !reflect.DeepEqual(gotOk, w.wantOk) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotOk, w.wantOk) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// key:"", -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// key:"", -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// gotValue, gotOk := m.Load(test.args.key) -// if err := checkFunc(test.want, gotValue, gotOk); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_entryGrpcConns_load(t *testing.T) { -// type fields struct { -// p unsafe.Pointer -// } -// type want struct { -// wantValue pool.Conn -// wantOk bool -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want, pool.Conn, bool) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want, gotValue pool.Conn, gotOk bool) error { -// if !reflect.DeepEqual(gotValue, w.wantValue) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotValue, w.wantValue) -// } -// if !reflect.DeepEqual(gotOk, w.wantOk) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotOk, w.wantOk) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// e := &entryGrpcConns{ -// p: test.fields.p, -// } -// -// gotValue, gotOk := e.load() -// if err := checkFunc(test.want, gotValue, gotOk); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_grpcConns_Store(t *testing.T) { -// type args struct { -// key string -// value pool.Conn -// } -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// } -// type test struct { -// name string -// args args -// fields fields -// want want -// checkFunc func(want) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want) error { -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// key:"", -// value:nil, -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// key:"", -// value:nil, -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// m.Store(test.args.key, test.args.value) -// if err := checkFunc(test.want); err != nil { -// tt.Errorf("error = %v", err) -// } -// }) -// } -// } -// -// func Test_entryGrpcConns_tryStore(t *testing.T) { -// type args struct { -// i *pool.Conn -// } -// type fields struct { -// p unsafe.Pointer -// } -// type want struct { -// want bool -// } -// type test struct { -// name string -// args args -// fields fields -// want want -// checkFunc func(want, bool) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want, got bool) error { -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// i:nil, -// }, -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// i:nil, -// }, -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// e := &entryGrpcConns{ -// p: test.fields.p, -// } -// -// got := e.tryStore(test.args.i) -// if err := checkFunc(test.want, got); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_entryGrpcConns_unexpungeLocked(t *testing.T) { -// type fields struct { -// p unsafe.Pointer -// } -// type want struct { -// wantWasExpunged bool -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want, bool) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want, gotWasExpunged bool) error { -// if !reflect.DeepEqual(gotWasExpunged, w.wantWasExpunged) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotWasExpunged, w.wantWasExpunged) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// e := &entryGrpcConns{ -// p: test.fields.p, -// } -// -// gotWasExpunged := e.unexpungeLocked() -// if err := checkFunc(test.want, gotWasExpunged); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_entryGrpcConns_storeLocked(t *testing.T) { -// type args struct { -// i *pool.Conn -// } -// type fields struct { -// p unsafe.Pointer -// } -// type want struct { -// } -// type test struct { -// name string -// args args -// fields fields -// want want -// checkFunc func(want) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want) error { -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// i:nil, -// }, -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// i:nil, -// }, -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// e := &entryGrpcConns{ -// p: test.fields.p, -// } -// -// e.storeLocked(test.args.i) -// if err := checkFunc(test.want); err != nil { -// tt.Errorf("error = %v", err) -// } -// }) -// } -// } -// -// func Test_grpcConns_Delete(t *testing.T) { -// type args struct { -// key string -// } -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// } -// type test struct { -// name string -// args args -// fields fields -// want want -// checkFunc func(want) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want) error { -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// key:"", -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// key:"", -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// m.Delete(test.args.key) -// if err := checkFunc(test.want); err != nil { -// tt.Errorf("error = %v", err) -// } -// }) -// } -// } -// -// func Test_entryGrpcConns_delete(t *testing.T) { -// type fields struct { -// p unsafe.Pointer -// } -// type want struct { -// wantHadValue bool -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want, bool) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want, gotHadValue bool) error { -// if !reflect.DeepEqual(gotHadValue, w.wantHadValue) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotHadValue, w.wantHadValue) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// e := &entryGrpcConns{ -// p: test.fields.p, -// } -// -// gotHadValue := e.delete() -// if err := checkFunc(test.want, gotHadValue); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_grpcConns_Range(t *testing.T) { -// type args struct { -// f func(key string, value pool.Conn) bool -// } -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// err error -// } -// type test struct { -// name string -// args args -// fields fields -// want want -// checkFunc func(want, error) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want, err error) error { -// if !errors.Is(err, w.err) { -// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// f:nil, -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// f:nil, -// }, -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// err := m.Range(test.args.f) -// if err := checkFunc(test.want, err); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_grpcConns_Len(t *testing.T) { -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// want int -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want, int) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want, got int) error { -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// got := m.Len() -// if err := checkFunc(test.want, got); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// -// func Test_grpcConns_missLocked(t *testing.T) { -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want) error { -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// m.missLocked() -// if err := checkFunc(test.want); err != nil { -// tt.Errorf("error = %v", err) -// } -// }) -// } -// } -// -// func Test_grpcConns_dirtyLocked(t *testing.T) { -// type fields struct { -// read atomic.Pointer[readOnlyGrpcConns] -// dirty map[string]*entryGrpcConns -// misses int -// } -// type want struct { -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want) error { -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// read:nil, -// dirty:nil, -// misses:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// m := &grpcConns{ -// read: test.fields.read, -// dirty: test.fields.dirty, -// misses: test.fields.misses, -// } -// -// m.dirtyLocked() -// if err := checkFunc(test.want); err != nil { -// tt.Errorf("error = %v", err) -// } -// }) -// } -// } -// -// func Test_entryGrpcConns_tryExpungeLocked(t *testing.T) { -// type fields struct { -// p unsafe.Pointer -// } -// type want struct { -// wantIsExpunged bool -// } -// type test struct { -// name string -// fields fields -// want want -// checkFunc func(want, bool) error -// beforeFunc func(*testing.T) -// afterFunc func(*testing.T) -// } -// defaultCheckFunc := func(w want, gotIsExpunged bool) error { -// if !reflect.DeepEqual(gotIsExpunged, w.wantIsExpunged) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotIsExpunged, w.wantIsExpunged) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// fields: fields { -// p:nil, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T,) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T,) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// e := &entryGrpcConns{ -// p: test.fields.p, -// } -// -// gotIsExpunged := e.tryExpungeLocked() -// if err := checkFunc(test.want, gotIsExpunged); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } diff --git a/internal/net/grpc/pool/pool.go b/internal/net/grpc/pool/pool.go index 8f195bc2c1..16980f4341 100644 --- a/internal/net/grpc/pool/pool.go +++ b/internal/net/grpc/pool/pool.go @@ -36,7 +36,6 @@ import ( "github.com/vdaas/vald/internal/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials/insecure" ) type ( @@ -95,31 +94,35 @@ func New(ctx context.Context, opts ...Option) (c Conn, err error) { p.init(true) p.closing.Store(false) - var isIPv4, isIPv6 bool + var ( + isIPv4, isIPv6 bool + port uint16 + ) p.host, p.port, _, isIPv4, isIPv6, err = net.Parse(p.addr) p.isIP = isIPv4 || isIPv6 if err != nil { log.Warnf("failed to parse addr %s: %s", p.addr, err) if p.host == "" { var ( - ok bool - port string + ok bool + portStr string ) - p.host, port, ok = strings.Cut(p.addr, ":") + p.host, portStr, ok = strings.Cut(p.addr, ":") if !ok { p.host = p.addr } else { - portNum, err := strconv.ParseUint(port, 10, 16) + portNum, err := strconv.ParseUint(portStr, 10, 16) if err != nil { p.port = uint16(portNum) } } } if p.port == 0 { - err = p.scanGRPCPort(ctx) + port, err = p.scanGRPCPort(ctx) if err != nil { return nil, err } + p.port = port } p.addr = net.JoinHostPort(p.host, p.port) } @@ -133,10 +136,12 @@ func New(ctx context.Context, opts ...Option) (c Conn, err error) { log.Warn("failed to close connection:", err) } } - err = p.scanGRPCPort(ctx) + + port, err := p.scanGRPCPort(ctx) if err != nil { return nil, err } + p.port = port p.addr = net.JoinHostPort(p.host, p.port) conn, err = grpc.DialContext(ctx, p.addr, p.dopts...) if err != nil { @@ -642,23 +647,33 @@ func (p *pool) lookupIPAddr(ctx context.Context) (ips []string, err error) { return ips, nil } -func (p *pool) scanGRPCPort(ctx context.Context) (err error) { +func (p *pool) scanGRPCPort(ctx context.Context) (port uint16, err error) { ports, err := net.ScanPorts(ctx, p.startPort, p.endPort, p.host) if err != nil { - return err + return 0, err } + var conn *ClientConn for _, port := range ports { select { case <-ctx.Done(): - return ctx.Err() + return 0, ctx.Err() default: - if isGRPCPort(ctx, p.host, port) { - p.port = port - return nil + // try gRPC dialing to target port + conn, err = grpc.DialContext(ctx, + net.JoinHostPort(p.host, port), + append(p.dopts, grpc.WithBlock())...) + + if err == nil && isHealthy(conn) && conn.Close() == nil { + // if no error and healthy the port is ready for gRPC + return port, nil + } + + if conn != nil { + _ = conn.Close() } } } - return errors.ErrInvalidGRPCPort(p.addr, p.host, p.port) + return 0, errors.ErrInvalidGRPCPort(p.addr, p.host, p.port) } func (p *pool) IsIPConn() (isIP bool) { @@ -733,22 +748,6 @@ func (pc *poolConn) Close(ctx context.Context, delay time.Duration) error { } } -func isGRPCPort(ctx context.Context, host string, port uint16) bool { - ctx, cancel := context.WithTimeout(ctx, time.Millisecond*5) - defer cancel() - conn, err := grpc.DialContext(ctx, - net.JoinHostPort(host, port), - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock()) - if err != nil { - if conn != nil { - _ = conn.Close() - } - return false - } - return conn.Close() == nil -} - func isHealthy(conn *ClientConn) bool { if conn == nil { log.Warn("gRPC target connection is nil") @@ -762,7 +761,7 @@ func isHealthy(conn *ClientConn) bool { log.Debugf("gRPC target %s's connection status will be Ready soon\tstatus: %s", conn.Target(), state.String()) return true case connectivity.Idle: - log.Debugf("gRPC target %s's connection status is waiting for target\tstatus: %s", conn.Target(), state.String()) + log.Warnf("gRPC target %s's connection status is waiting for target\tstatus: %s", conn.Target(), state.String()) return false case connectivity.Shutdown, connectivity.TransientFailure: log.Errorf("gRPC target %s's connection status is unhealthy\tstatus: %s", conn.Target(), state.String()) diff --git a/internal/net/grpc/pool/pool_bench_test.go b/internal/net/grpc/pool/pool_bench_test.go index 08823c4d5d..f287b64c43 100644 --- a/internal/net/grpc/pool/pool_bench_test.go +++ b/internal/net/grpc/pool/pool_bench_test.go @@ -33,7 +33,7 @@ import ( const ( DefaultServerAddr = "localhost:5001" - DefaultPoolSize = 10 + DefaultPoolSize = 4 ) type server struct { @@ -61,6 +61,7 @@ func (*server) Nodes(context.Context, *payload.Discoverer_Request) (*payload.Inf } func ListenAndServe(b *testing.B, addr string) func() { + b.Helper() lis, err := net.Listen("tcp", addr) if err != nil { b.Error(err) diff --git a/internal/net/grpc/pool/pool_test.go b/internal/net/grpc/pool/pool_test.go index b76a8e9640..a0455869c1 100644 --- a/internal/net/grpc/pool/pool_test.go +++ b/internal/net/grpc/pool/pool_test.go @@ -3533,21 +3533,25 @@ package pool // reconnectHash atomic.Pointer[string] // } // type want struct { -// err error +// wantPort uint16 +// err error // } // type test struct { // name string // args args // fields fields // want want -// checkFunc func(want, error) error +// checkFunc func(want, uint16, error) error // beforeFunc func(*testing.T, args) // afterFunc func(*testing.T, args) // } -// defaultCheckFunc := func(w want, err error) error { +// defaultCheckFunc := func(w want, gotPort uint16, err error) error { // if !errors.Is(err, w.err) { // return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) // } +// if !reflect.DeepEqual(gotPort, w.wantPort) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotPort, w.wantPort) +// } // return nil // } // tests := []test{ @@ -3663,8 +3667,8 @@ package pool // reconnectHash: test.fields.reconnectHash, // } // -// err := p.scanGRPCPort(test.args.ctx) -// if err := checkFunc(test.want, err); err != nil { +// gotPort, err := p.scanGRPCPort(test.args.ctx) +// if err := checkFunc(test.want, gotPort, err); err != nil { // tt.Errorf("error = %v", err) // } // @@ -4084,98 +4088,6 @@ package pool // } // } // -// func Test_isGRPCPort(t *testing.T) { -// type args struct { -// ctx context.Context -// host string -// port uint16 -// } -// type want struct { -// want bool -// } -// type test struct { -// name string -// args args -// want want -// checkFunc func(want, bool) error -// beforeFunc func(*testing.T, args) -// afterFunc func(*testing.T, args) -// } -// defaultCheckFunc := func(w want, got bool) error { -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) -// } -// return nil -// } -// tests := []test{ -// // TODO test cases -// /* -// { -// name: "test_case_1", -// args: args { -// ctx:nil, -// host:"", -// port:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// }, -// */ -// -// // TODO test cases -// /* -// func() test { -// return test { -// name: "test_case_2", -// args: args { -// ctx:nil, -// host:"", -// port:0, -// }, -// want: want{}, -// checkFunc: defaultCheckFunc, -// beforeFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// afterFunc: func(t *testing.T, args args) { -// t.Helper() -// }, -// } -// }(), -// */ -// } -// -// for _, tc := range tests { -// test := tc -// t.Run(test.name, func(tt *testing.T) { -// tt.Parallel() -// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) -// if test.beforeFunc != nil { -// test.beforeFunc(tt, test.args) -// } -// if test.afterFunc != nil { -// defer test.afterFunc(tt, test.args) -// } -// checkFunc := test.checkFunc -// if test.checkFunc == nil { -// checkFunc = defaultCheckFunc -// } -// -// got := isGRPCPort(test.args.ctx, test.args.host, test.args.port) -// if err := checkFunc(test.want, got); err != nil { -// tt.Errorf("error = %v", err) -// } -// -// }) -// } -// } -// // func Test_isHealthy(t *testing.T) { // type args struct { // conn *ClientConn diff --git a/internal/net/grpc/stream_test.go b/internal/net/grpc/stream_test.go index 27438cbdde..c65aebd935 100644 --- a/internal/net/grpc/stream_test.go +++ b/internal/net/grpc/stream_test.go @@ -295,3 +295,92 @@ func TestBidirectionalStream(t *testing.T) { // }) // } // } +// +// func Test_removeDuplicates(t *testing.T) { +// type args struct { +// x S +// less func(left, right E) int +// } +// type want struct { +// want S +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, S) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got S) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// x:nil, +// less:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// x:nil, +// less:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := removeDuplicates(test.args.x, test.args.less) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/strings/strings.go b/internal/strings/strings.go index 55060db866..e5ef6de97e 100644 --- a/internal/strings/strings.go +++ b/internal/strings/strings.go @@ -22,36 +22,63 @@ import ( ) type ( - Builder = strings.Builder + Builder = strings.Builder + Reader = strings.Reader + Replacer = strings.Replacer ) var ( + Clone = strings.Clone + Compare = strings.Compare Contains = strings.Contains + ContainsAny = strings.ContainsAny + ContainsFunc = strings.ContainsFunc + ContainsRune = strings.ContainsRune Count = strings.Count Cut = strings.Cut + CutPrefix = strings.CutPrefix + CutSuffix = strings.CutSuffix EqualFold = strings.EqualFold + Fields = strings.Fields + FieldsFunc = strings.FieldsFunc HasPrefix = strings.HasPrefix HasSuffix = strings.HasSuffix Index = strings.Index IndexAny = strings.IndexAny - NewReader = strings.NewReader - NewReplacer = strings.NewReplacer + IndexByte = strings.IndexByte + IndexFunc = strings.IndexFunc + IndexRune = strings.IndexRune + LastIndex = strings.LastIndex + LastIndexAny = strings.LastIndexAny + LastIndexByte = strings.LastIndexByte + LastIndexFunc = strings.LastIndexFunc + Map = strings.Map + Repeat = strings.Repeat Replace = strings.Replace ReplaceAll = strings.ReplaceAll Split = strings.Split SplitAfter = strings.SplitAfter SplitAfterN = strings.SplitAfterN SplitN = strings.SplitN + Title = strings.Title ToLower = strings.ToLower ToLowerSpecial = strings.ToLowerSpecial + ToTitle = strings.ToTitle + ToTitleSpecial = strings.ToTitleSpecial ToUpper = strings.ToUpper ToUpperSpecial = strings.ToUpperSpecial + ToValidUTF8 = strings.ToValidUTF8 Trim = strings.Trim + TrimFunc = strings.TrimFunc TrimLeft = strings.TrimLeft + TrimLeftFunc = strings.TrimLeftFunc TrimPrefix = strings.TrimPrefix TrimRight = strings.TrimRight + TrimRightFunc = strings.TrimRightFunc TrimSpace = strings.TrimSpace TrimSuffix = strings.TrimSuffix + NewReader = strings.NewReader + NewReplacer = strings.NewReplacer bufferPool = sync.Pool{ New: func() interface{} { diff --git a/internal/sync/semaphore/semaphore_test.go b/internal/sync/semaphore/semaphore_test.go index 6290b6affa..d55037c7f7 100644 --- a/internal/sync/semaphore/semaphore_test.go +++ b/internal/sync/semaphore/semaphore_test.go @@ -322,9 +322,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n int64 // } // type fields struct { +// waiters list.List // size int64 // cur int64 -// waiters list.List // } // type want struct { // err error @@ -354,9 +354,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -379,9 +379,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -412,9 +412,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // checkFunc = defaultCheckFunc // } // s := &Weighted{ +// waiters: test.fields.waiters, // size: test.fields.size, // cur: test.fields.cur, -// waiters: test.fields.waiters, // } // // err := s.Acquire(test.args.ctx, test.args.n) @@ -431,9 +431,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n int64 // } // type fields struct { +// waiters list.List // size int64 // cur int64 -// waiters list.List // } // type want struct { // want bool @@ -462,9 +462,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -486,9 +486,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -519,9 +519,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // checkFunc = defaultCheckFunc // } // s := &Weighted{ +// waiters: test.fields.waiters, // size: test.fields.size, // cur: test.fields.cur, -// waiters: test.fields.waiters, // } // // got := s.TryAcquire(test.args.n) @@ -538,9 +538,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n int64 // } // type fields struct { +// waiters list.List // size int64 // cur int64 -// waiters list.List // } // type want struct { // } @@ -565,9 +565,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -589,9 +589,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -622,9 +622,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // checkFunc = defaultCheckFunc // } // s := &Weighted{ +// waiters: test.fields.waiters, // size: test.fields.size, // cur: test.fields.cur, -// waiters: test.fields.waiters, // } // // s.Release(test.args.n) @@ -640,9 +640,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n int64 // } // type fields struct { +// waiters list.List // size int64 // cur int64 -// waiters list.List // } // type want struct { // } @@ -667,9 +667,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -691,9 +691,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // n:0, // }, // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -724,9 +724,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // checkFunc = defaultCheckFunc // } // s := &Weighted{ +// waiters: test.fields.waiters, // size: test.fields.size, // cur: test.fields.cur, -// waiters: test.fields.waiters, // } // // s.Resize(test.args.n) @@ -739,9 +739,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // // func TestWeighted_notifyWaiters(t *testing.T) { // type fields struct { +// waiters list.List // size int64 // cur int64 -// waiters list.List // } // type want struct { // } @@ -762,9 +762,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // { // name: "test_case_1", // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -783,9 +783,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // return test { // name: "test_case_2", // fields: fields { +// waiters:nil, // size:0, // cur:0, -// waiters:nil, // }, // want: want{}, // checkFunc: defaultCheckFunc, @@ -816,9 +816,9 @@ func TestAllocCancelDoesntStarve(t *testing.T) { // checkFunc = defaultCheckFunc // } // s := &Weighted{ +// waiters: test.fields.waiters, // size: test.fields.size, // cur: test.fields.cur, -// waiters: test.fields.waiters, // } // // s.notifyWaiters() diff --git a/internal/sync/singleflight/singleflight_test.go b/internal/sync/singleflight/singleflight_test.go index ca8f74cfc9..cd8df7f5e6 100644 --- a/internal/sync/singleflight/singleflight_test.go +++ b/internal/sync/singleflight/singleflight_test.go @@ -25,13 +25,13 @@ import ( "os/exec" "reflect" "runtime" - "strings" "sync/atomic" "testing" "time" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/test/goleak" ) diff --git a/k8s/agent/configmap.yaml b/k8s/agent/configmap.yaml index 0e57796161..95c8d5db53 100644 --- a/k8s/agent/configmap.yaml +++ b/k8s/agent/configmap.yaml @@ -215,6 +215,7 @@ data: enable_copy_on_write: false enable_in_memory_mode: true enable_proactive_gc: false + error_buffer_limit: 10 index_path: "" initial_delay_max_duration: 3m kvsdb: diff --git a/k8s/operator/helm/crds/valdrelease.yaml b/k8s/operator/helm/crds/valdrelease.yaml index 3854881577..9f36ef7f5d 100644 --- a/k8s/operator/helm/crds/valdrelease.yaml +++ b/k8s/operator/helm/crds/valdrelease.yaml @@ -250,6 +250,9 @@ spec: type: boolean enable_proactive_gc: type: boolean + error_buffer_limit: + type: integer + minimum: 1 index_path: type: string initial_delay_max_duration: diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index 3b598ceb7e..cdfe518702 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -192,6 +192,7 @@ func New(cfg *config.NGT, opts ...Option) (nn NGT, err error) { core.WithBulkInsertChunkSize(cfg.BulkInsertChunkSize), core.WithCreationEdgeSize(cfg.CreationEdgeSize), core.WithSearchEdgeSize(cfg.SearchEdgeSize), + core.WithErrorBufferLimit(cfg.ErrorBufferLimit), ) if err != nil { return nil, err diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go index 1153d4308e..1ec884c64d 100644 --- a/pkg/agent/core/ngt/service/ngt_test.go +++ b/pkg/agent/core/ngt/service/ngt_test.go @@ -1553,8 +1553,8 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit int // } // type want struct { -// want *payload.Search_Response -// err error +// wantRes *payload.Search_Response +// err error // } // type test struct { // name string @@ -1565,12 +1565,12 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // beforeFunc func(*testing.T, args) // afterFunc func(*testing.T, args) // } -// defaultCheckFunc := func(w want, got *payload.Search_Response, err error) error { +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { // if !errors.Is(err, w.err) { // return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) // } -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) // } // return nil // } @@ -1752,8 +1752,8 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit: test.fields.historyLimit, // } // -// got, err := n.Search(test.args.ctx, test.args.vec, test.args.size, test.args.epsilon, test.args.radius) -// if err := checkFunc(test.want, got, err); err != nil { +// gotRes, err := n.Search(test.args.ctx, test.args.vec, test.args.size, test.args.epsilon, test.args.radius) +// if err := checkFunc(test.want, gotRes, err); err != nil { // tt.Errorf("error = %v", err) // } // @@ -2022,6 +2022,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // // func Test_ngt_LinearSearch(t *testing.T) { // type args struct { +// ctx context.Context // vec []float32 // size uint32 // } @@ -2064,8 +2065,8 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit int // } // type want struct { -// want *payload.Search_Response -// err error +// wantRes *payload.Search_Response +// err error // } // type test struct { // name string @@ -2076,12 +2077,12 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // beforeFunc func(*testing.T, args) // afterFunc func(*testing.T, args) // } -// defaultCheckFunc := func(w want, got *payload.Search_Response, err error) error { +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { // if !errors.Is(err, w.err) { // return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) // } -// if !reflect.DeepEqual(got, w.want) { -// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) // } // return nil // } @@ -2091,6 +2092,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // { // name: "test_case_1", // args: args { +// ctx:nil, // vec:nil, // size:0, // }, @@ -2149,6 +2151,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // return test { // name: "test_case_2", // args: args { +// ctx:nil, // vec:nil, // size:0, // }, @@ -2257,8 +2260,8 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit: test.fields.historyLimit, // } // -// got, err := n.LinearSearch(test.args.vec, test.args.size) -// if err := checkFunc(test.want, got, err); err != nil { +// gotRes, err := n.LinearSearch(test.args.ctx, test.args.vec, test.args.size) +// if err := checkFunc(test.want, gotRes, err); err != nil { // tt.Errorf("error = %v", err) // } // @@ -2268,6 +2271,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // // func Test_ngt_LinearSearchByID(t *testing.T) { // type args struct { +// ctx context.Context // uuid string // size uint32 // } @@ -2341,6 +2345,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // { // name: "test_case_1", // args: args { +// ctx:nil, // uuid:"", // size:0, // }, @@ -2399,6 +2404,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // return test { // name: "test_case_2", // args: args { +// ctx:nil, // uuid:"", // size:0, // }, @@ -2507,7 +2513,7 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index { // historyLimit: test.fields.historyLimit, // } // -// gotVec, gotDst, err := n.LinearSearchByID(test.args.uuid, test.args.size) +// gotVec, gotDst, err := n.LinearSearchByID(test.args.ctx, test.args.uuid, test.args.size) // if err := checkFunc(test.want, gotVec, gotDst, err); err != nil { // tt.Errorf("error = %v", err) // } diff --git a/pkg/agent/core/ngt/service/vqueue/queue_test.go b/pkg/agent/core/ngt/service/vqueue/queue_test.go index a1dca0d2c5..4f95baca22 100644 --- a/pkg/agent/core/ngt/service/vqueue/queue_test.go +++ b/pkg/agent/core/ngt/service/vqueue/queue_test.go @@ -967,6 +967,115 @@ func TestGetVector(t *testing.T) { // } // } // +// func Test_vqueue_Range(t *testing.T) { +// type args struct { +// ctx context.Context +// f func(uuid string, vector []float32, ts int64) bool +// } +// type fields struct { +// il sync.Map[string, *index] +// dl sync.Map[string, *index] +// ic uint64 +// dc uint64 +// } +// type want struct { +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// f:nil, +// }, +// fields: fields { +// il:nil, +// dl:nil, +// ic:0, +// dc:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// f:nil, +// }, +// fields: fields { +// il:nil, +// dl:nil, +// ic:0, +// dc:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// v := &vqueue{ +// il: test.fields.il, +// dl: test.fields.dl, +// ic: test.fields.ic, +// dc: test.fields.dc, +// } +// +// v.Range(test.args.ctx, test.args.f) +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// // func Test_vqueue_IVQLen(t *testing.T) { // type fields struct { // il sync.Map[string, *index] diff --git a/pkg/gateway/filter/handler/grpc/handler_test.go b/pkg/gateway/filter/handler/grpc/handler_test.go index 0538955db6..cb78dcc7e4 100644 --- a/pkg/gateway/filter/handler/grpc/handler_test.go +++ b/pkg/gateway/filter/handler/grpc/handler_test.go @@ -6974,6 +6974,180 @@ package grpc // } // } // +// func Test_server_RemoveByTimestamp(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Remove_TimestampRequest +// } +// type fields struct { +// eg errgroup.Group +// defaultVectorizer string +// defaultFilters []string +// name string +// ip string +// ingress ingress.Client +// egress egress.Client +// gateway client.Client +// copts []grpc.CallOption +// streamConcurrency int +// Vectorizer string +// DistanceFilters []string +// ObjectFilters []string +// SearchFilters []string +// InsertFilters []string +// UpdateFilters []string +// UpsertFilters []string +// UnimplementedValdServerWithFilter vald.UnimplementedValdServerWithFilter +// } +// type want struct { +// want *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// eg:nil, +// defaultVectorizer:"", +// defaultFilters:nil, +// name:"", +// ip:"", +// ingress:nil, +// egress:nil, +// gateway:nil, +// copts:nil, +// streamConcurrency:0, +// Vectorizer:"", +// DistanceFilters:nil, +// ObjectFilters:nil, +// SearchFilters:nil, +// InsertFilters:nil, +// UpdateFilters:nil, +// UpsertFilters:nil, +// UnimplementedValdServerWithFilter:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// eg:nil, +// defaultVectorizer:"", +// defaultFilters:nil, +// name:"", +// ip:"", +// ingress:nil, +// egress:nil, +// gateway:nil, +// copts:nil, +// streamConcurrency:0, +// Vectorizer:"", +// DistanceFilters:nil, +// ObjectFilters:nil, +// SearchFilters:nil, +// InsertFilters:nil, +// UpdateFilters:nil, +// UpsertFilters:nil, +// UnimplementedValdServerWithFilter:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// eg: test.fields.eg, +// defaultVectorizer: test.fields.defaultVectorizer, +// defaultFilters: test.fields.defaultFilters, +// name: test.fields.name, +// ip: test.fields.ip, +// ingress: test.fields.ingress, +// egress: test.fields.egress, +// gateway: test.fields.gateway, +// copts: test.fields.copts, +// streamConcurrency: test.fields.streamConcurrency, +// Vectorizer: test.fields.Vectorizer, +// DistanceFilters: test.fields.DistanceFilters, +// ObjectFilters: test.fields.ObjectFilters, +// SearchFilters: test.fields.SearchFilters, +// InsertFilters: test.fields.InsertFilters, +// UpdateFilters: test.fields.UpdateFilters, +// UpsertFilters: test.fields.UpsertFilters, +// UnimplementedValdServerWithFilter: test.fields.UnimplementedValdServerWithFilter, +// } +// +// got, err := s.RemoveByTimestamp(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// // func Test_server_GetObject(t *testing.T) { // type args struct { // ctx context.Context diff --git a/pkg/gateway/lb/handler/grpc/handler_test.go b/pkg/gateway/lb/handler/grpc/handler_test.go index 77afaa590c..928adb73e0 100644 --- a/pkg/gateway/lb/handler/grpc/handler_test.go +++ b/pkg/gateway/lb/handler/grpc/handler_test.go @@ -3495,6 +3495,144 @@ package grpc // } // } // +// func Test_server_RemoveByTimestamp(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Remove_TimestampRequest +// } +// type fields struct { +// eg errgroup.Group +// gateway service.Gateway +// timeout time.Duration +// replica int +// streamConcurrency int +// multiConcurrency int +// name string +// ip string +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantLocs *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotLocs *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotLocs, w.wantLocs) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotLocs, w.wantLocs) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// eg:nil, +// gateway:nil, +// timeout:nil, +// replica:0, +// streamConcurrency:0, +// multiConcurrency:0, +// name:"", +// ip:"", +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// eg:nil, +// gateway:nil, +// timeout:nil, +// replica:0, +// streamConcurrency:0, +// multiConcurrency:0, +// name:"", +// ip:"", +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// eg: test.fields.eg, +// gateway: test.fields.gateway, +// timeout: test.fields.timeout, +// replica: test.fields.replica, +// streamConcurrency: test.fields.streamConcurrency, +// multiConcurrency: test.fields.multiConcurrency, +// name: test.fields.name, +// ip: test.fields.ip, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotLocs, err := s.RemoveByTimestamp(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotLocs, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// // func Test_server_GetObject(t *testing.T) { // type args struct { // ctx context.Context diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index a0c3d81c27..34df722886 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -51.9.0 +51.9.1 From 477a4f63f9a8c967f547c28ea70717249bc6bd14 Mon Sep 17 00:00:00 2001 From: kpango Date: Thu, 19 Oct 2023 16:34:41 +0900 Subject: [PATCH 02/10] add finalizer for pooling Signed-off-by: kpango --- cmd/tools/cli/benchmark/core/main.go | 6 +- internal/core/algorithm/ngt/ngt.go | 208 ++++++++++++++------------ internal/core/algorithm/ngt/option.go | 96 ++++++------ 3 files changed, 166 insertions(+), 144 deletions(-) diff --git a/cmd/tools/cli/benchmark/core/main.go b/cmd/tools/cli/benchmark/core/main.go index 9a97418b2e..455b1aa48a 100644 --- a/cmd/tools/cli/benchmark/core/main.go +++ b/cmd/tools/cli/benchmark/core/main.go @@ -278,7 +278,7 @@ func main() { }, "\t")) output("start") path, _ := file.MkdirTemp("") - sleep(ctx, time.Second*5, time.Minute*4, func() { + sleep(ctx, time.Second*5, time.Minute*1, func() { output("waiting for start") }, func() { runtime.GC() @@ -289,7 +289,7 @@ func main() { ids := make([]uint, len(vectors)) run(ctx, false, path, len(vectors[0]), vectors, ids, time.Hour*2, output) - sleep(ctx, time.Second*5, time.Minute*4, func() { + sleep(ctx, time.Second*5, time.Minute*2, func() { output("waiting for next") }, func() { runtime.GC() @@ -298,7 +298,7 @@ func main() { output("starting") }) run(ctx, true, path, len(vectors[0]), nil, nil, 0, output) - sleep(ctx, time.Second*5, time.Minute*4, func() { + sleep(ctx, time.Second*5, time.Minute*2, func() { output("waiting for next") }, func() { runtime.GC() diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go index 557bbb49a7..c70d100bde 100644 --- a/internal/core/algorithm/ngt/ngt.go +++ b/internal/core/algorithm/ngt/ngt.go @@ -27,6 +27,7 @@ import "C" import ( "context" "reflect" + "runtime" "sync/atomic" "unsafe" @@ -107,8 +108,31 @@ type ( mu *sync.RWMutex cmu *sync.RWMutex } + + ngtError struct { + err C.NGTError + destroyed atomic.Bool + } ) +func newNGTError() (n *ngtError) { + n = &ngtError{ + err: C.ngt_create_error_object(), + } + n.destroyed.Store(false) + runtime.SetFinalizer(n, func(ne *ngtError) { + ne.close() + }) + return n +} + +func (n *ngtError) close() { + if !n.destroyed.Load() { + C.ngt_destroy_error_object(n.err) + n.destroyed.Store(true) + } +} + // ObjectType is alias of object type in NGT. type objectType int @@ -274,20 +298,20 @@ func gen(isLoad bool, opts ...Option) (NGT, error) { func (n *ngt) setup() error { n.epool = sync.Pool{ New: func() interface{} { - return C.ngt_create_error_object() + return newNGTError() }, } for i := uint64(0); i < n.epl; i++ { - n.PutErrorBuffer(C.ngt_create_error_object()) + n.PutErrorBuffer(newNGTError()) } - ebuf := n.GetErrorBuffer() - n.prop = C.ngt_create_property(ebuf) + ne := n.GetErrorBuffer() + n.prop = C.ngt_create_property(ne.err) if n.prop == nil { - return errors.ErrCreateProperty(n.newGoError(ebuf)) + return errors.ErrCreateProperty(n.newGoError(ne)) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } @@ -316,22 +340,22 @@ func (n *ngt) create() (err error) { path := C.CString(n.idxPath) defer C.free(unsafe.Pointer(path)) - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() if !n.inMemory { - n.index = C.ngt_create_graph_and_tree(path, n.prop, ebuf) + n.index = C.ngt_create_graph_and_tree(path, n.prop, ne.err) if n.index == nil { - return n.newGoError(ebuf) + return n.newGoError(ne) } - if C.ngt_save_index(n.index, path, ebuf) == ErrorCode { - return n.newGoError(ebuf) + if C.ngt_save_index(n.index, path, ne.err) == ErrorCode { + return n.newGoError(ne) } } else { - n.index = C.ngt_create_graph_and_tree_in_memory(n.prop, ebuf) + n.index = C.ngt_create_graph_and_tree_in_memory(n.prop, ne.err) if n.index == nil { - return n.newGoError(ebuf) + return n.newGoError(ne) } } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } @@ -344,31 +368,31 @@ func (n *ngt) open() error { path := C.CString(n.idxPath) defer C.free(unsafe.Pointer(path)) - ebuf := n.GetErrorBuffer() - n.index = C.ngt_open_index(path, ebuf) + ne := n.GetErrorBuffer() + n.index = C.ngt_open_index(path, ne.err) if n.index == nil { - return n.newGoError(ebuf) + return n.newGoError(ne) } - if C.ngt_get_property(n.index, n.prop, ebuf) == ErrorCode { - return n.newGoError(ebuf) + if C.ngt_get_property(n.index, n.prop, ne.err) == ErrorCode { + return n.newGoError(ne) } - n.dimension = C.ngt_get_property_dimension(n.prop, ebuf) + n.dimension = C.ngt_get_property_dimension(n.prop, ne.err) if int(n.dimension) == -1 { - return n.newGoError(ebuf) + return n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } func (n *ngt) loadObjectSpace() error { - ebuf := n.GetErrorBuffer() - n.ospace = C.ngt_get_object_space(n.index, ebuf) + ne := n.GetErrorBuffer() + n.ospace = C.ngt_get_object_space(n.index, ne.err) if n.ospace == nil { - return n.newGoError(ebuf) + return n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } @@ -378,11 +402,11 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi return nil, errors.ErrIncompatibleDimensionSize(len(vec), int(n.dimension)) } - ebuf := n.GetErrorBuffer() - results := C.ngt_create_empty_results(ebuf) + ne := n.GetErrorBuffer() + results := C.ngt_create_empty_results(ne.err) defer C.ngt_destroy_results(results) if results == nil { - return nil, n.newGoError(ebuf) + return nil, n.newGoError(ne) } if epsilon == 0 { @@ -402,22 +426,21 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi *(*C.float)(unsafe.Pointer(&epsilon)), *(*C.float)(unsafe.Pointer(&radius)), results, - ebuf) + ne.err) vec = nil if ret == ErrorCode { - ne := ebuf n.rUnlock(true) return nil, n.newGoError(ne) } n.rUnlock(true) - rsize := int(C.ngt_get_result_size(results, ebuf)) + rsize := int(C.ngt_get_result_size(results, ne.err)) if rsize <= 0 { if n.cnt.Load() == 0 { - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil, errors.ErrSearchResultEmptyButNoDataStored } - err = n.newGoError(ebuf) + err = n.newGoError(ne) if err != nil { return nil, err } @@ -428,19 +451,19 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi for i := range result { select { case <-ctx.Done(): - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return result[:i], nil default: } - d := C.ngt_get_result(results, C.uint32_t(i), ebuf) + d := C.ngt_get_result(results, C.uint32_t(i), ne.err) if d.id == 0 && d.distance == 0 { - result[i] = SearchResult{0, 0, n.newGoError(ebuf)} - ebuf = n.GetErrorBuffer() + result[i] = SearchResult{0, 0, n.newGoError(ne)} + ne = n.GetErrorBuffer() } else { result[i] = SearchResult{uint32(d.id), float32(d.distance), nil} } } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return result, nil } @@ -451,11 +474,11 @@ func (n *ngt) LinearSearch(ctx context.Context, vec []float32, size int) (result return nil, errors.ErrIncompatibleDimensionSize(len(vec), int(n.dimension)) } - ebuf := n.GetErrorBuffer() - results := C.ngt_create_empty_results(ebuf) + ne := n.GetErrorBuffer() + results := C.ngt_create_empty_results(ne.err) defer C.ngt_destroy_results(results) if results == nil { - return nil, n.newGoError(ebuf) + return nil, n.newGoError(ne) } n.rLock(true) @@ -466,23 +489,22 @@ func (n *ngt) LinearSearch(ctx context.Context, vec []float32, size int) (result // C.size_t(size), *(*C.size_t)(unsafe.Pointer(&size)), results, - ebuf) + ne.err) vec = nil if ret == ErrorCode { - ne := ebuf n.rUnlock(true) return nil, n.newGoError(ne) } n.rUnlock(true) - rsize := int(C.ngt_get_result_size(results, ebuf)) + rsize := int(C.ngt_get_result_size(results, ne.err)) if rsize <= 0 { if n.cnt.Load() == 0 { - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil, errors.ErrSearchResultEmptyButNoDataStored } - err = n.newGoError(ebuf) + err = n.newGoError(ne) if err != nil { return nil, err } @@ -492,19 +514,19 @@ func (n *ngt) LinearSearch(ctx context.Context, vec []float32, size int) (result for i := range result { select { case <-ctx.Done(): - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return result[:i], nil default: } - d := C.ngt_get_result(results, C.uint32_t(i), ebuf) + d := C.ngt_get_result(results, C.uint32_t(i), ne.err) if d.id == 0 && d.distance == 0 { - result[i] = SearchResult{0, 0, n.newGoError(ebuf)} - ebuf = n.GetErrorBuffer() + result[i] = SearchResult{0, 0, n.newGoError(ne)} + ne = n.GetErrorBuffer() } else { result[i] = SearchResult{uint32(d.id), float32(d.distance), nil} } } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return result, nil } @@ -517,18 +539,18 @@ func (n *ngt) Insert(vec []float32) (id uint, err error) { } dim := C.uint32_t(n.dimension) cvec := (*C.float)(&vec[0]) - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() n.lock(true) - oid := C.ngt_insert_index_as_float(n.index, cvec, dim, ebuf) + oid := C.ngt_insert_index_as_float(n.index, cvec, dim, ne.err) n.unlock(true) id = uint(oid) cvec = nil vec = vec[:0:0] vec = nil if id == 0 { - return 0, n.newGoError(ebuf) + return 0, n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) n.cnt.Add(1) return id, nil @@ -622,14 +644,14 @@ func (n *ngt) CreateIndex(poolSize uint32) error { if poolSize == 0 { poolSize = n.poolSize } - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() n.lock(true) - ret := C.ngt_create_index(n.index, C.uint32_t(poolSize), ebuf) + ret := C.ngt_create_index(n.index, C.uint32_t(poolSize), ne.err) n.unlock(true) if ret == ErrorCode { - return n.newGoError(ebuf) + return n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } @@ -639,14 +661,14 @@ func (n *ngt) SaveIndex() error { if !n.inMemory { path := C.CString(n.idxPath) defer C.free(unsafe.Pointer(path)) - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() n.rLock(true) - ret := C.ngt_save_index(n.index, path, ebuf) + ret := C.ngt_save_index(n.index, path, ne.err) n.rUnlock(true) if ret == ErrorCode { - return n.newGoError(ebuf) + return n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) } return nil @@ -657,14 +679,14 @@ func (n *ngt) SaveIndexWithPath(idxPath string) error { if !n.inMemory && len(idxPath) != 0 { path := C.CString(idxPath) defer C.free(unsafe.Pointer(path)) - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() n.rLock(true) - ret := C.ngt_save_index(n.index, path, ebuf) + ret := C.ngt_save_index(n.index, path, ne.err) n.rUnlock(true) if ret == ErrorCode { - return n.newGoError(ebuf) + return n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) } return nil @@ -672,14 +694,14 @@ func (n *ngt) SaveIndexWithPath(idxPath string) error { // Remove removes from NGT index. func (n *ngt) Remove(id uint) error { - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() n.lock(true) - ret := C.ngt_remove_index(n.index, C.ObjectID(id), ebuf) + ret := C.ngt_remove_index(n.index, C.ObjectID(id), ne.err) n.unlock(true) if ret == ErrorCode { - return n.newGoError(ebuf) + return n.newGoError(ne) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) n.cnt.Add(^uint64(0)) @@ -700,23 +722,23 @@ func (n *ngt) BulkRemove(ids ...uint) (errs error) { // GetVector returns vector stored in NGT index. func (n *ngt) GetVector(id uint) (ret []float32, err error) { dimension := int(n.dimension) - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() switch n.objectType { case Float: n.rLock(false) - results := C.ngt_get_object_as_float(n.ospace, C.ObjectID(id), ebuf) + results := C.ngt_get_object_as_float(n.ospace, C.ObjectID(id), ne.err) n.rUnlock(false) if results == nil { - return nil, n.newGoError(ebuf) + return nil, n.newGoError(ne) } ret = (*[algorithm.MaximumVectorDimensionSize]float32)(unsafe.Pointer(results))[:dimension:dimension] case HalfFloat: n.rLock(false) - results := C.ngt_get_allocated_object_as_float(n.ospace, C.ObjectID(id), ebuf) + results := C.ngt_get_allocated_object_as_float(n.ospace, C.ObjectID(id), ne.err) n.rUnlock(false) defer C.free(unsafe.Pointer(results)) if results == nil { - return nil, n.newGoError(ebuf) + return nil, n.newGoError(ne) } ret = make([]float32, dimension) for i, elem := range (*[algorithm.MaximumVectorDimensionSize]float32)(unsafe.Pointer(results))[:dimension:dimension] { @@ -724,52 +746,52 @@ func (n *ngt) GetVector(id uint) (ret []float32, err error) { } case Uint8: n.rLock(false) - results := C.ngt_get_object_as_integer(n.ospace, C.ObjectID(id), ebuf) + results := C.ngt_get_object_as_integer(n.ospace, C.ObjectID(id), ne.err) n.rUnlock(false) if results == nil { - return nil, n.newGoError(ebuf) + return nil, n.newGoError(ne) } ret = make([]float32, 0, dimension) for _, elem := range (*[algorithm.MaximumVectorDimensionSize]C.uint8_t)(unsafe.Pointer(results))[:dimension:dimension] { ret = append(ret, float32(elem)) } default: - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil, errors.ErrUnsupportedObjectType } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return ret, nil } -func (n *ngt) newGoError(ebuf C.NGTError) (err error) { - msg := C.GoString(C.ngt_get_error_string(ebuf)) +func (n *ngt) newGoError(ne *ngtError) (err error) { + msg := C.GoString(C.ngt_get_error_string(ne.err)) if len(msg) == 0 { - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } if n.epl == 0 || n.eps.Load() < n.epl { - n.PutErrorBuffer(C.ngt_create_error_object()) + n.PutErrorBuffer(newNGTError()) } - C.ngt_destroy_error_object(ebuf) + ne.close() return errors.NewNGTError(msg) } -func (n *ngt) GetErrorBuffer() (ebuf C.NGTError) { +func (n *ngt) GetErrorBuffer() (ne *ngtError) { var ok bool - ebuf, ok = n.epool.Get().(C.NGTError) + ne, ok = n.epool.Get().(*ngtError) if !ok { - ebuf = C.ngt_create_error_object() + ne = newNGTError() } n.eps.Add(^uint64(0)) - return ebuf + return ne } -func (n *ngt) PutErrorBuffer(ebuf C.NGTError) { +func (n *ngt) PutErrorBuffer(ne *ngtError) { if n.epl != 0 && n.eps.Load() > n.epl { - C.ngt_destroy_error_object(ebuf) + ne.close() return } - n.epool.Put(ebuf) + n.epool.Put(ne) n.eps.Add(1) } diff --git a/internal/core/algorithm/ngt/option.go b/internal/core/algorithm/ngt/option.go index 29f9f74c10..a98adfe06f 100644 --- a/internal/core/algorithm/ngt/option.go +++ b/internal/core/algorithm/ngt/option.go @@ -93,12 +93,12 @@ func WithDimension(size int) Option { return errors.NewErrCriticalOption("dimension", size, err) } - ebuf := n.GetErrorBuffer() - if C.ngt_set_property_dimension(n.prop, C.int32_t(size), ebuf) == ErrorCode { - err := errors.ErrFailedToSetDimension(n.newGoError(ebuf)) + ne := n.GetErrorBuffer() + if C.ngt_set_property_dimension(n.prop, C.int32_t(size), ne.err) == ErrorCode { + err := errors.ErrFailedToSetDimension(n.newGoError(ne)) return errors.NewErrCriticalOption("dimension", size, err) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) n.dimension = C.int32_t(size) @@ -141,74 +141,74 @@ func WithDistanceTypeByString(dt string) Option { // WithDistanceType represents the option to set the distance type for NGT. func WithDistanceType(t distanceType) Option { return func(n *ngt) error { - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() switch t { case L1: - if C.ngt_set_property_distance_type_l1(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_l1(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case L2: - if C.ngt_set_property_distance_type_l2(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_l2(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case Angle: - if C.ngt_set_property_distance_type_angle(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_angle(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case Hamming: - if C.ngt_set_property_distance_type_hamming(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_hamming(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case Cosine: - if C.ngt_set_property_distance_type_cosine(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_cosine(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case Poincare: - if C.ngt_set_property_distance_type_poincare(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_poincare(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case Lorentz: - if C.ngt_set_property_distance_type_lorentz(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_lorentz(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case Jaccard: - if C.ngt_set_property_distance_type_jaccard(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_jaccard(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case SparseJaccard: - if C.ngt_set_property_distance_type_sparse_jaccard(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_sparse_jaccard(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case NormalizedL2: - if C.ngt_set_property_distance_type_normalized_l2(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_normalized_l2(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case NormalizedAngle: - if C.ngt_set_property_distance_type_normalized_angle(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_normalized_angle(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } case NormalizedCosine: - if C.ngt_set_property_distance_type_normalized_cosine(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetDistanceType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_distance_type_normalized_cosine(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetDistanceType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("distanceType", t, err) } default: err := errors.ErrUnsupportedDistanceType - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return errors.NewErrCriticalOption("distanceType", t, err) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } } @@ -230,29 +230,29 @@ func WithObjectTypeByString(ot string) Option { // WithObjectType represents the option to set the object type for NGT. func WithObjectType(t objectType) Option { return func(n *ngt) error { - ebuf := n.GetErrorBuffer() + ne := n.GetErrorBuffer() switch t { case Uint8: - if C.ngt_set_property_object_type_integer(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetObjectType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_object_type_integer(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetObjectType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("objectType", t, err) } case HalfFloat: - if C.ngt_set_property_object_type_float16(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetObjectType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_object_type_float16(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetObjectType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("objectType", t, err) } case Float: - if C.ngt_set_property_object_type_float(n.prop, ebuf) == ErrorCode { - err := errors.ErrFailedToSetObjectType(n.newGoError(ebuf), t.String()) + if C.ngt_set_property_object_type_float(n.prop, ne.err) == ErrorCode { + err := errors.ErrFailedToSetObjectType(n.newGoError(ne), t.String()) return errors.NewErrCriticalOption("objectType", t, err) } default: - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) err := errors.ErrUnsupportedObjectType return errors.NewErrCriticalOption("objectType", t, err) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) n.objectType = t return nil } @@ -261,12 +261,12 @@ func WithObjectType(t objectType) Option { // WithCreationEdgeSize represents the option to set the creation edge size for NGT. func WithCreationEdgeSize(size int) Option { return func(n *ngt) error { - ebuf := n.GetErrorBuffer() - if C.ngt_set_property_edge_size_for_creation(n.prop, C.int16_t(size), ebuf) == ErrorCode { - err := errors.ErrFailedToSetCreationEdgeSize(n.newGoError(ebuf)) + ne := n.GetErrorBuffer() + if C.ngt_set_property_edge_size_for_creation(n.prop, C.int16_t(size), ne.err) == ErrorCode { + err := errors.ErrFailedToSetCreationEdgeSize(n.newGoError(ne)) return errors.NewErrCriticalOption("creationEdgeSize", size, err) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } } @@ -274,12 +274,12 @@ func WithCreationEdgeSize(size int) Option { // WithSearchEdgeSize represents the option to set the search edge size for NGT. func WithSearchEdgeSize(size int) Option { return func(n *ngt) error { - ebuf := n.GetErrorBuffer() - if C.ngt_set_property_edge_size_for_search(n.prop, C.int16_t(size), ebuf) == ErrorCode { - err := errors.ErrFailedToSetSearchEdgeSize(n.newGoError(ebuf)) + ne := n.GetErrorBuffer() + if C.ngt_set_property_edge_size_for_search(n.prop, C.int16_t(size), ne.err) == ErrorCode { + err := errors.ErrFailedToSetSearchEdgeSize(n.newGoError(ne)) return errors.NewErrCriticalOption("searchEdgeSize", size, err) } - n.PutErrorBuffer(ebuf) + n.PutErrorBuffer(ne) return nil } } From 3a8f3bbcc2fd1e18c2a98ed0a3f4d9044cde51f8 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 30 Oct 2023 16:47:42 +0900 Subject: [PATCH 03/10] fix benchmark Signed-off-by: Kosuke Morimoto --- internal/core/algorithm/ngt/ngt_bench_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/core/algorithm/ngt/ngt_bench_test.go b/internal/core/algorithm/ngt/ngt_bench_test.go index dc0b8404ba..6345918032 100644 --- a/internal/core/algorithm/ngt/ngt_bench_test.go +++ b/internal/core/algorithm/ngt/ngt_bench_test.go @@ -34,7 +34,11 @@ var ( ) func init() { - vectors, _, _ = load("sift-128-euclidean.hdf5") + filename := os.Getenv("DATA_PATH") + if _, err := os.Stat(filename); err != nil { + return + } + vectors, _, _ = load(filename) n, _ = New( WithDimension(len(vectors[0])), WithDefaultPoolSize(8), @@ -45,6 +49,9 @@ func init() { } func BenchmarkNGT(b *testing.B) { + if len(vectors) == 0 { + return + } b.Logf("# of vectors: %v", len(vectors)) output := func(header string) { status := fmt.Sprintf("/proc/%d/status", pid) From 75d15f695f8d6a7c960ca566de389b811ff43485 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 30 Oct 2023 21:23:10 +0900 Subject: [PATCH 04/10] fix test Signed-off-by: Kosuke Morimoto --- internal/core/algorithm/ngt/ngt_test.go | 12 +- internal/core/algorithm/ngt/option.go | 9 +- .../metrics/mem/malloc/malloc_test.go | 161 ++++++++++++++++++ internal/test/comparator/comparators.go | 7 + 4 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 internal/observability/metrics/mem/malloc/malloc_test.go diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go index a90992b5c0..d2097ea062 100644 --- a/internal/core/algorithm/ngt/ngt_test.go +++ b/internal/core/algorithm/ngt/ngt_test.go @@ -43,9 +43,10 @@ var ( // !!! These fields will not be verified in the entire test // Do not validate C dependencies comparator.IgnoreFields(ngt{}, - "dimension", "prop", "epool", "index", "ospace"), + "dimension", "prop", "epool", "index", "ospace", "eps"), comparator.RWMutexComparer, comparator.ErrorComparer, + comparator.AtomicUint64Comparator, } searchResultComparator = []comparator.Option{ @@ -140,6 +141,7 @@ func TestNew(t *testing.T) { objectType: Float, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, comparators: append(ngtComparator, comparator.CompareField("idxPath", comparator.Comparer(func(s1, s2 string) bool { @@ -166,6 +168,7 @@ func TestNew(t *testing.T) { objectType: Float, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, } @@ -191,6 +194,7 @@ func TestNew(t *testing.T) { objectType: Uint8, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, } @@ -320,6 +324,7 @@ func TestLoad(t *testing.T) { objectType: Uint8, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -386,6 +391,7 @@ func TestLoad(t *testing.T) { objectType: Uint8, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -452,6 +458,7 @@ func TestLoad(t *testing.T) { objectType: Float, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -518,6 +525,7 @@ func TestLoad(t *testing.T) { objectType: Float, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -714,6 +722,7 @@ func Test_gen(t *testing.T) { objectType: Float, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, comparators: append(ngtComparator, comparator.CompareField("idxPath", comparator.Comparer(func(s1, s2 string) bool { @@ -761,6 +770,7 @@ func Test_gen(t *testing.T) { objectType: Uint8, mu: &sync.RWMutex{}, cmu: &sync.RWMutex{}, + epl: DefaultErrorBufferLimit, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error, comparators ...comparator.Option) error { diff --git a/internal/core/algorithm/ngt/option.go b/internal/core/algorithm/ngt/option.go index a98adfe06f..faeff63fcd 100644 --- a/internal/core/algorithm/ngt/option.go +++ b/internal/core/algorithm/ngt/option.go @@ -36,9 +36,10 @@ import ( type Option func(*ngt) error var ( - DefaultPoolSize = uint32(10000) - DefaultRadius = float32(-1.0) - DefaultEpsilon = float32(0.1) + DefaultPoolSize = uint32(10000) + DefaultRadius = float32(-1.0) + DefaultEpsilon = float32(0.1) + DefaultErrorBufferLimit = uint64(10) defaultOptions = []Option{ WithIndexPath("/tmp/ngt-" + strconv.FormatInt(fastime.UnixNanoNow(), 10)), @@ -51,7 +52,7 @@ var ( WithObjectType(Float), WithDistanceType(L2), WithBulkInsertChunkSize(100), - WithErrorBufferLimit(10), + WithErrorBufferLimit(DefaultErrorBufferLimit), } ) diff --git a/internal/observability/metrics/mem/malloc/malloc_test.go b/internal/observability/metrics/mem/malloc/malloc_test.go new file mode 100644 index 0000000000..70adfed31d --- /dev/null +++ b/internal/observability/metrics/mem/malloc/malloc_test.go @@ -0,0 +1,161 @@ +package malloc + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type want struct { +// want metrics.Metrics +// } +// type test struct { +// name string +// want want +// checkFunc func(want, metrics.Metrics) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got metrics.Metrics) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := New() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_mallocMetrics_View(t *testing.T) { +// type want struct { +// want []*metrics.View +// err error +// } +// type test struct { +// name string +// m *mallocMetrics +// want want +// checkFunc func(want, []*metrics.View, error) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got []*metrics.View, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// m := &mallocMetrics{} +// +// got, err := m.View() +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/test/comparator/comparators.go b/internal/test/comparator/comparators.go index 4ad9bfc461..1923cf6f8f 100644 --- a/internal/test/comparator/comparators.go +++ b/internal/test/comparator/comparators.go @@ -15,6 +15,7 @@ package comparator import ( "reflect" + "sync/atomic" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/sync" @@ -50,4 +51,10 @@ var ( // skipcq: VET-V0008 return reflect.DeepEqual(x, y) }) + + // skipcq: VET-V0008 + AtomicUint64Comparator = Comparer(func(x, y atomic.Uint64) bool { + // skipcq: VET-V0008 + return reflect.DeepEqual(x, y) + }) ) From 4661b2149e4c4fa66cd1afb120c859c8df378735 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 30 Oct 2023 22:11:27 +0900 Subject: [PATCH 05/10] fix test Signed-off-by: Kosuke Morimoto --- internal/strings/strings_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/strings/strings_test.go b/internal/strings/strings_test.go index b62c1c2f5b..0a058263df 100644 --- a/internal/strings/strings_test.go +++ b/internal/strings/strings_test.go @@ -14,10 +14,10 @@ package strings import ( + "fmt" "reflect" "testing" - "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/test/goleak" ) @@ -39,7 +39,7 @@ func TestJoin(t *testing.T) { } defaultCheckFunc := func(w want, gotStr string) error { if !reflect.DeepEqual(gotStr, w.wantStr) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotStr, w.wantStr) + return fmt.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotStr, w.wantStr) } return nil } From 182b2bcb15d0404c5e9ba79c5aa39e6f89a214b6 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 30 Oct 2023 22:27:41 +0900 Subject: [PATCH 06/10] fix test Signed-off-by: Kosuke Morimoto --- cmd/tools/cli/benchmark/core/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tools/cli/benchmark/core/main_test.go b/cmd/tools/cli/benchmark/core/main_test.go index 3748eb8a90..8d8297becb 100644 --- a/cmd/tools/cli/benchmark/core/main_test.go +++ b/cmd/tools/cli/benchmark/core/main_test.go @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package core +package main // NOT IMPLEMENTED BELOW // From 110efce5dcb011fbf1aea56e441bc1ed29307cdf Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Tue, 31 Oct 2023 14:54:53 +0900 Subject: [PATCH 07/10] fix reviewdog report Signed-off-by: Kosuke Morimoto --- cmd/tools/cli/benchmark/core/main.go | 49 ++++++++++++++++++---------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/cmd/tools/cli/benchmark/core/main.go b/cmd/tools/cli/benchmark/core/main.go index 455b1aa48a..6da8e5b660 100644 --- a/cmd/tools/cli/benchmark/core/main.go +++ b/cmd/tools/cli/benchmark/core/main.go @@ -219,14 +219,18 @@ func main() { var wg sync.WaitGroup wg.Add(1) go func() { + const timeout = time.Second * 5 defer wg.Done() srv := &http.Server{ - Addr: "0.0.0.0:6060", - Handler: metrics.NewPProfHandler(), + Addr: "0.0.0.0:6060", + Handler: metrics.NewPProfHandler(), + ReadHeaderTimeout: timeout, } go srv.ListenAndServe() <-ctx.Done() - srv.Shutdown(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + srv.Shutdown(ctx) }() vectors, _, _ := load(os.Getenv("DATA_PATH")) @@ -278,7 +282,11 @@ func main() { }, "\t")) output("start") path, _ := file.MkdirTemp("") - sleep(ctx, time.Second*5, time.Minute*1, func() { + const ( + logInterval = time.Second * 5 + waitForStart = time.Minute * 1 + ) + sleep(ctx, logInterval, waitForStart, func() { output("waiting for start") }, func() { runtime.GC() @@ -288,8 +296,10 @@ func main() { }) ids := make([]uint, len(vectors)) - run(ctx, false, path, len(vectors[0]), vectors, ids, time.Hour*2, output) - sleep(ctx, time.Second*5, time.Minute*2, func() { + const timeToRun = time.Hour * 2 + run(ctx, false, path, len(vectors[0]), vectors, ids, timeToRun, output) + const waitForNext = time.Minute * 2 + sleep(ctx, logInterval, waitForNext, func() { output("waiting for next") }, func() { runtime.GC() @@ -298,7 +308,7 @@ func main() { output("starting") }) run(ctx, true, path, len(vectors[0]), nil, nil, 0, output) - sleep(ctx, time.Second*5, time.Minute*2, func() { + sleep(ctx, logInterval, waitForNext, func() { output("waiting for next") }, func() { runtime.GC() @@ -306,25 +316,29 @@ func main() { time.Sleep(time.Minute) output("starting") }) - run(ctx, true, path, len(vectors[0]), vectors, ids, time.Hour*2, output) + run(ctx, true, path, len(vectors[0]), vectors, ids, timeToRun, output) ids = ids[:0:0] ids = nil vectors = vectors[:0:0] vectors = nil - sleep(ctx, time.Second*5, time.Minute*5, func() { + const ( + waitForGC = time.Minute * 5 + timeToFinalize = time.Minute * 5 + ) + sleep(ctx, time.Second*5, waitForGC, func() { output("waiting for gc") }, func() { runtime.GC() output("gc") }) - sleep(ctx, time.Second*5, time.Minute*5, func() { + sleep(ctx, time.Second*5, waitForGC, func() { output("waiting for gc") }, func() { runtime.GC() output("gc") }) - sleep(ctx, time.Second*5, time.Minute*5, func() { + sleep(ctx, time.Second*5, timeToFinalize, func() { output("finalizing") }, func() { cancel() @@ -333,18 +347,19 @@ func main() { } func run(ctx context.Context, load bool, path string, dim int, vectors [][]float32, ids []uint, dur time.Duration, output func(header string)) { + const poolSize = 8 var n ngt.NGT if load { n, _ = ngt.Load( ngt.WithDimension(dim), - ngt.WithDefaultPoolSize(8), + ngt.WithDefaultPoolSize(poolSize), ngt.WithObjectType(ngt.Float), ngt.WithDistanceType(ngt.L2), ) } else { n, _ = ngt.New( ngt.WithDimension(dim), - ngt.WithDefaultPoolSize(8), + ngt.WithDefaultPoolSize(poolSize), ngt.WithObjectType(ngt.Float), ngt.WithDistanceType(ngt.L2), ) @@ -374,7 +389,7 @@ func run(ctx context.Context, load bool, path string, dim int, vectors [][]float ids[i] = id } output("insert") - if err = n.CreateIndex(8); err != nil { + if err = n.CreateIndex(poolSize); err != nil { log.Fatal(err) } output("create index") @@ -392,7 +407,7 @@ func run(ctx context.Context, load bool, path string, dim int, vectors [][]float } } output("insert") - if err = n.CreateIndex(8); err != nil { + if err = n.CreateIndex(poolSize); err != nil { log.Fatal(err) } output("create index") @@ -431,7 +446,6 @@ func sleep(ctx context.Context, duration, limit time.Duration, fn, efn func()) { fn() } } - return } ticker := time.NewTicker(duration) defer ticker.Stop() @@ -445,7 +459,6 @@ func sleep(ctx context.Context, duration, limit time.Duration, fn, efn func()) { fn() } } - return } // load function loads training and test vector from hdf file. The size of ids is same to the number of training data. @@ -490,7 +503,7 @@ func load(path string) (train, test [][]float32, err error) { for i := 0; i < row; i++ { vecs[i] = make([]float32, dim) for j := 0; j < dim; j++ { - vecs[i][j] = float32(vec[i*dim+j]) + vecs[i][j] = vec[i*dim+j] } } From 02b0b5f2942235f5d2854fa9bf84ff411e836b1d Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Tue, 31 Oct 2023 17:42:55 +0900 Subject: [PATCH 08/10] fix reviewdog report Signed-off-by: Kosuke Morimoto --- cmd/tools/cli/benchmark/core/main.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/tools/cli/benchmark/core/main.go b/cmd/tools/cli/benchmark/core/main.go index 6da8e5b660..14dcee3c43 100644 --- a/cmd/tools/cli/benchmark/core/main.go +++ b/cmd/tools/cli/benchmark/core/main.go @@ -37,6 +37,7 @@ import ( ) func main() { + const columnSize = 42 var ( buf []byte err error @@ -60,7 +61,7 @@ func main() { pfile = fmt.Sprintf("/proc/%d/status", os.Getpid()) zero = float64(0.0) - format = "%s\t" + strings.TrimSuffix(strings.Repeat("%.2f\t", 42), "\t") + format = "%s\t" + strings.TrimSuffix(strings.Repeat("%.2f\t", columnSize), "\t") ) output := func(header string) { buf, err = os.ReadFile(pfile) @@ -228,9 +229,7 @@ func main() { } go srv.ListenAndServe() <-ctx.Done() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - srv.Shutdown(ctx) + srv.Shutdown(context.Background()) }() vectors, _, _ := load(os.Getenv("DATA_PATH")) @@ -326,19 +325,19 @@ func main() { waitForGC = time.Minute * 5 timeToFinalize = time.Minute * 5 ) - sleep(ctx, time.Second*5, waitForGC, func() { + sleep(ctx, logInterval, waitForGC, func() { output("waiting for gc") }, func() { runtime.GC() output("gc") }) - sleep(ctx, time.Second*5, waitForGC, func() { + sleep(ctx, logInterval, waitForGC, func() { output("waiting for gc") }, func() { runtime.GC() output("gc") }) - sleep(ctx, time.Second*5, timeToFinalize, func() { + sleep(ctx, logInterval, timeToFinalize, func() { output("finalizing") }, func() { cancel() From 52eb1c403144036f82fc18c872b3030af44def53 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Wed, 1 Nov 2023 14:06:19 +0900 Subject: [PATCH 09/10] fix deepsource report Signed-off-by: Kosuke Morimoto --- internal/net/grpc/client.go | 14 +++++++------- internal/sync/semaphore/semaphore_test.go | 6 ++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/net/grpc/client.go b/internal/net/grpc/client.go index 29729a8516..df9f84ddd0 100644 --- a/internal/net/grpc/client.go +++ b/internal/net/grpc/client.go @@ -161,7 +161,7 @@ func (g *gRPCClient) StartConnectionMonitor(ctx context.Context) (<-chan error, ech := make(chan error, len(addrs)) for _, addr := range addrs { - if len(addr) != 0 { + if addr != "" { _, err := g.Connect(ctx, addr, grpc.WithBlock()) if err != nil { if !errors.Is(err, context.Canceled) && @@ -218,7 +218,7 @@ func (g *gRPCClient) StartConnectionMonitor(ctx context.Context) (<-chan error, if g.enablePoolRebalance { err = g.rangeConns(func(addr string, p pool.Conn) bool { // if addr or pool is nil or empty the registration of conns is invalid let's disconnect them - if len(addr) == 0 || p == nil { + if addr == "" || p == nil { disconnectTargets = append(disconnectTargets, addr) return true } @@ -255,7 +255,7 @@ func (g *gRPCClient) StartConnectionMonitor(ctx context.Context) (<-chan error, case <-hcTick.C: err = g.rangeConns(func(addr string, p pool.Conn) bool { // if addr or pool is nil or empty the registration of conns is invalid let's disconnect them - if len(addr) == 0 || p == nil { + if addr == "" || p == nil { disconnectTargets = append(disconnectTargets, addr) return true } @@ -295,7 +295,7 @@ func (g *gRPCClient) StartConnectionMonitor(ctx context.Context) (<-chan error, } if err != nil && errors.Is(err, errors.ErrGRPCClientConnNotFound("*")) && len(addrs) != 0 { for _, addr := range addrs { - if len(addr) != 0 { + if addr != "" { log.Debugf("connection for %s not found in connection map will re-connect soon", addr) g.crl.Store(addr, false) } @@ -639,7 +639,7 @@ func (g *gRPCClient) RoundRobin(ctx context.Context, f func(ctx context.Context, } var boName string - if boName = FromGRPCMethod(sctx); len(boName) != 0 { + if boName = FromGRPCMethod(sctx); boName != "" { sctx = backoff.WithBackoffName(sctx, boName) } @@ -659,7 +659,7 @@ func (g *gRPCClient) RoundRobin(ctx context.Context, f func(ctx context.Context, }() var boName string ctx = WrapGRPCMethod(ctx, addr) - if boName = FromGRPCMethod(ctx); len(boName) != 0 { + if boName = FromGRPCMethod(ctx); boName != "" { ctx = backoff.WithBackoffName(ctx, boName) } if g.cb != nil && len(boName) > 0 { @@ -790,7 +790,7 @@ func (g *gRPCClient) connectWithBackoff(ctx context.Context, p pool.Conn, addr s if g.bo != nil && enableBackoff { var boName string sctx = WrapGRPCMethod(sctx, addr) - if boName = FromGRPCMethod(sctx); len(boName) != 0 { + if boName = FromGRPCMethod(sctx); boName != "" { sctx = backoff.WithBackoffName(sctx, boName) } do := func(ctx context.Context) (r interface{}, ret bool, err error) { diff --git a/internal/sync/semaphore/semaphore_test.go b/internal/sync/semaphore/semaphore_test.go index d55037c7f7..8888c44d38 100644 --- a/internal/sync/semaphore/semaphore_test.go +++ b/internal/sync/semaphore/semaphore_test.go @@ -90,8 +90,7 @@ func TestWeightedTryAcquire(t *testing.T) { sem := semaphore.NewWeighted(2) tries := []bool{} sem.Acquire(ctx, 1) - tries = append(tries, sem.TryAcquire(1)) - tries = append(tries, sem.TryAcquire(1)) + tries = append(tries, sem.TryAcquire(1), sem.TryAcquire(1)) sem.Release(2) @@ -120,8 +119,7 @@ func TestWeightedAcquire(t *testing.T) { tries := []bool{} sem.Acquire(ctx, 1) - tries = append(tries, tryAcquire(1)) - tries = append(tries, tryAcquire(1)) + tries = append(tries, tryAcquire(1), tryAcquire(1)) sem.Release(2) From c10d65cd660a73804cd70b2fea9e4dfaa905c2df Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Wed, 8 Nov 2023 11:16:41 +0900 Subject: [PATCH 10/10] fix according to comments Signed-off-by: Kosuke Morimoto --- cmd/tools/cli/benchmark/core/main.go | 3 ++- internal/core/algorithm/ngt/ngt_bench_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/tools/cli/benchmark/core/main.go b/cmd/tools/cli/benchmark/core/main.go index 14dcee3c43..27c95d6748 100644 --- a/cmd/tools/cli/benchmark/core/main.go +++ b/cmd/tools/cli/benchmark/core/main.go @@ -24,7 +24,6 @@ import ( "os" "runtime" "strconv" - "sync" "time" "github.com/vdaas/vald/internal/conv" @@ -33,9 +32,11 @@ import ( "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/http/metrics" "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" "gonum.org/v1/hdf5" ) +// This program is intended to get long-running results that are difficult to measure with Go standard benchmarks. func main() { const columnSize = 42 var ( diff --git a/internal/core/algorithm/ngt/ngt_bench_test.go b/internal/core/algorithm/ngt/ngt_bench_test.go index 6345918032..a10daac6dc 100644 --- a/internal/core/algorithm/ngt/ngt_bench_test.go +++ b/internal/core/algorithm/ngt/ngt_bench_test.go @@ -48,6 +48,7 @@ func init() { pid = os.Getpid() } +// BenchmarkNGT measures memory usage in insert/create index/remove steps. func BenchmarkNGT(b *testing.B) { if len(vectors) == 0 { return