From 6f344ca01637c43680eb71cf17f2d1a88fa496a3 Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 18 May 2020 16:23:11 +0900 Subject: [PATCH 1/5] [patch] add agent auto save indexing feature Signed-off-by: kpango --- charts/vald/values.yaml | 2 + go.mod | 12 ++-- go.sum | 22 +++--- hack/go.mod.default | 2 +- internal/config/ngt.go | 5 ++ internal/core/ngt/ngt.go | 3 + internal/core/ngt/option.go | 2 +- pkg/agent/ngt/service/ngt.go | 102 +++++++++++++++++---------- pkg/manager/index/service/indexer.go | 5 ++ versions/NGT_VERSION | 2 +- 10 files changed, 101 insertions(+), 56 deletions(-) diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 83530f2ff2..9083938b7d 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -746,6 +746,8 @@ agent: auto_index_check_duration: 30m # agent.ngt.auto_index_length -- number of cache to trigger automatic indexing auto_index_length: 100 + # agent.ngt.auto_index_save_duration -- duration of automatic save index + auto_index_save_duration: 35m # agent.ngt.dimension -- vector dimension dimension: 4096 # agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size diff --git a/go.mod b/go.mod index 9b0327ad03..f1ff19cdd1 100755 --- a/go.mod +++ b/go.mod @@ -8,11 +8,11 @@ replace ( github.com/coreos/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate v0.3.0-java - github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.5.1-0.20200513234351-f378f59f6710 - github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20200511135441-57b003a04490 + github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.5.1-0.20200517154853-096feaaf8e9f + github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20200515162754-0714040f3e35 github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.1 - github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.10.0 - github.com/gorilla/mux => github.com/gorilla/mux v1.7.4 + github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.11.0 + github.com/gorilla/mux => github.com/gorilla/mux v1.7.5-0.20200517040254-948bec34b516 github.com/gorilla/websocket => github.com/gorilla/websocket v1.4.2 github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0+incompatible golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 @@ -55,11 +55,11 @@ require ( go.uber.org/goleak v1.0.0 golang.org/x/mod v0.3.0 // indirect golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 - golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d // indirect + golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 // indirect gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614 gonum.org/v1/netlib v0.0.0-20200317120129-c5a04cffd98a // indirect gonum.org/v1/plot v0.7.0 - google.golang.org/genproto v0.0.0-20200514193133-8feb7f20f2a2 + google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 google.golang.org/grpc v1.29.1 gopkg.in/yaml.v2 v2.3.0 k8s.io/api v0.18.2 diff --git a/go.sum b/go.sum index d8ef847b2b..ea34ddeaa5 100644 --- a/go.sum +++ b/go.sum @@ -180,14 +180,14 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs= github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= -github.com/go-sql-driver/mysql v1.5.1-0.20200513234351-f378f59f6710 h1:AHajtDLQUi2wmBQbhsvy7imZ9CRaRaSTkPTruDOGUW0= -github.com/go-sql-driver/mysql v1.5.1-0.20200513234351-f378f59f6710/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.5.1-0.20200517154853-096feaaf8e9f h1:lGOphibtewuHJ9+UQFUs1rRyEcbIyD3jbmnDk+pRbjo= +github.com/go-sql-driver/mysql v1.5.1-0.20200517154853-096feaaf8e9f/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/gocql/gocql v0.0.0-20200511135441-57b003a04490 h1:DE7b84SsUn+1Y3LG4YpFn7KpUra0iuYwS6PkSayorcg= -github.com/gocql/gocql v0.0.0-20200511135441-57b003a04490/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/gocql/gocql v0.0.0-20200515162754-0714040f3e35 h1:5kRvQfb7q17Og+HNo8NXHjcbkyEqJp75ZzXHfsa2JvE= +github.com/gocql/gocql v0.0.0-20200515162754-0714040f3e35/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= github.com/gocraft/dbr/v2 v2.7.0 h1:x+UnhSBYPFBBdtikLSMLQ9KPuquSUj4yBijsQAhhNZo= github.com/gocraft/dbr/v2 v2.7.0/go.mod h1:wQdbxPBSloo2OlSedMxfNW0mgk0GXys9O1VFmQiwcx4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -243,10 +243,10 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/gophercloud/gophercloud v0.10.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= +github.com/gophercloud/gophercloud v0.11.0/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.7.5-0.20200517040254-948bec34b516 h1:BU5W11a5nPsU82VaJwaimHGO/Z50RGtaraAXe4srksU= +github.com/gorilla/mux v1.7.5-0.20200517040254-948bec34b516/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -641,8 +641,8 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d h1:n6zwymXmN9rCClNNmCWwV3qwMmBcRw/WeIGDK8Qnzk4= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 h1:vmsb6v0zUdmUlXfwKaYrHPPRCV0lHq/IwNIf0ASGjyQ= +golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -674,8 +674,8 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200514193133-8feb7f20f2a2 h1:RwW6+LxyOQJ7oeoZ76GIJlwt/O0J5cN2fk+q/jK27kQ= -google.golang.org/genproto v0.0.0-20200514193133-8feb7f20f2a2/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 h1:1Ym+vvUpq1ZHvxzn34gENJX8U4aKO+vhy2P/2+Xl6qQ= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/hack/go.mod.default b/hack/go.mod.default index aa96d007c7..6d1d5850ac 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -11,7 +11,7 @@ replace ( github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql master github.com/gocql/gocql => github.com/gocql/gocql master github.com/gogo/protobuf => github.com/gogo/protobuf master - github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.10.0 + github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.11.0 github.com/gorilla/mux => github.com/gorilla/mux master github.com/gorilla/websocket => github.com/gorilla/websocket master github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0 diff --git a/internal/config/ngt.go b/internal/config/ngt.go index 95339b2eb0..16f732ff33 100644 --- a/internal/config/ngt.go +++ b/internal/config/ngt.go @@ -46,6 +46,9 @@ type NGT struct { // AutoIndexCheckDuration represent checking loop duration about auto indexing execution AutoIndexCheckDuration string `yaml:"auto_index_check_duration" json:"auto_index_check_duration"` + // AutoIndexSaveDuration represent checking loop duration about auto save index execution + AutoIndexSaveDuration string `yaml:"auto_index_save_duration" json:"auto_index_save_duration"` + // AutoIndexLength represent auto index length limit AutoIndexLength int `yaml:"auto_index_length" json:"auto_index_length"` @@ -58,5 +61,7 @@ func (n *NGT) Bind() *NGT { n.DistanceType = GetActualValue(n.DistanceType) n.ObjectType = GetActualValue(n.ObjectType) n.AutoIndexCheckDuration = GetActualValue(n.AutoIndexCheckDuration) + n.AutoIndexDurationLimit = GetActualValue(n.AutoIndexDurationLimit) + n.AutoIndexSaveDuration = GetActualValue(n.AutoIndexSaveDuration) return n } diff --git a/internal/core/ngt/ngt.go b/internal/core/ngt/ngt.go index 1666e24912..b2964b3428 100644 --- a/internal/core/ngt/ngt.go +++ b/internal/core/ngt/ngt.go @@ -168,6 +168,9 @@ func gen(isLoad bool, opts ...Option) (NGT, error) { if isLoad { err = n.open() + if err != nil { + err = n.create() + } } else { err = n.create() } diff --git a/internal/core/ngt/option.go b/internal/core/ngt/option.go index 6ba34a2c08..b00b11efa3 100644 --- a/internal/core/ngt/option.go +++ b/internal/core/ngt/option.go @@ -94,7 +94,7 @@ func WithDistanceTypeByString(dt string) Option { d = Angle case "hamming": d = Hamming - case "cosine": + case "cosine", "cos": d = Cosine case "normalizedangle": d = NormalizedAngle diff --git a/pkg/agent/ngt/service/ngt.go b/pkg/agent/ngt/service/ngt.go index 2fc784a6dd..525cd577cd 100644 --- a/pkg/agent/ngt/service/ngt.go +++ b/pkg/agent/ngt/service/ngt.go @@ -69,6 +69,7 @@ type ngt struct { indexing atomic.Value lim time.Duration // auto indexing time limit dur time.Duration // auto indexing check duration + sdur time.Duration // auto save index check duration dps uint32 // default pool size ic uint64 // insert count nocie uint64 // number of create index execution @@ -79,6 +80,7 @@ type ngt struct { kvs kvs.BidiMap core core.NGT dcd bool // disable commit daemon + inMem bool } type vcache struct { @@ -92,9 +94,10 @@ const ( func New(cfg *config.NGT) (nn NGT, err error) { n := new(ngt) + n.inMem = cfg.EnableInMemoryMode cfg.IndexPath = strings.TrimSuffix(cfg.IndexPath, "/") opts := []core.Option{ - core.WithInMemoryMode(cfg.EnableInMemoryMode), + core.WithInMemoryMode(n.inMem), core.WithIndexPath(cfg.IndexPath), core.WithDimension(cfg.Dimension), core.WithDistanceTypeByString(cfg.DistanceType), @@ -104,13 +107,13 @@ func New(cfg *config.NGT) (nn NGT, err error) { core.WithSearchEdgeSize(cfg.SearchEdgeSize), } - if !cfg.EnableInMemoryMode && len(cfg.IndexPath) != 0 { + if !n.inMem && len(cfg.IndexPath) != 0 { n.path = cfg.IndexPath } n.kvs = kvs.New() - if _, err = os.Stat(cfg.IndexPath); os.IsNotExist(err) { + if _, err = os.Stat(cfg.IndexPath); os.IsNotExist(err) || n.inMem { n.core, err = core.New(opts...) } else { eg, _ := errgroup.New(context.Background()) @@ -119,7 +122,7 @@ func New(cfg *config.NGT) (nn NGT, err error) { return err })) eg.Go(safety.RecoverFunc(func() (err error) { - if len(n.path) != 0 { + if len(n.path) != 0 && !n.inMem { m := make(map[string]uint32) gob.Register(map[string]uint32{}) f := file.Open(n.path+"/"+kvsFileName, os.O_RDONLY|os.O_SYNC, os.ModePerm) @@ -156,6 +159,14 @@ func New(cfg *config.NGT) (nn NGT, err error) { n.lim = d } + if cfg.AutoIndexSaveDuration != "" { + d, err := timeutil.Parse(cfg.AutoIndexSaveDuration) + if err != nil { + d = 0 + } + n.sdur = d + } + n.alen = cfg.AutoIndexLength n.eg = errgroup.Get() @@ -183,29 +194,45 @@ func (n *ngt) Start(ctx context.Context) <-chan error { } ech := make(chan error, 2) n.eg.Go(safety.RecoverFunc(func() error { + if n.sdur == 0 { + n.sdur = n.dur + time.Second + } + if n.lim == 0 { + n.lim = n.dur * 2 + } defer close(ech) tick := time.NewTicker(n.dur) + sTick := time.NewTicker(n.sdur) limit := time.NewTicker(n.lim) defer tick.Stop() + defer sTick.Stop() defer limit.Stop() + var err error for { + err = nil select { case <-ctx.Done(): + err = n.CreateAndSaveIndex(ctx, n.dps) + if err != nil { + ech <- err + } + if err != nil { + return errors.Wrap(ctx.Err(), err.Error()) + } return ctx.Err() case <-tick.C: if int(atomic.LoadUint64(&n.ic)) >= n.alen { - err := n.CreateIndex(n.dps) - if err != nil && err != errors.ErrUncommittedIndexNotFound { - ech <- err - runtime.Gosched() - } + err = n.CreateIndex(n.dps) } case <-limit.C: - err := n.CreateIndex(n.dps) - if err != nil && err != errors.ErrUncommittedIndexNotFound { - ech <- err - runtime.Gosched() - } + err = n.CreateAndSaveIndex(ctx, n.dps) + case <-sTick.C: + err = n.SaveIndex(ctx) + } + if err != nil && err != errors.ErrUncommittedIndexNotFound { + ech <- err + runtime.Gosched() + err = nil } } })) @@ -497,28 +524,31 @@ func (n *ngt) CreateIndex(poolSize uint32) (err error) { } func (n *ngt) SaveIndex(ctx context.Context) (err error) { - eg, ctx := errgroup.New(ctx) - eg.Go(safety.RecoverFunc(func() error { - if len(n.path) != 0 { - m := make(map[string]uint32, n.kvs.Len()) - var mu sync.Mutex - n.kvs.Range(ctx, func(key string, id uint32) bool { - mu.Lock() - m[key] = id - mu.Unlock() - return true - }) - f := file.Open(n.path+"/"+kvsFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) - defer f.Close() - gob.Register(map[string]uint32{}) - return gob.NewEncoder(f).Encode(&m) - } - return nil - })) - eg.Go(safety.RecoverFunc(func() error { - return n.core.SaveIndex() - })) - return eg.Wait() + if len(n.path) != 0 && !n.inMem { + eg, ctx := errgroup.New(ctx) + eg.Go(safety.RecoverFunc(func() error { + if len(n.path) != 0 { + m := make(map[string]uint32, n.kvs.Len()) + var mu sync.Mutex + n.kvs.Range(ctx, func(key string, id uint32) bool { + mu.Lock() + m[key] = id + mu.Unlock() + return true + }) + f := file.Open(n.path+"/"+kvsFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + defer f.Close() + gob.Register(map[string]uint32{}) + return gob.NewEncoder(f).Encode(&m) + } + return nil + })) + eg.Go(safety.RecoverFunc(func() error { + return n.core.SaveIndex() + })) + err = eg.Wait() + } + return } func (n *ngt) CreateAndSaveIndex(ctx context.Context, poolSize uint32) (err error) { diff --git a/pkg/manager/index/service/indexer.go b/pkg/manager/index/service/indexer.go index 739f490a4e..28346f94b5 100644 --- a/pkg/manager/index/service/indexer.go +++ b/pkg/manager/index/service/indexer.go @@ -152,6 +152,11 @@ func (idx *index) execute(ctx context.Context, enableLowIndexSkip bool) (err err log.Debug(addr, err) return err } + _, err = agent.NewAgentClient(conn).SaveIndex(ctx, &payload.Empty{}, copts...) + if err != nil { + log.Debug(addr, err) + return err + } } return nil }) diff --git a/versions/NGT_VERSION b/versions/NGT_VERSION index 3d0e62313c..e6dbb7c238 100644 --- a/versions/NGT_VERSION +++ b/versions/NGT_VERSION @@ -1 +1 @@ -1.11.4 +1.11.5 From 28a3dfaf4a42161da721b510fed2f458d9bcf3c5 Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 18 May 2020 17:00:35 +0900 Subject: [PATCH 2/5] fix Signed-off-by: kpango --- pkg/manager/index/service/indexer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/manager/index/service/indexer.go b/pkg/manager/index/service/indexer.go index 28346f94b5..fad5ee073b 100644 --- a/pkg/manager/index/service/indexer.go +++ b/pkg/manager/index/service/indexer.go @@ -145,14 +145,15 @@ func (idx *index) execute(ctx context.Context, enableLowIndexSkip bool) (err err return nil } } - _, err := agent.NewAgentClient(conn).CreateIndex(ctx, &payload.Control_CreateIndexRequest{ + ac := agent.NewAgentClient(conn) + _, err = ac.CreateIndex(ctx, &payload.Control_CreateIndexRequest{ PoolSize: idx.creationPoolSize, }, copts...) if err != nil { log.Debug(addr, err) return err } - _, err = agent.NewAgentClient(conn).SaveIndex(ctx, &payload.Empty{}, copts...) + _, err = ac.SaveIndex(ctx, &payload.Empty{}, copts...) if err != nil { log.Debug(addr, err) return err From 608a69b1f197e392815d0d1a25e575acecdd07cb Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 18 May 2020 17:01:43 +0900 Subject: [PATCH 3/5] fix Signed-off-by: kpango --- hack/benchmark/e2e/agent/ngt/ngt_bench_test.go | 4 ++-- internal/client/agent/{ngt => }/grpc/client.go | 0 internal/client/agent/{ngt => }/grpc/client_test.go | 0 internal/client/agent/{ngt => }/grpc/option.go | 0 internal/client/agent/{ngt => }/grpc/option_test.go | 0 internal/client/agent/{ngt => }/rest/client.go | 0 internal/client/agent/{ngt => }/rest/client_test.go | 0 internal/client/agent/{ngt => }/rest/option.go | 0 internal/client/agent/{ngt => }/rest/option_test.go | 0 9 files changed, 2 insertions(+), 2 deletions(-) rename internal/client/agent/{ngt => }/grpc/client.go (100%) rename internal/client/agent/{ngt => }/grpc/client_test.go (100%) rename internal/client/agent/{ngt => }/grpc/option.go (100%) rename internal/client/agent/{ngt => }/grpc/option_test.go (100%) rename internal/client/agent/{ngt => }/rest/client.go (100%) rename internal/client/agent/{ngt => }/rest/client_test.go (100%) rename internal/client/agent/{ngt => }/rest/option.go (100%) rename internal/client/agent/{ngt => }/rest/option_test.go (100%) diff --git a/hack/benchmark/e2e/agent/ngt/ngt_bench_test.go b/hack/benchmark/e2e/agent/ngt/ngt_bench_test.go index 4b3a019719..65a43d3eeb 100644 --- a/hack/benchmark/e2e/agent/ngt/ngt_bench_test.go +++ b/hack/benchmark/e2e/agent/ngt/ngt_bench_test.go @@ -25,8 +25,8 @@ import ( "github.com/vdaas/vald/hack/benchmark/internal/e2e" "github.com/vdaas/vald/hack/benchmark/internal/e2e/strategy" "github.com/vdaas/vald/hack/benchmark/internal/starter/agent/ngt" - "github.com/vdaas/vald/internal/client/agent/ngt/grpc" - "github.com/vdaas/vald/internal/client/agent/ngt/rest" + "github.com/vdaas/vald/internal/client/agent/grpc" + "github.com/vdaas/vald/internal/client/agent/rest" "github.com/vdaas/vald/internal/log" ) diff --git a/internal/client/agent/ngt/grpc/client.go b/internal/client/agent/grpc/client.go similarity index 100% rename from internal/client/agent/ngt/grpc/client.go rename to internal/client/agent/grpc/client.go diff --git a/internal/client/agent/ngt/grpc/client_test.go b/internal/client/agent/grpc/client_test.go similarity index 100% rename from internal/client/agent/ngt/grpc/client_test.go rename to internal/client/agent/grpc/client_test.go diff --git a/internal/client/agent/ngt/grpc/option.go b/internal/client/agent/grpc/option.go similarity index 100% rename from internal/client/agent/ngt/grpc/option.go rename to internal/client/agent/grpc/option.go diff --git a/internal/client/agent/ngt/grpc/option_test.go b/internal/client/agent/grpc/option_test.go similarity index 100% rename from internal/client/agent/ngt/grpc/option_test.go rename to internal/client/agent/grpc/option_test.go diff --git a/internal/client/agent/ngt/rest/client.go b/internal/client/agent/rest/client.go similarity index 100% rename from internal/client/agent/ngt/rest/client.go rename to internal/client/agent/rest/client.go diff --git a/internal/client/agent/ngt/rest/client_test.go b/internal/client/agent/rest/client_test.go similarity index 100% rename from internal/client/agent/ngt/rest/client_test.go rename to internal/client/agent/rest/client_test.go diff --git a/internal/client/agent/ngt/rest/option.go b/internal/client/agent/rest/option.go similarity index 100% rename from internal/client/agent/ngt/rest/option.go rename to internal/client/agent/rest/option.go diff --git a/internal/client/agent/ngt/rest/option_test.go b/internal/client/agent/rest/option_test.go similarity index 100% rename from internal/client/agent/ngt/rest/option_test.go rename to internal/client/agent/rest/option_test.go From f5e54177d9864d33e9dfeb8589964c894edb7bf2 Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 18 May 2020 17:18:58 +0900 Subject: [PATCH 4/5] fix Signed-off-by: kpango --- charts/vald/values.yaml | 4 ++-- internal/config/ngt.go | 6 +++--- pkg/agent/ngt/service/ngt.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index d8926b7984..003caff19e 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -750,8 +750,8 @@ agent: auto_index_check_duration: 30m # agent.ngt.auto_index_length -- number of cache to trigger automatic indexing auto_index_length: 100 - # agent.ngt.auto_index_save_duration -- duration of automatic save index - auto_index_save_duration: 35m + # agent.ngt.auto_save_index_duration -- duration of automatic save index + auto_save_index_duration: 35m # agent.ngt.dimension -- vector dimension dimension: 4096 # agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size diff --git a/internal/config/ngt.go b/internal/config/ngt.go index 16f732ff33..e1f09cb2c5 100644 --- a/internal/config/ngt.go +++ b/internal/config/ngt.go @@ -46,8 +46,8 @@ type NGT struct { // AutoIndexCheckDuration represent checking loop duration about auto indexing execution AutoIndexCheckDuration string `yaml:"auto_index_check_duration" json:"auto_index_check_duration"` - // AutoIndexSaveDuration represent checking loop duration about auto save index execution - AutoIndexSaveDuration string `yaml:"auto_index_save_duration" json:"auto_index_save_duration"` + // AutoSaveIndexDuration represent checking loop duration about auto save index execution + AutoSaveIndexDuration string `yaml:"auto_save_index_duration" json:"auto_save_index_duration"` // AutoIndexLength represent auto index length limit AutoIndexLength int `yaml:"auto_index_length" json:"auto_index_length"` @@ -62,6 +62,6 @@ func (n *NGT) Bind() *NGT { n.ObjectType = GetActualValue(n.ObjectType) n.AutoIndexCheckDuration = GetActualValue(n.AutoIndexCheckDuration) n.AutoIndexDurationLimit = GetActualValue(n.AutoIndexDurationLimit) - n.AutoIndexSaveDuration = GetActualValue(n.AutoIndexSaveDuration) + n.AutoSaveIndexDuration = GetActualValue(n.AutoSaveIndexDuration) return n } diff --git a/pkg/agent/ngt/service/ngt.go b/pkg/agent/ngt/service/ngt.go index 1c88049154..cbc17751e7 100644 --- a/pkg/agent/ngt/service/ngt.go +++ b/pkg/agent/ngt/service/ngt.go @@ -159,8 +159,8 @@ func New(cfg *config.NGT) (nn NGT, err error) { n.lim = d } - if cfg.AutoIndexSaveDuration != "" { - d, err := timeutil.Parse(cfg.AutoIndexSaveDuration) + if cfg.AutoSaveIndexDuration != "" { + d, err := timeutil.Parse(cfg.AutoSaveIndexDuration) if err != nil { d = 0 } From f3d774b871d3583b28214770c5625f56998122b6 Mon Sep 17 00:00:00 2001 From: kpango Date: Mon, 18 May 2020 17:29:01 +0900 Subject: [PATCH 5/5] fix Signed-off-by: kpango --- internal/client/agent/grpc/client.go | 30 ++++++---------------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/internal/client/agent/grpc/client.go b/internal/client/agent/grpc/client.go index d74eaa933d..2fb029341c 100644 --- a/internal/client/agent/grpc/client.go +++ b/internal/client/agent/grpc/client.go @@ -115,10 +115,7 @@ func (c *agentClient) StreamSearch( return nil, streamSearch(st, func() interface{} { - if d := dataProvider(); d != nil { - return d - } - return nil + return dataProvider() }, f) }, ) @@ -141,10 +138,7 @@ func (c *agentClient) StreamSearchByID( return nil, streamSearch(st, func() interface{} { - if d := dataProvider(); d != nil { - return d - } - return nil + return dataProvider() }, f, ) }, @@ -180,10 +174,7 @@ func (c *agentClient) StreamInsert( return nil, stream(st, func() interface{} { - if d := dataProvider(); d != nil { - return d - } - return nil + return dataProvider() }, f, ) }, @@ -231,10 +222,7 @@ func (c *agentClient) StreamUpdate( return nil, stream(st, func() interface{} { - if d := dataProvider(); d != nil { - return d - } - return nil + return dataProvider() }, f, ) }, @@ -280,10 +268,7 @@ func (c *agentClient) StreamRemove( return nil, stream(st, func() interface{} { - if d := dataProvider(); d != nil { - return d - } - return nil + return dataProvider() }, f, ) }, @@ -334,10 +319,7 @@ func (c *agentClient) StreamGetObject( return nil, grpc.BidirectionalStreamClient(st, func() interface{} { - if d := dataProvider(); d != nil { - return d - } - return nil + return dataProvider() }, func() interface{} { return new(client.ObjectVector) }, func(res interface{}, err error) {