diff --git a/cmd/agent/core/ngt/sample.yaml b/cmd/agent/core/ngt/sample.yaml index cfca962b82..4aac23e307 100644 --- a/cmd/agent/core/ngt/sample.yaml +++ b/cmd/agent/core/ngt/sample.yaml @@ -152,7 +152,7 @@ ngt: bulk_insert_chunk_size: 10 creation_edge_size: 20 default_epsilon: 0.01 - default_pool_size: 10000 + default_pool_size: 100 default_radius: -1 dimension: 6 distance_type: l2 diff --git a/go.mod b/go.mod index 4956750da0..1a73d9ac27 100755 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.16 replace ( cloud.google.com/go => cloud.google.com/go v0.77.0 github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.37.14 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.37.15 github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1 github.com/chzyer/logex => github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2 github.com/coreos/etcd => go.etcd.io/etcd v3.3.25+incompatible @@ -21,7 +21,7 @@ replace ( github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.15.0 github.com/gorilla/websocket => github.com/gorilla/websocket v1.4.2 github.com/hailocab/go-hostpool => github.com/monzo/go-hostpool v0.0.0-20200724120130-287edbb29340 - github.com/klauspost/compress => github.com/klauspost/compress v1.11.8-0.20210205131735-b5cd74455ec9 + github.com/klauspost/compress => github.com/klauspost/compress v1.11.8-0.20210219140231-0afc4af7e6ba github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.2+incompatible golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df google.golang.org/grpc => google.golang.org/grpc v1.35.0 @@ -75,11 +75,11 @@ require ( go.uber.org/zap v1.16.0 golang.org/x/net v0.0.0-20210119194325-5f4716e94777 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a - golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b + golang.org/x/sys v0.0.0-20210219172841-57ea560cfca1 gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614 gonum.org/v1/plot v0.8.1 google.golang.org/api v0.40.0 - google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 + google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b google.golang.org/grpc v1.35.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.20.4 diff --git a/go.sum b/go.sum index c348c9f937..bff893de80 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.37.14 h1:thuR1hd1doCvsaMDYDMhqCGSmw39bSvZaw+DPGhMm5w= -github.com/aws/aws-sdk-go v1.37.14/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.37.15 h1:W7l7gLLMcYRlg6a+uvf3Zz4jYwdqYzhe5ymqwWoOhp4= +github.com/aws/aws-sdk-go v1.37.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -296,8 +296,8 @@ github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5Pt github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.8-0.20210205131735-b5cd74455ec9 h1:OAhCmjDqWViQo/ZB+YeKShJ4DFrkmVRWRUGgzzJLs28= -github.com/klauspost/compress v1.11.8-0.20210205131735-b5cd74455ec9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.8-0.20210219140231-0afc4af7e6ba h1:QdsKSRMQoB/H6WIbEwhTV/SeafLuYELLmbxoKEf1En4= +github.com/klauspost/compress v1.11.8-0.20210219140231-0afc4af7e6ba/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -685,8 +685,8 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g= -golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210219172841-57ea560cfca1 h1:mDSj8NPponP6fRpRDblAGl5bpSHjPulHtk5lGl0gLSY= +golang.org/x/sys v0.0.0-20210219172841-57ea560cfca1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -813,8 +813,8 @@ google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210212180131-e7f2df4ecc2d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06 h1:Px6YyLaNKEo5eoniIBAv6Es0jbvyEmSYqOac64iS2Rs= -google.golang.org/genproto v0.0.0-20210218151259-fe80b386bf06/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b h1:zTeTu5p/EXQSqNJboHUw32wdNFYQTT9vSc+ibSpCoLk= +google.golang.org/genproto v0.0.0-20210219173056-d891e3cb3b5b/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= diff --git a/internal/config/net_test.go b/internal/config/net_test.go index d27fc726ef..2917198145 100644 --- a/internal/config/net_test.go +++ b/internal/config/net_test.go @@ -187,7 +187,7 @@ func TestDialer_Bind(t *testing.T) { } } -func TestTCP_Bind(t *testing.T) { +func TestNet_Bind(t *testing.T) { type fields struct { DNS *DNS Dialer *Dialer @@ -267,7 +267,7 @@ func TestTCP_Bind(t *testing.T) { } } -func TestTCP_Opts(t *testing.T) { +func TestNet_Opts(t *testing.T) { type fields struct { DNS *DNS Dialer *Dialer diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go index f119b9c294..bbefc1df61 100644 --- a/internal/core/algorithm/ngt/ngt.go +++ b/internal/core/algorithm/ngt/ngt.go @@ -499,16 +499,14 @@ func (n *ngt) Remove(id uint) error { } // BulkRemove removes multiple index from NGT index. -func (n *ngt) BulkRemove(ids ...uint) error { - n.mu.Lock() - defer n.mu.Unlock() - for _, id := range ids { - if C.ngt_remove_index(n.index, C.ObjectID(id), n.ebuf) == ErrorCode { - ne := n.ebuf - return n.newGoError(ne) +func (n *ngt) BulkRemove(ids ...uint) (errs error) { + for i, id := range ids { + err := n.Remove(id) + if err != nil { + errs = errors.Wrapf(errs, "bulkremove error detected index number: %d,\tid: %d\terr: %v", i, id, err) } } - return nil + return errs } // GetVector returns vector stored in NGT index. diff --git a/internal/errors/vqueue.go b/internal/errors/vqueue.go new file mode 100644 index 0000000000..b2fc5b4a01 --- /dev/null +++ b/internal/errors/vqueue.go @@ -0,0 +1,20 @@ +// +// Copyright (C) 2019-2021 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package errors provides error types and function +package errors + +var ErrVQueueFinalizing = New("error vector queue is now finalizing...") diff --git a/pkg/agent/core/ngt/handler/grpc/handler_test.go b/pkg/agent/core/ngt/handler/grpc/handler_test.go index 2afa9cee2a..b3d25d4aeb 100644 --- a/pkg/agent/core/ngt/handler/grpc/handler_test.go +++ b/pkg/agent/core/ngt/handler/grpc/handler_test.go @@ -431,8 +431,8 @@ func Test_server_Search(t *testing.T) { streamConcurrency int } type want struct { - want *payload.Search_Response - err error + wantRes *payload.Search_Response + err error } type test struct { name string @@ -443,12 +443,12 @@ func Test_server_Search(t *testing.T) { beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got *payload.Search_Response, err error) error { + defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) } - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + if !reflect.DeepEqual(gotRes, w.wantRes) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) } return nil } @@ -518,8 +518,8 @@ func Test_server_Search(t *testing.T) { streamConcurrency: test.fields.streamConcurrency, } - got, err := s.Search(test.args.ctx, test.args.req) - if err := test.checkFunc(test.want, got, err); err != nil { + gotRes, err := s.Search(test.args.ctx, test.args.req) + if err := test.checkFunc(test.want, gotRes, err); err != nil { tt.Errorf("error = %v", err) } }) @@ -540,8 +540,8 @@ func Test_server_SearchByID(t *testing.T) { streamConcurrency int } type want struct { - want *payload.Search_Response - err error + wantRes *payload.Search_Response + err error } type test struct { name string @@ -552,12 +552,12 @@ func Test_server_SearchByID(t *testing.T) { beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got *payload.Search_Response, err error) error { + defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) } - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + if !reflect.DeepEqual(gotRes, w.wantRes) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) } return nil } @@ -627,8 +627,8 @@ func Test_server_SearchByID(t *testing.T) { streamConcurrency: test.fields.streamConcurrency, } - got, err := s.SearchByID(test.args.ctx, test.args.req) - if err := test.checkFunc(test.want, got, err); err != nil { + gotRes, err := s.SearchByID(test.args.ctx, test.args.req) + if err := test.checkFunc(test.want, gotRes, err); err != nil { tt.Errorf("error = %v", err) } }) @@ -1791,8 +1791,8 @@ func Test_server_Upsert(t *testing.T) { streamConcurrency int } type want struct { - want *payload.Object_Location - err error + wantLoc *payload.Object_Location + err error } type test struct { name string @@ -1803,12 +1803,12 @@ func Test_server_Upsert(t *testing.T) { beforeFunc func(args) afterFunc func(args) } - defaultCheckFunc := func(w want, got *payload.Object_Location, err error) error { + defaultCheckFunc := func(w want, gotLoc *payload.Object_Location, err error) error { if !errors.Is(err, w.err) { return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) } - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + if !reflect.DeepEqual(gotLoc, w.wantLoc) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotLoc, w.wantLoc) } return nil } @@ -1878,8 +1878,8 @@ func Test_server_Upsert(t *testing.T) { streamConcurrency: test.fields.streamConcurrency, } - got, err := s.Upsert(test.args.ctx, test.args.req) - if err := test.checkFunc(test.want, got, err); err != nil { + gotLoc, err := s.Upsert(test.args.ctx, test.args.req) + if err := test.checkFunc(test.want, gotLoc, err); err != nil { tt.Errorf("error = %v", err) } }) diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index dfcc4579cb..ef16cbde8d 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -39,6 +39,7 @@ import ( "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue" "github.com/vdaas/vald/pkg/agent/internal/metadata" ) @@ -63,7 +64,6 @@ type NGT interface { NumberOfCreateIndexExecution() uint64 NumberOfProactiveGCExecution() uint64 UUIDs(context.Context) (uuids []string) - UncommittedUUIDs() (uuids []string) DeleteVCacheLen() uint64 InsertVCacheLen() uint64 Close(ctx context.Context) error @@ -74,8 +74,7 @@ type ngt struct { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches // insertion vector cache - dvc *vcaches // deletion vector cache + vq vqueue.Queue // statuses indexing atomic.Value @@ -83,7 +82,6 @@ type ngt struct { lastNoice uint64 // last number of create index execution this value prevent unnecessary saveindex. // counters - ic uint64 // insert count nocie uint64 // number of create index execution nogce uint64 // number of proactive GC execution @@ -112,11 +110,6 @@ type ngt struct { dcd bool // disable commit daemon } -type vcache struct { - vector []float32 - date int64 -} - const ( kvsFileName = "ngt-meta.kvsdb" ) @@ -152,11 +145,8 @@ func New(cfg *config.NGT, opts ...Option) (nn NGT, err error) { if n.dur == 0 || n.alen == 0 { n.dcd = true } - if n.ivc == nil { - n.ivc = new(vcaches) - } - if n.dvc == nil { - n.dvc = new(vcaches) + if n.vq == nil { + n.vq = vqueue.New(n.eg) } n.indexing.Store(false) @@ -312,6 +302,10 @@ func (n *ngt) Start(ctx context.Context) <-chan error { } ech := make(chan error, 2) n.eg.Go(safety.RecoverFunc(func() (err error) { + vqech, err := n.vq.Start(ctx) + if err != nil { + return err + } if n.sdur == 0 { n.sdur = n.dur + time.Second } @@ -346,13 +340,18 @@ func (n *ngt) Start(ctx context.Context) <-chan error { } return ctx.Err() case <-tick.C: - if int(atomic.LoadUint64(&n.ic)) >= n.alen { + if n.vq.IVQLen() >= n.alen { err = n.CreateIndex(ctx, n.poolSize) } case <-limit.C: err = n.CreateAndSaveIndex(ctx, n.poolSize) case <-sTick.C: err = n.SaveIndex(ctx) + case err := <-vqech: + if err != nil { + ech <- err + err = nil + } } if err != nil && err != errors.ErrUncommittedIndexNotFound { ech <- err @@ -361,7 +360,6 @@ func (n *ngt) Start(ctx context.Context) <-chan error { } } })) - return ech } @@ -386,6 +384,8 @@ func (n *ngt) Search(vec []float32, size uint32, epsilon, radius float32) ([]mod ID: key, Distance: d.Distance, }) + } else { + log.Warn("not found", d.ID, d.Distance) } } @@ -421,18 +421,8 @@ func (n *ngt) insert(uuid string, vec []float32, t int64, validation bool) (err err = errors.ErrUUIDAlreadyExists(uuid, uint(id)) return err } - _, ok = n.insertCache(uuid) - if ok { - err = errors.ErrUUIDAlreadyExists(uuid, uint(id)) - return err - } } - n.ivc.Store(uuid, vcache{ - vector: vec, - date: t, - }) - atomic.AddUint64(&n.ic, 1) - return nil + return n.vq.PushInsert(uuid, vec, t) } func (n *ngt) InsertMultiple(vecs map[string][]float32) (err error) { @@ -474,26 +464,9 @@ func (n *ngt) UpdateMultiple(vecs map[string][]float32) (err error) { } err = n.DeleteMultiple(uuids...) if err != nil { - for _, uuid := range uuids { - n.dvc.Delete(uuid) - } return err } - t := time.Now().UnixNano() - for uuid, vec := range vecs { - ierr := n.insert(uuid, vec, t, false) - if ierr != nil { - n.dvc.Delete(uuid) - n.ivc.Delete(uuid) - atomic.AddUint64(&n.ic, ^uint64(0)) - if err != nil { - err = errors.Wrap(ierr, err.Error()) - } else { - err = ierr - } - } - } - return err + return n.InsertMultiple(vecs) } func (n *ngt) Delete(uuid string) (err error) { @@ -507,20 +480,9 @@ func (n *ngt) delete(uuid string, t int64) (err error) { } _, ok := n.kvs.Get(uuid) if !ok { - _, ok := n.insertCache(uuid) - if !ok { - err = errors.ErrObjectIDNotFound(uuid) - return err - } - } - if vc, ok := n.ivc.Load(uuid); ok && vc.date < t { - n.ivc.Delete(uuid) - atomic.AddUint64(&n.ic, ^uint64(0)) + return errors.ErrObjectIDNotFound(uuid) } - n.dvc.Store(uuid, vcache{ - date: t, - }) - return nil + return n.vq.PushDelete(uuid, t) } func (n *ngt) DeleteMultiple(uuids ...string) (err error) { @@ -541,13 +503,13 @@ func (n *ngt) DeleteMultiple(uuids ...string) (err error) { func (n *ngt) GetObject(uuid string) (vec []float32, err error) { oid, ok := n.kvs.Get(uuid) if !ok { - log.Debugf("GetObject\tuuid: %s's kvs data not found, trying to read from vcache", uuid) - ivc, ok := n.insertCache(uuid) + log.Debugf("GetObject\tuuid: %s's kvs data not found, trying to read from vqueue", uuid) + vec, ok := n.vq.GetVector(uuid) if !ok { log.Debugf("GetObject\tuuid: %s's vcache data not found", uuid) return nil, errors.ErrObjectIDNotFound(uuid) } - return ivc.vector, nil + return vec, nil } log.Debugf("GetObject\tGetVector oid: %d", oid) vec, err = n.core.GetVector(uint(oid)) @@ -569,89 +531,46 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) { if n.IsIndexing() || n.IsSaving() { return nil } - ic := atomic.LoadUint64(&n.ic) + ic := n.vq.IVQLen() + n.vq.DVQLen() if ic == 0 { return errors.ErrUncommittedIndexNotFound } n.indexing.Store(true) - atomic.StoreUint64(&n.ic, 0) - t := time.Now().UnixNano() defer n.indexing.Store(false) defer n.gc() log.Infof("create index operation started, uncommitted indexes = %d", ic) - delList := make([]string, 0, ic) - n.dvc.Range(func(uuid string, dvc vcache) bool { - if dvc.date > t { - return true - } - if ivc, ok := n.ivc.Load(uuid); ok && ivc.date < t && ivc.date < dvc.date { - n.ivc.Delete(uuid) + log.Info("create index delete phase started") + n.vq.RangePopDelete(ctx, func(uuid string) bool { + var ierr error + oid, ok := n.kvs.Delete(uuid) + if ok { + ierr = n.core.Remove(uint(oid)) + if ierr != nil { + log.Error(ierr) + err = errors.Wrap(err, ierr.Error()) + } + } else { + ierr = errors.ErrObjectIDNotFound(uuid) + log.Error(ierr) + err = errors.Wrap(err, ierr.Error()) } - delList = append(delList, uuid) return true }) - log.Info("create index delete kvs phase started") - log.Debugf("deleting kvs: %#v", delList) - doids := make([]uint, 0, ic) - for _, duuid := range delList { - n.dvc.Delete(duuid) - id, ok := n.kvs.Delete(duuid) - if !ok { - log.Error(errors.ErrObjectIDNotFound(duuid).Error()) - err = errors.Wrap(err, errors.ErrObjectIDNotFound(duuid).Error()) + log.Info("create index delete phase finished") + n.gc() + log.Info("create index insert phase started") + n.vq.RangePopInsert(ctx, func(uuid string, vector []float32) bool { + oid, ierr := n.core.Insert(vector) + if ierr != nil { + log.Error(ierr) + err = errors.Wrap(err, ierr.Error()) } else { - doids = append(doids, uint(id)) - } - } - log.Info("create index delete kvs phase finished") - - log.Info("create index delete index phase started") - log.Debugf("deleting index: %#v", doids) - brerr := n.core.BulkRemove(doids...) - log.Info("create index delete index phase finished") - if brerr != nil { - log.Error("an error occurred on deleting index phase:", brerr) - err = errors.Wrap(err, brerr.Error()) - } - uuids := make([]string, 0, ic) - vecs := make([][]float32, 0, ic) - n.ivc.Range(func(uuid string, ivc vcache) bool { - if ivc.date <= t { - uuids = append(uuids, uuid) - vecs = append(vecs, ivc.vector) + n.kvs.Set(uuid, uint32(oid)) } return true }) - for _, uuid := range uuids { - n.ivc.Delete(uuid) - } - n.gc() - log.Info("create index insert index phase started") - log.Debugf("inserting index: %#v", vecs) - oids, errs := n.core.BulkInsert(vecs) - log.Info("create index insert index phase finished") - if errs != nil && len(errs) != 0 { - for _, bierr := range errs { - if bierr != nil { - log.Error("an error occurred on inserting index phase:", bierr) - err = errors.Wrap(err, bierr.Error()) - } - } - } - - log.Info("create index insert kvs phase started") - log.Debugf("uuids = %#v\t\toids = %#v", uuids, oids) - for i, uuid := range uuids { - if len(oids) > i { - oid := uint32(oids[i]) - if oid != 0 { - n.kvs.Set(uuid, oid) - } - } - } - log.Info("create index insert kvs phase finished") - + log.Info("create index insert phase finished") log.Info("create graph and tree phase started") log.Debugf("pool size = %d", poolSize) cierr := n.core.CreateIndex(poolSize) @@ -766,7 +685,7 @@ func (n *ngt) CreateAndSaveIndex(ctx context.Context, poolSize uint32) (err erro func (n *ngt) Exists(uuid string) (oid uint32, ok bool) { oid, ok = n.kvs.Get(uuid) if !ok { - _, ok = n.insertCache(uuid) + _, ok = n.vq.GetVector(uuid) } return oid, ok } @@ -790,22 +709,6 @@ func (n *ngt) readyForUpdate(uuid string, vec []float32) (ready bool) { return false } -func (n *ngt) insertCache(uuid string) (*vcache, bool) { - iv, ok := n.ivc.Load(uuid) - if ok { - dv, ok := n.dvc.Load(uuid) - if !ok { - return &iv, true - } - if ok && dv.date <= iv.date { - return &iv, true - } - n.ivc.Delete(uuid) - atomic.AddUint64(&n.ic, ^uint64(0)) - } - return nil, false -} - func (n *ngt) IsSaving() bool { s, ok := n.saving.Load().(bool) return s && ok @@ -825,18 +728,6 @@ func (n *ngt) UUIDs(ctx context.Context) (uuids []string) { return uuids } -func (n *ngt) UncommittedUUIDs() (uuids []string) { - var mu sync.Mutex - uuids = make([]string, 0, atomic.LoadUint64(&n.ic)) - n.ivc.Range(func(uuid string, vc vcache) bool { - mu.Lock() - uuids = append(uuids, uuid) - mu.Unlock() - return true - }) - return uuids -} - func (n *ngt) NumberOfCreateIndexExecution() uint64 { return atomic.LoadUint64(&n.nocie) } @@ -857,11 +748,11 @@ func (n *ngt) Len() uint64 { } func (n *ngt) InsertVCacheLen() uint64 { - return n.ivc.Len() + return uint64(n.vq.IVQLen()) } func (n *ngt) DeleteVCacheLen() uint64 { - return n.dvc.Len() + return uint64(n.vq.DVQLen()) } func (n *ngt) Close(ctx context.Context) (err error) { diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go index a97defb030..940219088a 100644 --- a/pkg/agent/core/ngt/service/ngt_test.go +++ b/pkg/agent/core/ngt/service/ngt_test.go @@ -30,6 +30,7 @@ import ( "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/pkg/agent/core/ngt/model" "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue" "go.uber.org/goleak" ) @@ -122,12 +123,10 @@ func Test_ngt_initNGT(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -176,12 +175,10 @@ func Test_ngt_initNGT(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -217,12 +214,10 @@ func Test_ngt_initNGT(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -266,12 +261,10 @@ func Test_ngt_initNGT(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -305,12 +298,10 @@ func Test_ngt_loadKVS(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -355,12 +346,10 @@ func Test_ngt_loadKVS(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -393,12 +382,10 @@ func Test_ngt_loadKVS(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -442,12 +429,10 @@ func Test_ngt_loadKVS(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -484,12 +469,10 @@ func Test_ngt_Start(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -538,12 +521,10 @@ func Test_ngt_Start(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -579,12 +560,10 @@ func Test_ngt_Start(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -628,12 +607,10 @@ func Test_ngt_Start(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -673,12 +650,10 @@ func Test_ngt_Search(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -734,12 +709,10 @@ func Test_ngt_Search(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -778,12 +751,10 @@ func Test_ngt_Search(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -827,12 +798,10 @@ func Test_ngt_Search(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -872,12 +841,10 @@ func Test_ngt_SearchByID(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -933,12 +900,10 @@ func Test_ngt_SearchByID(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -977,12 +942,10 @@ func Test_ngt_SearchByID(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1026,12 +989,10 @@ func Test_ngt_SearchByID(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -1069,12 +1030,10 @@ func Test_ngt_Insert(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -1124,12 +1083,10 @@ func Test_ngt_Insert(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1166,12 +1123,10 @@ func Test_ngt_Insert(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1215,12 +1170,10 @@ func Test_ngt_Insert(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -1260,12 +1213,10 @@ func Test_ngt_insert(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -1317,12 +1268,10 @@ func Test_ngt_insert(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1361,12 +1310,10 @@ func Test_ngt_insert(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1410,12 +1357,10 @@ func Test_ngt_insert(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -1452,12 +1397,10 @@ func Test_ngt_InsertMultiple(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -1506,12 +1449,10 @@ func Test_ngt_InsertMultiple(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1547,12 +1488,10 @@ func Test_ngt_InsertMultiple(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1596,12 +1535,10 @@ func Test_ngt_InsertMultiple(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -1639,12 +1576,10 @@ func Test_ngt_Update(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -1694,12 +1629,10 @@ func Test_ngt_Update(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1736,12 +1669,10 @@ func Test_ngt_Update(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1785,12 +1716,10 @@ func Test_ngt_Update(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -1827,12 +1756,10 @@ func Test_ngt_UpdateMultiple(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -1881,12 +1808,10 @@ func Test_ngt_UpdateMultiple(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1922,12 +1847,10 @@ func Test_ngt_UpdateMultiple(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -1971,12 +1894,10 @@ func Test_ngt_UpdateMultiple(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -2013,12 +1934,10 @@ func Test_ngt_Delete(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -2067,12 +1986,10 @@ func Test_ngt_Delete(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2108,12 +2025,10 @@ func Test_ngt_Delete(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2157,12 +2072,10 @@ func Test_ngt_Delete(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -2200,12 +2113,10 @@ func Test_ngt_delete(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -2255,12 +2166,10 @@ func Test_ngt_delete(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2297,12 +2206,10 @@ func Test_ngt_delete(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2346,12 +2253,10 @@ func Test_ngt_delete(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -2388,12 +2293,10 @@ func Test_ngt_DeleteMultiple(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -2442,12 +2345,10 @@ func Test_ngt_DeleteMultiple(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2483,12 +2384,10 @@ func Test_ngt_DeleteMultiple(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2532,12 +2431,10 @@ func Test_ngt_DeleteMultiple(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -2574,12 +2471,10 @@ func Test_ngt_GetObject(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -2632,12 +2527,10 @@ func Test_ngt_GetObject(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2673,12 +2566,10 @@ func Test_ngt_GetObject(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2722,12 +2613,10 @@ func Test_ngt_GetObject(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -2765,12 +2654,10 @@ func Test_ngt_CreateIndex(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -2820,12 +2707,10 @@ func Test_ngt_CreateIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2862,12 +2747,10 @@ func Test_ngt_CreateIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -2911,12 +2794,10 @@ func Test_ngt_CreateIndex(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -2953,12 +2834,10 @@ func Test_ngt_SaveIndex(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -3007,12 +2886,10 @@ func Test_ngt_SaveIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3048,12 +2925,10 @@ func Test_ngt_SaveIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3097,12 +2972,10 @@ func Test_ngt_SaveIndex(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -3139,12 +3012,10 @@ func Test_ngt_saveIndex(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -3193,12 +3064,10 @@ func Test_ngt_saveIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3234,12 +3103,10 @@ func Test_ngt_saveIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3283,12 +3150,10 @@ func Test_ngt_saveIndex(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -3326,12 +3191,10 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -3381,12 +3244,10 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3423,12 +3284,10 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3472,12 +3331,10 @@ func Test_ngt_CreateAndSaveIndex(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -3514,12 +3371,10 @@ func Test_ngt_Exists(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -3572,12 +3427,10 @@ func Test_ngt_Exists(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3613,12 +3466,10 @@ func Test_ngt_Exists(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3662,12 +3513,10 @@ func Test_ngt_Exists(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -3705,12 +3554,10 @@ func Test_ngt_readyForUpdate(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -3760,12 +3607,10 @@ func Test_ngt_readyForUpdate(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3802,12 +3647,10 @@ func Test_ngt_readyForUpdate(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -3851,12 +3694,10 @@ func Test_ngt_readyForUpdate(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -3884,208 +3725,16 @@ func Test_ngt_readyForUpdate(t *testing.T) { } } -func Test_ngt_insertCache(t *testing.T) { - t.Parallel() - type args struct { - uuid string - } - type fields struct { - core core.NGT - eg errgroup.Group - kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches - indexing atomic.Value - saving atomic.Value - lastNoice uint64 - ic uint64 - nocie uint64 - nogce uint64 - inMem bool - alen int - lim time.Duration - dur time.Duration - sdur time.Duration - minLit time.Duration - maxLit time.Duration - litFactor time.Duration - enableProactiveGC bool - path string - poolSize uint32 - radius float32 - epsilon float32 - idelay time.Duration - dcd bool - } - type want struct { - want *vcache - want1 bool - } - type test struct { - name string - args args - fields fields - want want - checkFunc func(want, *vcache, bool) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, got *vcache, got1 bool) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - if !reflect.DeepEqual(got1, w.want1) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - uuid: "", - }, - fields: fields { - core: nil, - eg: nil, - kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, - indexing: nil, - saving: nil, - lastNoice: 0, - ic: 0, - nocie: 0, - nogce: 0, - inMem: false, - alen: 0, - lim: nil, - dur: nil, - sdur: nil, - minLit: nil, - maxLit: nil, - litFactor: nil, - enableProactiveGC: false, - path: "", - poolSize: 0, - radius: 0, - epsilon: 0, - idelay: nil, - dcd: false, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - uuid: "", - }, - fields: fields { - core: nil, - eg: nil, - kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, - indexing: nil, - saving: nil, - lastNoice: 0, - ic: 0, - nocie: 0, - nogce: 0, - inMem: false, - alen: 0, - lim: nil, - dur: nil, - sdur: nil, - minLit: nil, - maxLit: nil, - litFactor: nil, - enableProactiveGC: false, - path: "", - poolSize: 0, - radius: 0, - epsilon: 0, - idelay: nil, - dcd: false, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - n := &ngt{ - core: test.fields.core, - eg: test.fields.eg, - kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - indexing: test.fields.indexing, - saving: test.fields.saving, - lastNoice: test.fields.lastNoice, - ic: test.fields.ic, - nocie: test.fields.nocie, - nogce: test.fields.nogce, - inMem: test.fields.inMem, - alen: test.fields.alen, - lim: test.fields.lim, - dur: test.fields.dur, - sdur: test.fields.sdur, - minLit: test.fields.minLit, - maxLit: test.fields.maxLit, - litFactor: test.fields.litFactor, - enableProactiveGC: test.fields.enableProactiveGC, - path: test.fields.path, - poolSize: test.fields.poolSize, - radius: test.fields.radius, - epsilon: test.fields.epsilon, - idelay: test.fields.idelay, - dcd: test.fields.dcd, - } - - got, got1 := n.insertCache(test.args.uuid) - if err := test.checkFunc(test.want, got, got1); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - func Test_ngt_IsSaving(t *testing.T) { t.Parallel() type fields struct { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -4130,12 +3779,10 @@ func Test_ngt_IsSaving(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4168,12 +3815,10 @@ func Test_ngt_IsSaving(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4217,12 +3862,10 @@ func Test_ngt_IsSaving(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -4256,12 +3899,10 @@ func Test_ngt_IsIndexing(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -4306,12 +3947,10 @@ func Test_ngt_IsIndexing(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4344,12 +3983,10 @@ func Test_ngt_IsIndexing(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4393,12 +4030,10 @@ func Test_ngt_IsIndexing(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -4435,12 +4070,10 @@ func Test_ngt_UUIDs(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -4489,12 +4122,10 @@ func Test_ngt_UUIDs(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4530,12 +4161,10 @@ func Test_ngt_UUIDs(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4579,12 +4208,10 @@ func Test_ngt_UUIDs(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -4612,194 +4239,16 @@ func Test_ngt_UUIDs(t *testing.T) { } } -func Test_ngt_UncommittedUUIDs(t *testing.T) { - t.Parallel() - type fields struct { - core core.NGT - eg errgroup.Group - kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches - indexing atomic.Value - saving atomic.Value - lastNoice uint64 - ic uint64 - nocie uint64 - nogce uint64 - inMem bool - alen int - lim time.Duration - dur time.Duration - sdur time.Duration - minLit time.Duration - maxLit time.Duration - litFactor time.Duration - enableProactiveGC bool - path string - poolSize uint32 - radius float32 - epsilon float32 - idelay time.Duration - dcd bool - } - type want struct { - wantUuids []string - } - type test struct { - name string - fields fields - want want - checkFunc func(want, []string) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, gotUuids []string) error { - if !reflect.DeepEqual(gotUuids, w.wantUuids) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotUuids, w.wantUuids) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - core: nil, - eg: nil, - kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, - indexing: nil, - saving: nil, - lastNoice: 0, - ic: 0, - nocie: 0, - nogce: 0, - inMem: false, - alen: 0, - lim: nil, - dur: nil, - sdur: nil, - minLit: nil, - maxLit: nil, - litFactor: nil, - enableProactiveGC: false, - path: "", - poolSize: 0, - radius: 0, - epsilon: 0, - idelay: nil, - dcd: false, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - core: nil, - eg: nil, - kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, - indexing: nil, - saving: nil, - lastNoice: 0, - ic: 0, - nocie: 0, - nogce: 0, - inMem: false, - alen: 0, - lim: nil, - dur: nil, - sdur: nil, - minLit: nil, - maxLit: nil, - litFactor: nil, - enableProactiveGC: false, - path: "", - poolSize: 0, - radius: 0, - epsilon: 0, - idelay: nil, - dcd: false, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - n := &ngt{ - core: test.fields.core, - eg: test.fields.eg, - kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, - indexing: test.fields.indexing, - saving: test.fields.saving, - lastNoice: test.fields.lastNoice, - ic: test.fields.ic, - nocie: test.fields.nocie, - nogce: test.fields.nogce, - inMem: test.fields.inMem, - alen: test.fields.alen, - lim: test.fields.lim, - dur: test.fields.dur, - sdur: test.fields.sdur, - minLit: test.fields.minLit, - maxLit: test.fields.maxLit, - litFactor: test.fields.litFactor, - enableProactiveGC: test.fields.enableProactiveGC, - path: test.fields.path, - poolSize: test.fields.poolSize, - radius: test.fields.radius, - epsilon: test.fields.epsilon, - idelay: test.fields.idelay, - dcd: test.fields.dcd, - } - - gotUuids := n.UncommittedUUIDs() - if err := test.checkFunc(test.want, gotUuids); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { t.Parallel() type fields struct { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -4844,12 +4293,10 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4882,12 +4329,10 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -4931,12 +4376,10 @@ func Test_ngt_NumberOfCreateIndexExecution(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -4970,12 +4413,10 @@ func Test_ngt_NumberOfProactiveGCExecution(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -5020,12 +4461,10 @@ func Test_ngt_NumberOfProactiveGCExecution(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5058,12 +4497,10 @@ func Test_ngt_NumberOfProactiveGCExecution(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5107,12 +4544,10 @@ func Test_ngt_NumberOfProactiveGCExecution(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -5146,12 +4581,10 @@ func Test_ngt_gc(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -5191,12 +4624,10 @@ func Test_ngt_gc(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5229,12 +4660,10 @@ func Test_ngt_gc(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5278,12 +4707,10 @@ func Test_ngt_gc(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -5317,12 +4744,10 @@ func Test_ngt_Len(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -5367,12 +4792,10 @@ func Test_ngt_Len(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5405,12 +4828,10 @@ func Test_ngt_Len(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5454,12 +4875,10 @@ func Test_ngt_Len(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -5493,12 +4912,10 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -5543,12 +4960,10 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5581,12 +4996,10 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5630,12 +5043,10 @@ func Test_ngt_InsertVCacheLen(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -5669,12 +5080,10 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -5719,12 +5128,10 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5757,12 +5164,10 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5806,12 +5211,10 @@ func Test_ngt_DeleteVCacheLen(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, @@ -5848,12 +5251,10 @@ func Test_ngt_Close(t *testing.T) { core core.NGT eg errgroup.Group kvs kvs.BidiMap - ivc *vcaches - dvc *vcaches + vq vqueue.Queue indexing atomic.Value saving atomic.Value lastNoice uint64 - ic uint64 nocie uint64 nogce uint64 inMem bool @@ -5902,12 +5303,10 @@ func Test_ngt_Close(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5943,12 +5342,10 @@ func Test_ngt_Close(t *testing.T) { core: nil, eg: nil, kvs: nil, - ivc: vcaches{}, - dvc: vcaches{}, + vq: nil, indexing: nil, saving: nil, lastNoice: 0, - ic: 0, nocie: 0, nogce: 0, inMem: false, @@ -5992,12 +5389,10 @@ func Test_ngt_Close(t *testing.T) { core: test.fields.core, eg: test.fields.eg, kvs: test.fields.kvs, - ivc: test.fields.ivc, - dvc: test.fields.dvc, + vq: test.fields.vq, indexing: test.fields.indexing, saving: test.fields.saving, lastNoice: test.fields.lastNoice, - ic: test.fields.ic, nocie: test.fields.nocie, nogce: test.fields.nogce, inMem: test.fields.inMem, diff --git a/pkg/agent/core/ngt/service/vcaches.go b/pkg/agent/core/ngt/service/vcaches.go deleted file mode 100644 index bbeb77fa7a..0000000000 --- a/pkg/agent/core/ngt/service/vcaches.go +++ /dev/null @@ -1,213 +0,0 @@ -// -// Copyright (C) 2019-2021 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package service - -import ( - "sync" - "sync/atomic" - "unsafe" -) - -type vcaches struct { - length uint64 - mu sync.Mutex - read atomic.Value // readOnly - dirty map[string]*entryVCache - misses int -} - -type readOnlyVCache struct { - m map[string]*entryVCache - amended bool -} - -var expungedVCache = unsafe.Pointer(new(vcache)) - -type entryVCache struct { - p unsafe.Pointer -} - -func newEntryVCache(i vcache) *entryVCache { - return &entryVCache{p: unsafe.Pointer(&i)} -} - -func (m *vcaches) Load(key string) (value vcache, ok bool) { - read, _ := m.read.Load().(readOnlyVCache) - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - read, _ = m.read.Load().(readOnlyVCache) - e, ok = read.m[key] - if !ok && read.amended { - e, ok = m.dirty[key] - m.missLocked() - } - m.mu.Unlock() - } - if !ok { - return value, false - } - return e.load() -} - -func (e *entryVCache) load() (value vcache, ok bool) { - p := atomic.LoadPointer(&e.p) - if p == nil || p == expungedVCache { - return value, false - } - return *(*vcache)(p), true -} - -func (m *vcaches) Store(key string, value vcache) { - defer atomic.AddUint64(&m.length, 1) - read, _ := m.read.Load().(readOnlyVCache) - if e, ok := read.m[key]; ok && e.tryStore(&value) { - return - } - - m.mu.Lock() - read, _ = m.read.Load().(readOnlyVCache) - if e, ok := read.m[key]; ok { - if e.unexpungeLocked() { - m.dirty[key] = e - } - e.storeLocked(&value) - } else if e, ok := m.dirty[key]; ok { - e.storeLocked(&value) - } else { - if !read.amended { - m.dirtyLocked() - m.read.Store(readOnlyVCache{m: read.m, amended: true}) - } - m.dirty[key] = newEntryVCache(value) - } - m.mu.Unlock() -} - -func (e *entryVCache) tryStore(i *vcache) bool { - for { - p := atomic.LoadPointer(&e.p) - if p == expungedVCache { - return false - } - if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { - return true - } - } -} - -func (e *entryVCache) unexpungeLocked() (wasExpunged bool) { - return atomic.CompareAndSwapPointer(&e.p, expungedVCache, nil) -} - -func (e *entryVCache) storeLocked(i *vcache) { - atomic.StorePointer(&e.p, unsafe.Pointer(i)) -} - -func (m *vcaches) Delete(key string) { - atomic.AddUint64(&m.length, ^uint64(0)) - read, _ := m.read.Load().(readOnlyVCache) - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - read, _ = m.read.Load().(readOnlyVCache) - e, ok = read.m[key] - if !ok && read.amended { - delete(m.dirty, key) - } - m.mu.Unlock() - } - if ok { - e.delete() - } -} - -func (e *entryVCache) delete() (hadValue bool) { - for { - p := atomic.LoadPointer(&e.p) - if p == nil || p == expungedVCache { - return false - } - if atomic.CompareAndSwapPointer(&e.p, p, nil) { - return true - } - } -} - -func (m *vcaches) Range(f func(key string, value vcache) bool) { - read, _ := m.read.Load().(readOnlyVCache) - if read.amended { - m.mu.Lock() - read, _ = m.read.Load().(readOnlyVCache) - if read.amended { - read = readOnlyVCache{m: m.dirty} - m.read.Store(read) - m.dirty = nil - m.misses = 0 - } - m.mu.Unlock() - } - - for k, e := range read.m { - v, ok := e.load() - if !ok { - continue - } - if !f(k, v) { - break - } - } -} - -func (m *vcaches) missLocked() { - m.misses++ - if m.misses < len(m.dirty) { - return - } - m.read.Store(readOnlyVCache{m: m.dirty}) - m.dirty = nil - m.misses = 0 -} - -func (m *vcaches) dirtyLocked() { - if m.dirty != nil { - return - } - - read, _ := m.read.Load().(readOnlyVCache) - m.dirty = make(map[string]*entryVCache, len(read.m)) - for k, e := range read.m { - if !e.tryExpungeLocked() { - m.dirty[k] = e - } - } -} - -func (e *entryVCache) tryExpungeLocked() (isExpunged bool) { - p := atomic.LoadPointer(&e.p) - for p == nil { - if atomic.CompareAndSwapPointer(&e.p, nil, expungedVCache) { - return true - } - p = atomic.LoadPointer(&e.p) - } - return p == expungedVCache -} - -func (m *vcaches) Len() uint64 { - return atomic.LoadUint64(&m.length) -} diff --git a/pkg/agent/core/ngt/service/vcaches_test.go b/pkg/agent/core/ngt/service/vcaches_test.go deleted file mode 100644 index 6710e2c2c8..0000000000 --- a/pkg/agent/core/ngt/service/vcaches_test.go +++ /dev/null @@ -1,1242 +0,0 @@ -// -// Copyright (C) 2019-2021 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package service - -import ( - "reflect" - "sync" - "sync/atomic" - "testing" - "unsafe" - - "github.com/vdaas/vald/internal/errors" - "go.uber.org/goleak" -) - -func Test_newEntryVCache(t *testing.T) { - t.Parallel() - type args struct { - i vcache - } - type want struct { - want *entryVCache - } - type test struct { - name string - args args - want want - checkFunc func(want, *entryVCache) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, got *entryVCache) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - i: vcache{}, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - i: vcache{}, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - got := newEntryVCache(test.args.i) - if err := test.checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_Load(t *testing.T) { - t.Parallel() - type args struct { - key string - } - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct { - wantValue vcache - wantOk bool - } - type test struct { - name string - args args - fields fields - want want - checkFunc func(want, vcache, bool) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, gotValue vcache, gotOk bool) error { - if !reflect.DeepEqual(gotValue, w.wantValue) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotValue, w.wantValue) - } - if !reflect.DeepEqual(gotOk, w.wantOk) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotOk, w.wantOk) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - key: "", - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - key: "", - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - gotValue, gotOk := m.Load(test.args.key) - if err := test.checkFunc(test.want, gotValue, gotOk); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_entryVCache_load(t *testing.T) { - t.Parallel() - type fields struct { - p unsafe.Pointer - } - type want struct { - wantValue vcache - wantOk bool - } - type test struct { - name string - fields fields - want want - checkFunc func(want, vcache, bool) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, gotValue vcache, gotOk bool) error { - if !reflect.DeepEqual(gotValue, w.wantValue) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotValue, w.wantValue) - } - if !reflect.DeepEqual(gotOk, w.wantOk) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotOk, w.wantOk) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - e := &entryVCache{ - p: test.fields.p, - } - - gotValue, gotOk := e.load() - if err := test.checkFunc(test.want, gotValue, gotOk); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_Store(t *testing.T) { - t.Parallel() - type args struct { - key string - value vcache - } - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct{} - type test struct { - name string - args args - fields fields - want want - checkFunc func(want) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want) error { - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - key: "", - value: vcache{}, - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - key: "", - value: vcache{}, - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - m.Store(test.args.key, test.args.value) - if err := test.checkFunc(test.want); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_entryVCache_tryStore(t *testing.T) { - t.Parallel() - type args struct { - i *vcache - } - type fields struct { - p unsafe.Pointer - } - type want struct { - want bool - } - type test struct { - name string - args args - fields fields - want want - checkFunc func(want, bool) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want, got bool) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - i: vcache{}, - }, - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - i: vcache{}, - }, - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - e := &entryVCache{ - p: test.fields.p, - } - - got := e.tryStore(test.args.i) - if err := test.checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_entryVCache_unexpungeLocked(t *testing.T) { - t.Parallel() - type fields struct { - p unsafe.Pointer - } - type want struct { - wantWasExpunged bool - } - type test struct { - name string - fields fields - want want - checkFunc func(want, bool) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, gotWasExpunged bool) error { - if !reflect.DeepEqual(gotWasExpunged, w.wantWasExpunged) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotWasExpunged, w.wantWasExpunged) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - e := &entryVCache{ - p: test.fields.p, - } - - gotWasExpunged := e.unexpungeLocked() - if err := test.checkFunc(test.want, gotWasExpunged); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_entryVCache_storeLocked(t *testing.T) { - t.Parallel() - type args struct { - i *vcache - } - type fields struct { - p unsafe.Pointer - } - type want struct{} - type test struct { - name string - args args - fields fields - want want - checkFunc func(want) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want) error { - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - i: vcache{}, - }, - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - i: vcache{}, - }, - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - e := &entryVCache{ - p: test.fields.p, - } - - e.storeLocked(test.args.i) - if err := test.checkFunc(test.want); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_Delete(t *testing.T) { - t.Parallel() - type args struct { - key string - } - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct{} - type test struct { - name string - args args - fields fields - want want - checkFunc func(want) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want) error { - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - key: "", - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - key: "", - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - m.Delete(test.args.key) - if err := test.checkFunc(test.want); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_entryVCache_delete(t *testing.T) { - t.Parallel() - type fields struct { - p unsafe.Pointer - } - type want struct { - wantHadValue bool - } - type test struct { - name string - fields fields - want want - checkFunc func(want, bool) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, gotHadValue bool) error { - if !reflect.DeepEqual(gotHadValue, w.wantHadValue) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotHadValue, w.wantHadValue) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - e := &entryVCache{ - p: test.fields.p, - } - - gotHadValue := e.delete() - if err := test.checkFunc(test.want, gotHadValue); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_Range(t *testing.T) { - t.Parallel() - type args struct { - f func(key string, value vcache) bool - } - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct{} - type test struct { - name string - args args - fields fields - want want - checkFunc func(want) error - beforeFunc func(args) - afterFunc func(args) - } - defaultCheckFunc := func(w want) error { - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - f: nil, - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - f: nil, - }, - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(test.args) - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - m.Range(test.args.f) - if err := test.checkFunc(test.want); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_missLocked(t *testing.T) { - t.Parallel() - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct{} - type test struct { - name string - fields fields - want want - checkFunc func(want) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want) error { - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - m.missLocked() - if err := test.checkFunc(test.want); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_dirtyLocked(t *testing.T) { - t.Parallel() - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct{} - type test struct { - name string - fields fields - want want - checkFunc func(want) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want) error { - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - m.dirtyLocked() - if err := test.checkFunc(test.want); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_entryVCache_tryExpungeLocked(t *testing.T) { - t.Parallel() - type fields struct { - p unsafe.Pointer - } - type want struct { - wantIsExpunged bool - } - type test struct { - name string - fields fields - want want - checkFunc func(want, bool) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, gotIsExpunged bool) error { - if !reflect.DeepEqual(gotIsExpunged, w.wantIsExpunged) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotIsExpunged, w.wantIsExpunged) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - p: nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - e := &entryVCache{ - p: test.fields.p, - } - - gotIsExpunged := e.tryExpungeLocked() - if err := test.checkFunc(test.want, gotIsExpunged); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func Test_vcaches_Len(t *testing.T) { - t.Parallel() - type fields struct { - length uint64 - mu sync.Mutex - read atomic.Value - dirty map[string]*entryVCache - misses int - } - type want struct { - want uint64 - } - type test struct { - name string - fields fields - want want - checkFunc func(want, uint64) error - beforeFunc func() - afterFunc func() - } - defaultCheckFunc := func(w want, got uint64) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - fields: fields { - length: 0, - mu: sync.Mutex{}, - read: nil, - dirty: nil, - misses: 0, - }, - want: want{}, - checkFunc: defaultCheckFunc, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc() - } - if test.afterFunc != nil { - defer test.afterFunc() - } - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - m := &vcaches{ - length: test.fields.length, - mu: test.fields.mu, - read: test.fields.read, - dirty: test.fields.dirty, - misses: test.fields.misses, - } - - got := m.Len() - if err := test.checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} diff --git a/pkg/agent/core/ngt/service/vqueue/queue.go b/pkg/agent/core/ngt/service/vqueue/queue.go new file mode 100644 index 0000000000..f2113be468 --- /dev/null +++ b/pkg/agent/core/ngt/service/vqueue/queue.go @@ -0,0 +1,294 @@ +// +// Copyright (C) 2019-2021 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package vqueue manages the vector cache layer for reducing FFI overhead for fast Agent processing. +package vqueue + +import ( + "context" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/safety" +) + +type Queue interface { + Start(ctx context.Context) (<-chan error, error) + PushInsert(uuid string, vector []float32, date int64) error + PushDelete(uuid string, date int64) error + PopInsert() (uuid string, vector []float32) + PopDelete() (uuid string) + GetVector(uuid string) ([]float32, bool) + RangePopInsert(ctx context.Context, f func(uuid string, vector []float32) bool) + RangePopDelete(ctx context.Context, f func(uuid string) bool) + IVQLen() int + DVQLen() int +} + +type vqueue struct { + ich chan index + uii []index // un inserted index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key // un deleted key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value +} + +type index struct { + uuid string + vector []float32 + date int64 +} + +type key struct { + uuid string + vector []float32 + date int64 +} + +func New(eg errgroup.Group) Queue { + vq := &vqueue{ + ich: make(chan index, 1000), + uii: make([]index, 0, 10000), + uiil: make(map[string][]float32, 100), + dch: make(chan key, 1000), + udk: make([]key, 0, 10000), + eg: eg, + } + vq.finalizing.Store(false) + return vq +} + +func (v *vqueue) Start(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 1) + v.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + for { + select { + case <-ctx.Done(): + v.finalizing.Store(true) + close(v.dch) + close(v.ich) + for d := range v.dch { + v.addDelete(d) + } + for i := range v.ich { + v.addInsert(i) + } + v.finalizing.Store(false) + return ctx.Err() + case i := <-v.ich: + v.addInsert(i) + case d := <-v.dch: + v.addDelete(d) + } + } + })) + return ech, nil +} + +func (v *vqueue) PushInsert(uuid string, vector []float32, date int64) error { + if v.finalizing.Load().(bool) { + return errors.ErrVQueueFinalizing + } + if date == 0 { + date = time.Now().UnixNano() + } + + v.ich <- index{ + uuid: uuid, + vector: vector, + date: date, + } + return nil +} + +func (v *vqueue) PushDelete(uuid string, date int64) error { + if v.finalizing.Load().(bool) { + return errors.ErrVQueueFinalizing + } + if date == 0 { + date = time.Now().UnixNano() + } + v.dch <- key{ + uuid: uuid, + date: date, + } + return nil +} + +func (v *vqueue) PopInsert() (uuid string, vector []float32) { + i := v.popInsert() + return i.uuid, i.vector +} + +func (v *vqueue) PopDelete() (uuid string) { + d := v.popDelete() + return d.uuid +} + +func (v *vqueue) RangePopInsert(ctx context.Context, f func(uuid string, vector []float32) bool) { + if v.finalizing.Load().(bool) { + for !v.finalizing.Load().(bool) { + time.Sleep(time.Millisecond * 100) + } + } + for _, idx := range v.flushAndLoadInsert() { + select { + case <-ctx.Done(): + return + default: + if !f(idx.uuid, idx.vector) { + return + } + } + } +} + +func (v *vqueue) RangePopDelete(ctx context.Context, f func(uuid string) bool) { + if v.finalizing.Load().(bool) { + for !v.finalizing.Load().(bool) { + time.Sleep(time.Millisecond * 100) + } + } + for _, key := range v.flushAndLoadDelete() { + select { + case <-ctx.Done(): + return + default: + if !f(key.uuid) { + return + } + } + } +} + +func (v *vqueue) GetVector(uuid string) ([]float32, bool) { + v.imu.Lock() + vec, ok := v.uiil[uuid] + v.imu.Unlock() + return vec, ok +} + +func (v *vqueue) addInsert(i index) { + v.imu.Lock() + v.uii = append(v.uii, i) + v.uiil[i.uuid] = i.vector + v.imu.Unlock() +} + +func (v *vqueue) addDelete(d key) { + v.dmu.Lock() + v.udk = append(v.udk, d) + v.dmu.Unlock() +} + +func (v *vqueue) popInsert() (i index) { + v.imu.Lock() + i = v.uii[0] + v.uii = v.uii[1:] + delete(v.uiil, i.uuid) + v.imu.Unlock() + return i +} + +func (v *vqueue) popDelete() (d key) { + v.dmu.Lock() + d = v.udk[0] + v.udk = v.udk[1:] + v.dmu.Unlock() + return d +} + +func (v *vqueue) flushAndLoadInsert() (uii []index) { + v.imu.Lock() + uii = make([]index, len(v.uii)) + copy(uii, v.uii) + v.uii = v.uii[:0] + v.imu.Unlock() + sort.Slice(uii, func(i, j int) bool { + return uii[i].date > uii[j].date + }) + dup := make(map[string]bool, len(uii)/2) + dl := make([]int, 0, len(uii)/2) + for i, idx := range uii { + v.imu.Lock() + delete(v.uiil, idx.uuid) + v.imu.Unlock() + if dup[idx.uuid] { + dl = append(dl, i) + } else { + dup[idx.uuid] = true + } + } + sort.Sort(sort.Reverse(sort.IntSlice(dl))) + for _, i := range dl { + uii = append(uii[:i], uii[i+1:]...) + } + sort.Slice(uii, func(i, j int) bool { + return uii[i].date < uii[j].date + }) + return uii +} + +func (v *vqueue) flushAndLoadDelete() (udk []key) { + v.dmu.Lock() + udk = make([]key, len(v.udk)) + copy(udk, v.udk) + v.udk = v.udk[:0] + v.dmu.Unlock() + sort.Slice(udk, func(i, j int) bool { + return udk[i].date > udk[j].date + }) + dup := make(map[string]bool, len(udk)/2) + dl := make([]int, 0, len(udk)/2) + for i, idx := range udk { + if dup[idx.uuid] { + dl = append(dl, i) + } else { + dup[idx.uuid] = true + } + } + sort.Sort(sort.Reverse(sort.IntSlice(dl))) + for _, i := range dl { + udk = append(udk[:i], udk[i+1:]...) + } + sort.Slice(udk, func(i, j int) bool { + return udk[i].date < udk[j].date + }) + return udk +} + +func (v *vqueue) IVQLen() (l int) { + v.imu.Lock() + l = len(v.uii) + v.imu.Unlock() + return l +} + +func (v *vqueue) DVQLen() (l int) { + v.dmu.Lock() + l = len(v.udk) + v.dmu.Unlock() + return l +} diff --git a/pkg/agent/core/ngt/service/vqueue/queue_test.go b/pkg/agent/core/ngt/service/vqueue/queue_test.go new file mode 100644 index 0000000000..92492edc7b --- /dev/null +++ b/pkg/agent/core/ngt/service/vqueue/queue_test.go @@ -0,0 +1,1918 @@ +// +// Copyright (C) 2019-2021 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package vqueue manages the vector cache layer for reducing FFI overhead for fast Agent processing. +package vqueue + +import ( + "context" + "reflect" + "sync" + "sync/atomic" + "testing" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "go.uber.org/goleak" +) + +func TestNew(t *testing.T) { + t.Parallel() + type args struct { + eg errgroup.Group + } + type want struct { + want Queue + } + type test struct { + name string + args args + want want + checkFunc func(want, Queue) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got Queue) error { + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + eg: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + eg: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := New(test.args.eg) + if err := test.checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_Start(t *testing.T) { + t.Parallel() + type args struct { + ctx context.Context + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + want <-chan error + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, <-chan error, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got <-chan error, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + ctx: nil, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + ctx: nil, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + got, err := v.Start(test.args.ctx) + if err := test.checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_PushInsert(t *testing.T) { + t.Parallel() + type args struct { + uuid string + vector []float32 + date int64 + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + uuid: "", + vector: nil, + date: 0, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + uuid: "", + vector: nil, + date: 0, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + err := v.PushInsert(test.args.uuid, test.args.vector, test.args.date) + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_PushDelete(t *testing.T) { + t.Parallel() + type args struct { + uuid string + date int64 + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + uuid: "", + date: 0, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + uuid: "", + date: 0, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + err := v.PushDelete(test.args.uuid, test.args.date) + if err := test.checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_PopInsert(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantUuid string + wantVector []float32 + } + type test struct { + name string + fields fields + want want + checkFunc func(want, string, []float32) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotUuid string, gotVector []float32) error { + if !reflect.DeepEqual(gotUuid, w.wantUuid) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotUuid, w.wantUuid) + } + if !reflect.DeepEqual(gotVector, w.wantVector) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotVector, w.wantVector) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotUuid, gotVector := v.PopInsert() + if err := test.checkFunc(test.want, gotUuid, gotVector); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_PopDelete(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantUuid string + } + type test struct { + name string + fields fields + want want + checkFunc func(want, string) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotUuid string) error { + if !reflect.DeepEqual(gotUuid, w.wantUuid) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotUuid, w.wantUuid) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotUuid := v.PopDelete() + if err := test.checkFunc(test.want, gotUuid); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_RangePopInsert(t *testing.T) { + t.Parallel() + type args struct { + ctx context.Context + f func(uuid string, vector []float32) bool + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct{} + type test struct { + name string + args args + fields fields + want want + checkFunc func(want) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want) error { + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + ctx: nil, + f: nil, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + ctx: nil, + f: nil, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + v.RangePopInsert(test.args.ctx, test.args.f) + if err := test.checkFunc(test.want); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_RangePopDelete(t *testing.T) { + t.Parallel() + type args struct { + ctx context.Context + f func(uuid string) bool + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct{} + type test struct { + name string + args args + fields fields + want want + checkFunc func(want) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want) error { + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + ctx: nil, + f: nil, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + ctx: nil, + f: nil, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + v.RangePopDelete(test.args.ctx, test.args.f) + if err := test.checkFunc(test.want); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_GetVector(t *testing.T) { + t.Parallel() + type args struct { + uuid string + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + want []float32 + want1 bool + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, []float32, bool) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want, got []float32, got1 bool) error { + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + if !reflect.DeepEqual(got1, w.want1) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + uuid: "", + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + uuid: "", + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + got, got1 := v.GetVector(test.args.uuid) + if err := test.checkFunc(test.want, got, got1); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_addInsert(t *testing.T) { + t.Parallel() + type args struct { + i index + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct{} + type test struct { + name string + args args + fields fields + want want + checkFunc func(want) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want) error { + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + i: index{}, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + i: index{}, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + v.addInsert(test.args.i) + if err := test.checkFunc(test.want); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_addDelete(t *testing.T) { + t.Parallel() + type args struct { + d key + } + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct{} + type test struct { + name string + args args + fields fields + want want + checkFunc func(want) error + beforeFunc func(args) + afterFunc func(args) + } + defaultCheckFunc := func(w want) error { + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + args: args { + d: key{}, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + args: args { + d: key{}, + }, + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + v.addDelete(test.args.d) + if err := test.checkFunc(test.want); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_popInsert(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantI index + } + type test struct { + name string + fields fields + want want + checkFunc func(want, index) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotI index) error { + if !reflect.DeepEqual(gotI, w.wantI) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotI, w.wantI) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotI := v.popInsert() + if err := test.checkFunc(test.want, gotI); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_popDelete(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantD key + } + type test struct { + name string + fields fields + want want + checkFunc func(want, key) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotD key) error { + if !reflect.DeepEqual(gotD, w.wantD) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotD, w.wantD) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotD := v.popDelete() + if err := test.checkFunc(test.want, gotD); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_flushAndLoadInsert(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantUii []index + } + type test struct { + name string + fields fields + want want + checkFunc func(want, []index) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotUii []index) error { + if !reflect.DeepEqual(gotUii, w.wantUii) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotUii, w.wantUii) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotUii := v.flushAndLoadInsert() + if err := test.checkFunc(test.want, gotUii); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_flushAndLoadDelete(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantUdk []key + } + type test struct { + name string + fields fields + want want + checkFunc func(want, []key) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotUdk []key) error { + if !reflect.DeepEqual(gotUdk, w.wantUdk) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotUdk, w.wantUdk) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotUdk := v.flushAndLoadDelete() + if err := test.checkFunc(test.want, gotUdk); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_IVQLen(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantL int + } + type test struct { + name string + fields fields + want want + checkFunc func(want, int) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotL int) error { + if !reflect.DeepEqual(gotL, w.wantL) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotL, w.wantL) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotL := v.IVQLen() + if err := test.checkFunc(test.want, gotL); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_vqueue_DVQLen(t *testing.T) { + t.Parallel() + type fields struct { + ich chan index + uii []index + imu sync.Mutex + uiil map[string][]float32 + dch chan key + udk []key + dmu sync.Mutex + eg errgroup.Group + finalizing atomic.Value + } + type want struct { + wantL int + } + type test struct { + name string + fields fields + want want + checkFunc func(want, int) error + beforeFunc func() + afterFunc func() + } + defaultCheckFunc := func(w want, gotL int) error { + if !reflect.DeepEqual(gotL, w.wantL) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotL, w.wantL) + } + return nil + } + tests := []test{ + // TODO test cases + /* + { + name: "test_case_1", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + }, + */ + + // TODO test cases + /* + func() test { + return test { + name: "test_case_2", + fields: fields { + ich: nil, + uii: nil, + imu: sync.Mutex{}, + uiil: nil, + dch: nil, + udk: nil, + dmu: sync.Mutex{}, + eg: nil, + finalizing: nil, + }, + want: want{}, + checkFunc: defaultCheckFunc, + } + }(), + */ + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc() + } + if test.afterFunc != nil { + defer test.afterFunc() + } + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + v := &vqueue{ + ich: test.fields.ich, + uii: test.fields.uii, + imu: test.fields.imu, + uiil: test.fields.uiil, + dch: test.fields.dch, + udk: test.fields.udk, + dmu: test.fields.dmu, + eg: test.fields.eg, + finalizing: test.fields.finalizing, + } + + gotL := v.DVQLen() + if err := test.checkFunc(test.want, gotL); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +}