Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use runtime.GC for reducing indexing memory & replace saveMu with atomic busy loop for race control #682

Merged
merged 13 commits into from
Sep 11, 2020
Merged

Use runtime.GC for reducing indexing memory & replace saveMu with atomic busy loop for race control #682

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
f62d60d
use runtime.GC for reducing indexing memory
Sep 8, 2020
b2a6c97
downgrade go version for debug
Sep 8, 2020
7ea859e
fix
Sep 8, 2020
e16022e
reduce create index & save index timing using busy loop and atomic co…
Sep 10, 2020
b673e5b
fix
Sep 10, 2020
c22261d
fix
Sep 10, 2020
409407a
fix
Sep 10, 2020
5d72a6c
fix
Sep 10, 2020
aac607c
fix
Sep 10, 2020
1e42abb
fix
Sep 10, 2020
c6285c6
fix
Sep 10, 2020
752ae71
Update charts/vald/values.yaml
Sep 10, 2020
fca1052
Merge branch 'master' into performance/agent/use-userdefined-gc-call
Sep 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.15
replace (
cloud.google.com/go => cloud.google.com/go v0.65.0
github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.34.18
github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.34.20
github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1
github.com/coreos/etcd => go.etcd.io/etcd v3.3.25+incompatible
github.com/docker/docker => github.com/moby/moby v1.13.1
Expand All @@ -21,10 +21,10 @@ replace (
github.com/gorilla/websocket => github.com/gorilla/websocket v1.4.2
github.com/hailocab/go-hostpool => github.com/monzo/go-hostpool v0.0.0-20200724120130-287edbb29340
github.com/hydrogen18/memlistener => github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91
github.com/klauspost/compress => github.com/klauspost/compress v1.10.12-0.20200903102441-28b892527237
github.com/klauspost/compress => github.com/klauspost/compress v1.11.1-0.20200908135004-a2bf5b1ec3aa
github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0+incompatible
golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
google.golang.org/grpc => google.golang.org/grpc v1.31.1
google.golang.org/grpc => google.golang.org/grpc v1.32.0
google.golang.org/protobuf => google.golang.org/protobuf v1.25.0
k8s.io/api => k8s.io/api v0.18.8
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.8
Expand Down Expand Up @@ -71,7 +71,7 @@ require (
go.uber.org/goleak v1.1.10
golang.org/x/net v0.0.0-20200904194848-62affa334b73
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f
golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
gonum.org/v1/plot v0.8.0
google.golang.org/api v0.31.0
Expand Down
16 changes: 8 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.34.18 h1:Mo/Clq3u1dQFzpg8YQqBii8m+Vl3fWIfHi6kXs5wpuM=
github.com/aws/aws-sdk-go v1.34.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.34.20 h1:D9otznteZZyN5pRyFETqveYia/85Xzk7+RaPGB1I9fE=
github.com/aws/aws-sdk-go v1.34.20/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -286,8 +286,8 @@ github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5Pt
github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.12-0.20200903102441-28b892527237 h1:/d9euTqPnSaB1CVc/lFFns/hHf1JTZAxfEWS5dbn+x0=
github.com/klauspost/compress v1.10.12-0.20200903102441-28b892527237/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.1-0.20200908135004-a2bf5b1ec3aa h1:9xEtLQvhsiWZvijuoPGoFVxijpWuacg3KDA+kvlI4+4=
github.com/klauspost/compress v1.11.1-0.20200908135004-a2bf5b1ec3aa/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kpango/fastime v1.0.8/go.mod h1:Y5XY5bLG5yc7g2XmMUzc22XYV1XaH+KgUOHkDvLp4SA=
Expand Down Expand Up @@ -622,8 +622,8 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
Expand Down Expand Up @@ -746,8 +746,8 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M=
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs=
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
Expand Down
41 changes: 28 additions & 13 deletions pkg/agent/core/ngt/service/ngt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,9 @@ type ngt struct {
dvc *vcaches // deletion vector cache

// statuses
indexing atomic.Value
saveMu sync.Mutex // creating or saving index
indexing atomic.Value
saving atomic.Value
lastNoice uint64 // last number of create index execution this value prevent unnecessary saveindex.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice! 👍


// counters
ic uint64 // insert count
Expand Down Expand Up @@ -154,6 +155,7 @@ func New(cfg *config.NGT, opts ...Option) (nn NGT, err error) {
}

n.indexing.Store(false)
n.saving.Store(false)

return n, nil
}
Expand Down Expand Up @@ -333,7 +335,7 @@ func (n *ngt) Start(ctx context.Context) <-chan error {
}

func (n *ngt) Search(vec []float32, size uint32, epsilon, radius float32) ([]model.Distance, error) {
if n.indexing.Load().(bool) {
if n.IsIndexing() {
return make([]model.Distance, 0), nil
}
sr, err := n.core.Search(vec, int(size), epsilon, radius)
Expand All @@ -360,7 +362,7 @@ func (n *ngt) Search(vec []float32, size uint32, epsilon, radius float32) ([]mod
}

func (n *ngt) SearchByID(uuid string, size uint32, epsilon, radius float32) (dst []model.Distance, err error) {
if n.indexing.Load().(bool) {
if n.IsIndexing() {
log.Debug("SearchByID\t now indexing...")
return make([]model.Distance, 0), nil
}
Expand Down Expand Up @@ -526,10 +528,7 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) {
}
}()

n.saveMu.Lock()
defer n.saveMu.Unlock()

if n.indexing.Load().(bool) {
if n.IsIndexing() || n.IsSaving() {
return nil
}
ic := atomic.LoadUint64(&n.ic)
Expand All @@ -538,8 +537,10 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) {
}
n.indexing.Store(true)
atomic.StoreUint64(&n.ic, 0)
runtime.GC()
t := time.Now().UnixNano()
defer n.indexing.Store(false)
defer runtime.GC()

log.Infof("create index operation started, uncommitted indexes = %d", ic)
delList := make([]string, 0, ic)
Expand Down Expand Up @@ -632,10 +633,6 @@ func (n *ngt) SaveIndex(ctx context.Context) (err error) {
span.End()
}
}()

n.saveMu.Lock()
defer n.saveMu.Unlock()

if len(n.path) != 0 && !n.inMem {
err = n.saveIndex(ctx)
}
Expand All @@ -644,6 +641,18 @@ func (n *ngt) SaveIndex(ctx context.Context) (err error) {
}

func (n *ngt) saveIndex(ctx context.Context) (err error) {
noice := atomic.LoadUint64(&n.nocie)
if atomic.LoadUint64(&n.lastNoice) == noice {
return
}
atomic.SwapUint64(&n.lastNoice, noice)
// wait for not indexing & not saving
for n.IsIndexing() || n.IsSaving() {
}
kpango marked this conversation as resolved.
Show resolved Hide resolved
n.saving.Store(true)
defer runtime.GC()
defer n.saving.Store(false)

eg, ctx := errgroup.New(ctx)

eg.Go(safety.RecoverFunc(func() error {
Expand Down Expand Up @@ -730,8 +739,14 @@ func (n *ngt) insertCache(uuid string) (*vcache, bool) {
return nil, false
}

func (n *ngt) IsSaving() bool {
s, ok := n.saving.Load().(bool)
return s && ok
}

func (n *ngt) IsIndexing() bool {
return n.indexing.Load().(bool)
i, ok := n.indexing.Load().(bool)
return i && ok
}

func (n *ngt) UUIDs(ctx context.Context) (uuids []string) {
Expand Down
Loading