Use runtime.GC for reducing indexing memory & replace saveMu with atomic busy loop for race control

charts/vald/values.yaml

@@ -1285,6 +1285,9 @@ agent:
     # agent.ngt.load_index_timeout_factor -- a factor of load index timeout.
     # timeout duration will be calculated by (index count to be loaded) * (factor).
     load_index_timeout_factor: 1ms
+    # @schema {"name": "agent.ngt.enable_proactive_gc", "type": "boolean"}
+    # agent.ngt.enable_proactive_gc -- enable proactive GC call for reducing heap memory allocation
+    enable_proactive_gc: true
   # @schema {"name": "agent.sidecar", "type": "object"}
   sidecar:
     # @schema {"name": "agent.sidecar.enabled", "type": "boolean"}

go.mod

@@ -5,8 +5,9 @@ go 1.15
 replace (
 	cloud.google.com/go => cloud.google.com/go v0.65.0
 	github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
-	github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.34.18
+	github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.34.20
 	github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1
+	github.com/chzyer/logex => github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2
 	github.com/coreos/etcd => go.etcd.io/etcd v3.3.25+incompatible
 	github.com/docker/docker => github.com/moby/moby v1.13.1
 	github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate v0.4.1
@@ -20,11 +21,10 @@ replace (
 	github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.12.0
 	github.com/gorilla/websocket => github.com/gorilla/websocket v1.4.2
 	github.com/hailocab/go-hostpool => github.com/monzo/go-hostpool v0.0.0-20200724120130-287edbb29340
-	github.com/hydrogen18/memlistener => github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91
-	github.com/klauspost/compress => github.com/klauspost/compress v1.10.12-0.20200903102441-28b892527237
+	github.com/klauspost/compress => github.com/klauspost/compress v1.11.1-0.20200908135004-a2bf5b1ec3aa
 	github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0+incompatible
 	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
-	google.golang.org/grpc => google.golang.org/grpc v1.31.1
+	google.golang.org/grpc => google.golang.org/grpc v1.32.0
 	google.golang.org/protobuf => google.golang.org/protobuf v1.25.0
 	k8s.io/api => k8s.io/api v0.18.8
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.8
@@ -71,7 +71,7 @@ require (
 	go.uber.org/goleak v1.1.10
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f
+	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
 	gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
 	gonum.org/v1/plot v0.8.0
 	google.golang.org/api v0.31.0

go.sum

@@ -58,8 +58,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.34.18 h1:Mo/Clq3u1dQFzpg8YQqBii8m+Vl3fWIfHi6kXs5wpuM=
-github.com/aws/aws-sdk-go v1.34.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.34.20 h1:D9otznteZZyN5pRyFETqveYia/85Xzk7+RaPGB1I9fE=
+github.com/aws/aws-sdk-go v1.34.20/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -76,7 +76,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/logex v1.1.11-0.20170329064859-445be9e134b2/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -286,8 +286,8 @@ github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5Pt
 github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.10.12-0.20200903102441-28b892527237 h1:/d9euTqPnSaB1CVc/lFFns/hHf1JTZAxfEWS5dbn+x0=
-github.com/klauspost/compress v1.10.12-0.20200903102441-28b892527237/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.1-0.20200908135004-a2bf5b1ec3aa h1:9xEtLQvhsiWZvijuoPGoFVxijpWuacg3KDA+kvlI4+4=
+github.com/klauspost/compress v1.11.1-0.20200908135004-a2bf5b1ec3aa/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kpango/fastime v1.0.8/go.mod h1:Y5XY5bLG5yc7g2XmMUzc22XYV1XaH+KgUOHkDvLp4SA=
@@ -622,8 +622,8 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -746,8 +746,8 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d h1:92D1fum1bJLKSdr11OJ+54YeCMCGYIygTA7R/YZxH5M=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
+google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=

hack/go.mod.default

@@ -7,6 +7,7 @@ replace (
 	github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0
 	github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go latest
 	github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1
+	github.com/chzyer/logex => github.com/chzyer/logex master
 	github.com/coreos/etcd =>  go.etcd.io/etcd latest
 	github.com/docker/docker => github.com/moby/moby latest
 	github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate latest
@@ -20,7 +21,6 @@ replace (
 	github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud latest
 	github.com/gorilla/websocket => github.com/gorilla/websocket latest
 	github.com/hailocab/go-hostpool => github.com/monzo/go-hostpool latest
-	github.com/hydrogen18/memlistener => github.com/hydrogen18/memlistener master
 	github.com/klauspost/compress => github.com/klauspost/compress master
 	github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0
 	golang.org/x/crypto => golang.org/x/crypto latest

internal/config/ngt.go

@@ -75,6 +75,9 @@ type NGT struct {
 
 	// LoadIndexTimeoutFactor represents a factor of load index timeout
 	LoadIndexTimeoutFactor string `yaml:"load_index_timeout_factor" json:"load_index_timeout_factor"`
+
+	// EnableProactiveGC enables more proactive GC call for reducing heap memory allocation
+	EnableProactiveGC bool `yaml:"enable_proactive_gc" json:"enable_proactive_gc"`
 }
 
 // Bind returns NGT object whose some string value is filed value or environment value.

internal/errors/http.go

@@ -30,8 +30,8 @@ var (
 		return Wrap(err, "handler returned error")
 	}
 
-	ErrHandlerTimeout = func(err error, t time.Time) error {
-		return Wrapf(err, "handler timeout %v", t)
+	ErrHandlerTimeout = func(err error, dur time.Duration) error {
+		return Wrapf(err, "handler timeout %s", dur.String())
 	}
 
 	ErrRequestBodyCloseAndFlush = func(err error) error {

internal/net/http/middleware/timeout.go

@@ -78,7 +78,7 @@ func (t *timeout) Wrap(h rest.Func) rest.Func {
 		case <-ctx.Done():
 			// timeout passed or parent context canceled first,
 			// it is the responsibility for handler to response to the user
-			return http.StatusRequestTimeout, errors.ErrHandlerTimeout(ctx.Err(), time.Unix(0, fastime.UnixNanoNow()-start))
+			return http.StatusRequestTimeout, errors.ErrHandlerTimeout(ctx.Err(), time.Duration(fastime.UnixNanoNow()-start))
 		}
 	}
 }

internal/observability/metrics/agent/core/ngt/ngt.go

@@ -31,7 +31,9 @@ type ngtMetrics struct {
 	insertVCacheCount         metrics.Int64Measure
 	deleteVCacheCount         metrics.Int64Measure
 	completedCreateIndexTotal metrics.Int64Measure
+	executedProactiveGCTotal  metrics.Int64Measure
 	isIndexing                metrics.Int64Measure
+	isSaving                  metrics.Int64Measure
 }
 
 func New(n service.NGT) metrics.Metric {
@@ -57,10 +59,18 @@ func New(n service.NGT) metrics.Metric {
 			metrics.ValdOrg+"/ngt/completed_create_index_total",
 			"the cumulative count of completed create index execution",
 			metrics.UnitDimensionless),
+		executedProactiveGCTotal: *metrics.Int64(
+			metrics.ValdOrg+"/ngt/executed_proactive_gc_total",
+			"the cumulative count of proactive GC execution",
+			metrics.UnitDimensionless),
 		isIndexing: *metrics.Int64(
 			metrics.ValdOrg+"/ngt/is_indexing",
 			"currently indexing or not",
 			metrics.UnitDimensionless),
+		isSaving: *metrics.Int64(
+			metrics.ValdOrg+"/ngt/is_saving",
+			"currently saving or not",
+			metrics.UnitDimensionless),
 	}
 }
 
@@ -70,13 +80,20 @@ func (n *ngtMetrics) Measurement(ctx context.Context) ([]metrics.Measurement, er
 		isIndexing = 1
 	}
 
+	var isSaving int64
+	if n.ngt.IsSaving() {
+		isSaving = 1
+	}
+
 	return []metrics.Measurement{
 		n.indexCount.M(int64(n.ngt.Len())),
 		n.uncommittedIndexCount.M(int64(n.ngt.InsertVCacheLen() + n.ngt.DeleteVCacheLen())),
 		n.insertVCacheCount.M(int64(n.ngt.InsertVCacheLen())),
 		n.deleteVCacheCount.M(int64(n.ngt.DeleteVCacheLen())),
 		n.completedCreateIndexTotal.M(int64(n.ngt.NumberOfCreateIndexExecution())),
+		n.executedProactiveGCTotal.M(int64(n.ngt.NumberOfProactiveGCExecution())),
 		n.isIndexing.M(isIndexing),
+		n.isSaving.M(isSaving),
 	}, nil
 }
 
@@ -116,11 +133,23 @@ func (n *ngtMetrics) View() []*metrics.View {
 			Measure:     &n.completedCreateIndexTotal,
 			Aggregation: metrics.LastValue(),
 		},
+		&metrics.View{
+			Name:        "ngt_executed_proactive_gc_total",
+			Description: n.executedProactiveGCTotal.Description(),
+			Measure:     &n.executedProactiveGCTotal,
+			Aggregation: metrics.LastValue(),
+		},
 		&metrics.View{
 			Name:        "ngt_is_indexing",
 			Description: n.isIndexing.Description(),
 			Measure:     &n.isIndexing,
 			Aggregation: metrics.LastValue(),
 		},
+		&metrics.View{
+			Name:        "ngt_is_saving",
+			Description: n.isSaving.Description(),
+			Measure:     &n.isSaving,
+			Aggregation: metrics.LastValue(),
+		},
 	}
 }