feat(metric): Add dgraph_memory_alloc_bytes to track jemalloc memory. (…

…#7051)

This adds a new metric called dgraph_memory_alloc_bytes that
tracks the jemalloc memory reported by z.NumAllocBytes().

Changes

* Add dgraph_memory_alloc_bytes Prometheus metric.
* Add a comment about runtime.ReadMemStats stopping the world.

dgraph/cmd/alpha/metrics_test.go

@@ -96,6 +96,7 @@ func TestMetrics(t *testing.T) {
 
 		// Dgraph Memory Metrics
 		"dgraph_memory_idle_bytes", "dgraph_memory_inuse_bytes", "dgraph_memory_proc_bytes",
+		"dgraph_memory_alloc_bytes",
 		// Dgraph Activity Metrics
 		"dgraph_active_mutations_total", "dgraph_pending_proposals_total",
 		"dgraph_pending_queries_total",

x/metrics.go

@@ -70,6 +70,9 @@ var (
 	// PendingProposals records the current number of pending RAFT proposals.
 	PendingProposals = stats.Int64("pending_proposals_total",
 		"Number of pending proposals", stats.UnitDimensionless)
+	// MemoryAlloc records the amount of memory allocated via jemalloc
+	MemoryAlloc = stats.Int64("memory_alloc_bytes",
+		"Amount of memory allocated", stats.UnitBytes)
 	// MemoryInUse records the current amount of used memory by Dgraph.
 	MemoryInUse = stats.Int64("memory_inuse_bytes",
 		"Amount of memory in use", stats.UnitBytes)
@@ -192,6 +195,13 @@ var (
 			Aggregation: view.LastValue(),
 			TagKeys:     nil,
 		},
+		{
+			Name:        MemoryAlloc.Name(),
+			Measure:     MemoryAlloc,
+			Description: MemoryAlloc.Description(),
+			Aggregation: view.LastValue(),
+			TagKeys:     allTagKeys,
+		},
 		{
 			Name:        MemoryInUse.Name(),
 			Measure:     MemoryInUse,
@@ -447,8 +457,13 @@ func MonitorMemoryMetrics(lc *z.Closer) {
 	defer lc.Done()
 	ticker := time.NewTicker(time.Minute)
 	defer ticker.Stop()
+	fastTicker := time.NewTicker(time.Second)
+	defer fastTicker.Stop()
 
 	update := func() {
+		// ReadMemStats stops the world which is expensive especially when the
+		// heap is large. So don't call it too frequently. Calling it every
+		// minute is OK.
 		var ms runtime.MemStats
 		runtime.ReadMemStats(&ms)
 
@@ -467,14 +482,20 @@ func MonitorMemoryMetrics(lc *z.Closer) {
 			MemoryIdle.M(int64(idle)),
 			MemoryProc.M(int64(getMemUsage())))
 	}
+	updateAlloc := func() {
+		ostats.Record(context.Background(), MemoryAlloc.M(z.NumAllocBytes()))
+	}
 	// Call update immediately so that Dgraph reports memory stats without
 	// having to wait for the first tick.
 	update()
+	updateAlloc()
 
 	for {
 		select {
 		case <-lc.HasBeenClosed():
 			return
+		case <-fastTicker.C:
+			updateAlloc()
 		case <-ticker.C:
 			update()
 		}