Add a metric to collect the number of background routines

server/backend/background/background.go

@@ -25,6 +25,7 @@ import (
 	"sync/atomic"
 
 	"github.com/yorkie-team/yorkie/server/logging"
+	"github.com/yorkie-team/yorkie/server/profiling/prometheus"
 )
 
 type routineID int32
@@ -60,7 +61,11 @@ func New() *Background {
 
 // AttachGoroutine creates a goroutine on a given function and tracks it using
 // the background's WaitGroup.
-func (b *Background) AttachGoroutine(f func(ctx context.Context)) {
+func (b *Background) AttachGoroutine(
+	f func(ctx context.Context),
+	metrics *prometheus.Metrics,
+	taskType string,
+) {
 	b.wgMu.RLock() // this blocks with ongoing close(b.closing)
 	defer b.wgMu.RUnlock()
 	select {
@@ -73,8 +78,12 @@ func (b *Background) AttachGoroutine(f func(ctx context.Context)) {
 	// now safe to add since WaitGroup wait has not started yet
 	b.wg.Add(1)
 	routineLogger := logging.New(b.routineID.next())
+	metrics.AddBackgroundGoroutines(taskType)
 	go func() {
-		defer b.wg.Done()
+		defer func() {
+			b.wg.Done()
+			metrics.RemoveBackgroundGoroutines(taskType)
+		}()
 		f(logging.With(context.Background(), routineLogger))
 	}()
 }

server/packs/packs.go

@@ -204,7 +204,7 @@ func PushPull(
 			be.Metrics.ObservePushPullSnapshotDurationSeconds(
 				gotime.Since(start).Seconds(),
 			)
-		})
+		}, be.Metrics, "pushpull")
 	}
 
 	return respPack, nil

server/profiling/prometheus/metrics.go

@@ -36,6 +36,7 @@ const (
 	projectIDLabel   = "project_id"
 	projectNameLabel = "project_name"
 	hostnameLabel    = "hostname"
+	taskTypeLabel    = "task_type"
 )
 
 var (
@@ -61,6 +62,8 @@ type Metrics struct {
 	pushPullSnapshotDurationSeconds prometheus.Histogram
 	pushPullSnapshotBytesTotal      prometheus.Counter
 
+	backgroundGoroutinesTotal *prometheus.GaugeVec
+
 	userAgentTotal *prometheus.CounterVec
 }
 
@@ -134,6 +137,12 @@ func NewMetrics() (*Metrics, error) {
 			Name:      "snapshot_bytes_total",
 			Help:      "The total bytes of snapshots for response packs in PushPull.",
 		}),
+		backgroundGoroutinesTotal: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: "background",
+			Name:      "goroutines_total",
+			Help:      "The total number of goroutines attached by a particular background task.",
+		}, []string{taskTypeLabel}),
 		userAgentTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: "user_agent",
@@ -234,6 +243,20 @@ func (m *Metrics) AddServerHandledCounter(
 	}).Inc()
 }
 
+// AddBackgroundGoroutines adds the number of goroutines attached by a particular background task.
+func (m *Metrics) AddBackgroundGoroutines(taskType string) {
+	m.backgroundGoroutinesTotal.With(prometheus.Labels{
+		taskTypeLabel: taskType,
+	}).Inc()
+}
+
+// RemoveBackgroundGoroutines removes the number of goroutines attached by a particular background task.
+func (m *Metrics) RemoveBackgroundGoroutines(taskType string) {
+	m.backgroundGoroutinesTotal.With(prometheus.Labels{
+		taskTypeLabel: taskType,
+	}).Dec()
+}
+
 // Registry returns the registry of this metrics.
 func (m *Metrics) Registry() *prometheus.Registry {
 	return m.registry