[dbnode] Emit metric with dbnode health status (#2588)

Emit metric with dbnode health status

Problem:

In a large m3db cluster, when a database node becomes non-functional (service fails to start or host is down), it may go unnoticed. If it goes unnoticed long enough, and one more node that owns the same shard(s) becomes non-functional, a quorum may be lost and block writes to the database.

Solution:

The connection pool in `src/dbnode/client/connection_pool.go` already does periodic health check from the client's node/process. Let that code emit a gauge metric with the result of the healthcheck. The metrics scope passed to `newConnectionPool` is already tagged with `hostID`.

Since the healthcheck is done from the client, it implies that node is in M3DB placement and expected to be functional. Thus, alerting can be set up based on this metric alone.

This behavior is optional, and disabled by default, to prevent accidental explosion of metric cardinality. When enabled, the callsites must ensure that the tags they set on the scope passed to m3db node client will not cause high cardinality of combinations with `hostID` tag.

Considered Alternatives:

1. Emit a heartbeat metric from `src/dbnode/network/server/tchannelthrift/node/service.go`. Alerting on lost heartbeat requires knowledge about whether the node is in placement, i.e. expected to be functional.

2. Let independent monitoring/canary system actively probe healthcheck endpoint of every database node, determine whether the node is expected to be functional by comparing to M3DB placement data, and alert operator. Such solution would be ideal but has much higher cost.

src/dbnode/client/connection_pool.go

@@ -32,8 +32,9 @@ import (
 	"github.com/m3db/m3/src/dbnode/generated/thrift/rpc"
 	"github.com/m3db/m3/src/dbnode/topology"
 	xclose "github.com/m3db/m3/src/x/close"
-	"github.com/m3db/stackmurmur3/v2"
+	murmur3 "github.com/m3db/stackmurmur3/v2"
 
+	"github.com/uber-go/tally"
 	"github.com/uber/tchannel-go/thrift"
 	"go.uber.org/zap"
 )
@@ -63,6 +64,7 @@ type connPool struct {
 	sleepHealth        sleepFn
 	sleepHealthRetry   sleepFn
 	status             status
+	healthStatus       tally.Gauge
 }
 
 type conn struct {
@@ -94,6 +96,7 @@ func newConnectionPool(host topology.Host, opts Options) connectionPool {
 		sleepConnect:       time.Sleep,
 		sleepHealth:        time.Sleep,
 		sleepHealthRetry:   time.Sleep,
+		healthStatus:       opts.InstrumentOptions().MetricsScope().Gauge("health-status"),
 	}
 
 	return p
@@ -186,11 +189,13 @@ func (p *connPool) connectEvery(interval time.Duration, stutter time.Duration) {
 
 				// Health check the connection
 				if err := p.healthCheckNewConn(client, p.opts); err != nil {
+					p.maybeEmitHealthStatus(healthStatusCheckFailed)
 					log.Debug("could not connect, failed health check", zap.String("host", address), zap.Error(err))
 					channel.Close()
 					return
 				}
 
+				p.maybeEmitHealthStatus(healthStatusOK)
 				p.Lock()
 				if p.status == statusOpen {
 					p.pool = append(p.pool, conn{channel, client})
@@ -206,6 +211,12 @@ func (p *connPool) connectEvery(interval time.Duration, stutter time.Duration) {
 	}
 }
 
+func (p *connPool) maybeEmitHealthStatus(hs healthStatus) {
+	if p.opts.HostQueueEmitsHealthStatus() {
+		p.healthStatus.Update(float64(hs))
+	}
+}
+
 func (p *connPool) healthCheckEvery(interval time.Duration, stutter time.Duration) {
 	log := p.opts.InstrumentOptions().Logger()
 	nowFn := p.opts.ClockOptions().NowFn()

src/dbnode/client/options.go

@@ -119,6 +119,9 @@ const (
 	// defaultHostQueueOpsArrayPoolSize is the default host queue ops array pool size
 	defaultHostQueueOpsArrayPoolSize = 8
 
+	// defaultHostQueueEmitsHealthStatus is false
+	defaultHostQueueEmitsHealthStatus = false
+
 	// defaultBackgroundConnectInterval is the default background connect interval
 	defaultBackgroundConnectInterval = 4 * time.Second
 
@@ -261,6 +264,7 @@ type options struct {
 	hostQueueOpsFlushSize                   int
 	hostQueueOpsFlushInterval               time.Duration
 	hostQueueOpsArrayPoolSize               int
+	hostQueueEmitsHealthStatus              bool
 	seriesIteratorPoolSize                  int
 	seriesIteratorArrayPoolBuckets          []pool.Bucket
 	checkedBytesWrapperPoolSize             int
@@ -381,6 +385,7 @@ func newOptions() *options {
 		hostQueueOpsFlushSize:                   defaultHostQueueOpsFlushSize,
 		hostQueueOpsFlushInterval:               defaultHostQueueOpsFlushInterval,
 		hostQueueOpsArrayPoolSize:               defaultHostQueueOpsArrayPoolSize,
+		hostQueueEmitsHealthStatus:              defaultHostQueueEmitsHealthStatus,
 		seriesIteratorPoolSize:                  defaultSeriesIteratorPoolSize,
 		seriesIteratorArrayPoolBuckets:          defaultSeriesIteratorArrayPoolBuckets,
 		checkedBytesWrapperPoolSize:             defaultCheckedBytesWrapperPoolSize,
@@ -884,6 +889,16 @@ func (o *options) HostQueueOpsArrayPoolSize() int {
 	return o.hostQueueOpsArrayPoolSize
 }
 
+func (o *options) SetHostQueueEmitsHealthStatus(value bool) Options {
+	opts := *o
+	opts.hostQueueEmitsHealthStatus = value
+	return &opts
+}
+
+func (o *options) HostQueueEmitsHealthStatus() bool {
+	return o.hostQueueEmitsHealthStatus
+}
+
 func (o *options) SetSeriesIteratorPoolSize(value int) Options {
 	opts := *o
 	opts.seriesIteratorPoolSize = value

src/dbnode/client/types.go

@@ -515,6 +515,12 @@ type Options interface {
 	// HostQueueOpsArrayPoolSize returns the hostQueueOpsArrayPoolSize.
 	HostQueueOpsArrayPoolSize() int
 
+	// SetHostQueueEmitsHealthStatus sets the hostQueueEmitHealthStatus.
+	SetHostQueueEmitsHealthStatus(value bool) Options
+
+	// HostQueueEmitsHealthStatus returns the hostQueueEmitHealthStatus.
+	HostQueueEmitsHealthStatus() bool
+
 	// SetSeriesIteratorPoolSize sets the seriesIteratorPoolSize.
 	SetSeriesIteratorPoolSize(value int) Options
 
@@ -710,6 +716,13 @@ const (
 	statusClosed
 )
 
+type healthStatus int
+
+const (
+	healthStatusCheckFailed healthStatus = iota
+	healthStatusOK
+)
+
 type op interface {
 	// Size returns the effective size of inner operations.
 	Size() int