Merge #104008

104008: kvserver: improve system lease observability r=erikgrinaker a=erikgrinaker

**kvserver: add `leases.liveness` metric**

This patch adds the metric `leases.liveness` tracking the number of liveness range leases per node (generally 1 or 0). This is useful to find out which node had the liveness lease at a particular time.

I ran a 10k range cluster to look at the CPU cost of the key comparisons, it didn't show up on CPU profiles.

Epic: none
Release note (ops change): added the metric `leases.liveness` showing the number of liveness range leases per node (generally 1 or 0), to track the liveness range leaseholder.
  
**kvserver: log system range lease acquisition**

This patch logs acquisition of meta/liveness range leases to the health log. These leases are critical to cluster health, and during debugging it's useful to know their location over time.

Resolves #99472.

Epic: none
Release note: none

Co-authored-by: Erik Grinaker <[email protected]>

pkg/kv/kvserver/metrics.go

@@ -164,6 +164,12 @@ var (
 		Measurement: "Replicas",
 		Unit:        metric.Unit_COUNT,
 	}
+	metaLeaseLivenessCount = metric.Metadata{
+		Name:        "leases.liveness",
+		Help:        "Number of replica leaseholders for the liveness range(s)",
+		Measurement: "Replicas",
+		Unit:        metric.Unit_COUNT,
+	}
 
 	// Storage metrics.
 	metaLiveBytes = metric.Metadata{
@@ -2022,6 +2028,7 @@ type StoreMetrics struct {
 	LeaseTransferErrorCount   *metric.Counter
 	LeaseExpirationCount      *metric.Gauge
 	LeaseEpochCount           *metric.Gauge
+	LeaseLivenessCount        *metric.Gauge
 
 	// Storage metrics.
 	ResolveCommitCount *metric.Counter
@@ -2653,6 +2660,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		LeaseTransferErrorCount:   metric.NewCounter(metaLeaseTransferErrorCount),
 		LeaseExpirationCount:      metric.NewGauge(metaLeaseExpirationCount),
 		LeaseEpochCount:           metric.NewGauge(metaLeaseEpochCount),
+		LeaseLivenessCount:        metric.NewGauge(metaLeaseLivenessCount),
 
 		// Intent resolution metrics.
 		ResolveCommitCount: metric.NewCounter(metaResolveCommit),

pkg/kv/kvserver/replica_metrics.go

@@ -15,6 +15,7 @@ import (
 	"math"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/allocatorimpl"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb"
@@ -27,11 +28,12 @@ import (
 
 // ReplicaMetrics contains details on the current status of the replica.
 type ReplicaMetrics struct {
-	Leader      bool
-	LeaseValid  bool
-	Leaseholder bool
-	LeaseType   roachpb.LeaseType
-	LeaseStatus kvserverpb.LeaseStatus
+	Leader        bool
+	LeaseValid    bool
+	Leaseholder   bool
+	LeaseType     roachpb.LeaseType
+	LeaseStatus   kvserverpb.LeaseStatus
+	LivenessLease bool
 
 	// Quiescent indicates whether the replica believes itself to be quiesced.
 	Quiescent bool
@@ -125,12 +127,14 @@ type calcReplicaMetricsInput struct {
 }
 
 func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics {
-	var validLease, validLeaseOwner bool
+	var validLease, validLeaseOwner, livenessLease bool
 	var validLeaseType roachpb.LeaseType
 	if d.leaseStatus.IsValid() {
 		validLease = true
 		validLeaseOwner = d.leaseStatus.Lease.OwnedBy(d.storeID)
 		validLeaseType = d.leaseStatus.Lease.Type()
+		livenessLease = validLeaseOwner &&
+			keys.NodeLivenessSpan.Overlaps(d.desc.RSpan().AsRawSpanWithNoLocals())
 	}
 
 	rangeCounter, unavailable, underreplicated, overreplicated := calcRangeCounter(
@@ -152,6 +156,7 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics {
 		Leaseholder:     validLeaseOwner,
 		LeaseType:       validLeaseType,
 		LeaseStatus:     d.leaseStatus,
+		LivenessLease:   livenessLease,
 		Quiescent:       d.quiescent,
 		Ticking:         d.ticking,
 		RangeCounter:    rangeCounter,

pkg/kv/kvserver/replica_proposal.go

@@ -366,6 +366,17 @@ func (r *Replica) leasePostApplyLocked(
 		r.gossipFirstRangeLocked(ctx)
 	}
 
+	// Log acquisition of meta and liveness range leases. These are critical to
+	// cluster health, so it's useful to know their location over time.
+	if leaseChangingHands && iAmTheLeaseHolder &&
+		r.descRLocked().StartKey.Less(roachpb.RKey(keys.NodeLivenessKeyMax)) {
+		if r.ownsValidLeaseRLocked(ctx, now) {
+			log.Health.Infof(ctx, "acquired system range lease: %s", newLease)
+		} else {
+			log.Health.Warningf(ctx, "applied system range lease after it expired: %s", newLease)
+		}
+	}
+
 	st := r.leaseStatusAtRLocked(ctx, now)
 	if leaseChangingHands && newLease.Type() == roachpb.LeaseExpiration &&
 		r.ownsValidLeaseRLocked(ctx, now) && !r.shouldUseExpirationLeaseRLocked() {

pkg/kv/kvserver/store.go

@@ -2892,6 +2892,7 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 		leaseHolderCount              int64
 		leaseExpirationCount          int64
 		leaseEpochCount               int64
+		leaseLivenessCount            int64
 		raftLeaderNotLeaseHolderCount int64
 		raftLeaderInvalidLeaseCount   int64
 		quiescentCount                int64
@@ -2963,6 +2964,9 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 			case roachpb.LeaseEpoch:
 				leaseEpochCount++
 			}
+			if metrics.LivenessLease {
+				leaseLivenessCount++
+			}
 		}
 		if metrics.Quiescent {
 			quiescentCount++
@@ -3021,6 +3025,7 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 	s.metrics.LeaseHolderCount.Update(leaseHolderCount)
 	s.metrics.LeaseExpirationCount.Update(leaseExpirationCount)
 	s.metrics.LeaseEpochCount.Update(leaseEpochCount)
+	s.metrics.LeaseLivenessCount.Update(leaseLivenessCount)
 	s.metrics.QuiescentCount.Update(quiescentCount)
 	s.metrics.UninitializedCount.Update(uninitializedCount)
 	s.metrics.AverageQueriesPerSecond.Update(averageQueriesPerSecond)