Merge pull request #104077 from erikgrinaker/backport22.2-104008

release-22.2: kvserver: improve system lease observability
cockroachdb · Jun 6, 2023 · 0e56ed0 · 0e56ed0
2 parents 5b83009 + 4f274ed
commit 0e56ed0
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 5 deletions.
diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go
@@ -154,6 +154,12 @@ var (
 		Measurement: "Replicas",
 		Unit:        metric.Unit_COUNT,
 	}
+	metaLeaseLivenessCount = metric.Metadata{
+		Name:        "leases.liveness",
+		Help:        "Number of replica leaseholders for the liveness range(s)",
+		Measurement: "Replicas",
+		Unit:        metric.Unit_COUNT,
+	}
 
 	// Storage metrics.
 	metaLiveBytes = metric.Metadata{
@@ -1674,6 +1680,7 @@ type StoreMetrics struct {
 	LeaseTransferErrorCount   *metric.Counter
 	LeaseExpirationCount      *metric.Gauge
 	LeaseEpochCount           *metric.Gauge
+	LeaseLivenessCount        *metric.Gauge
 
 	// Storage metrics.
 	ResolveCommitCount *metric.Counter
@@ -2208,6 +2215,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		LeaseTransferErrorCount:   metric.NewCounter(metaLeaseTransferErrorCount),
 		LeaseExpirationCount:      metric.NewGauge(metaLeaseExpirationCount),
 		LeaseEpochCount:           metric.NewGauge(metaLeaseEpochCount),
+		LeaseLivenessCount:        metric.NewGauge(metaLeaseLivenessCount),
 
 		// Intent resolution metrics.
 		ResolveCommitCount: metric.NewCounter(metaResolveCommit),

diff --git a/pkg/kv/kvserver/replica_metrics.go b/pkg/kv/kvserver/replica_metrics.go
@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/allocatorimpl"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb"
@@ -27,11 +28,12 @@ import (
 
 // ReplicaMetrics contains details on the current status of the replica.
 type ReplicaMetrics struct {
-	Leader      bool
-	LeaseValid  bool
-	Leaseholder bool
-	LeaseType   roachpb.LeaseType
-	LeaseStatus kvserverpb.LeaseStatus
+	Leader        bool
+	LeaseValid    bool
+	Leaseholder   bool
+	LeaseType     roachpb.LeaseType
+	LeaseStatus   kvserverpb.LeaseStatus
+	LivenessLease bool
 
 	// Quiescent indicates whether the replica believes itself to be quiesced.
 	Quiescent bool
@@ -135,6 +137,8 @@ func calcReplicaMetrics(
 		m.LeaseValid = true
 		leaseOwner = leaseStatus.Lease.OwnedBy(storeID)
 		m.LeaseType = leaseStatus.Lease.Type()
+		m.LivenessLease = leaseOwner &&
+			keys.NodeLivenessSpan.Overlaps(desc.RSpan().AsRawSpanWithNoLocals())
 	}
 	m.Leaseholder = m.LeaseValid && leaseOwner
 	m.Leader = isRaftLeader(raftStatus)

diff --git a/pkg/kv/kvserver/replica_proposal.go b/pkg/kv/kvserver/replica_proposal.go
@@ -368,6 +368,17 @@ func (r *Replica) leasePostApplyLocked(
 		r.gossipFirstRangeLocked(ctx)
 	}
 
+	// Log acquisition of meta and liveness range leases. These are critical to
+	// cluster health, so it's useful to know their location over time.
+	if leaseChangingHands && iAmTheLeaseHolder &&
+		r.descRLocked().StartKey.Less(roachpb.RKey(keys.NodeLivenessKeyMax)) {
+		if r.ownsValidLeaseRLocked(ctx, now) {
+			log.Health.Infof(ctx, "acquired system range lease: %s", newLease)
+		} else {
+			log.Health.Warningf(ctx, "applied system range lease after it expired: %s", newLease)
+		}
+	}
+
 	if (leaseChangingHands || maybeSplit) && iAmTheLeaseHolder && hasExpirationBasedLease {
 		if requiresExpirationBasedLease {
 			// Whenever we first acquire an expiration-based lease for a range that

diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go
@@ -3264,6 +3264,7 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 		leaseHolderCount              int64
 		leaseExpirationCount          int64
 		leaseEpochCount               int64
+		leaseLivenessCount            int64
 		raftLeaderNotLeaseHolderCount int64
 		raftLeaderInvalidLeaseCount   int64
 		quiescentCount                int64
@@ -3333,6 +3334,9 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 			case roachpb.LeaseEpoch:
 				leaseEpochCount++
 			}
+			if metrics.LivenessLease {
+				leaseLivenessCount++
+			}
 		}
 		if metrics.Quiescent {
 			quiescentCount++
@@ -3396,6 +3400,7 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 	s.metrics.LeaseHolderCount.Update(leaseHolderCount)
 	s.metrics.LeaseExpirationCount.Update(leaseExpirationCount)
 	s.metrics.LeaseEpochCount.Update(leaseEpochCount)
+	s.metrics.LeaseLivenessCount.Update(leaseLivenessCount)
 	s.metrics.QuiescentCount.Update(quiescentCount)
 	s.metrics.UninitializedCount.Update(uninitializedCount)
 	s.metrics.AverageQueriesPerSecond.Update(averageQueriesPerSecond)

diff --git a/pkg/ts/catalog/chart_catalog.go b/pkg/ts/catalog/chart_catalog.go
@@ -1692,6 +1692,7 @@ var charts = []sectionDescription{
 				Metrics: []string{
 					"leases.epoch",
 					"leases.expiration",
+					"leases.liveness",
 					"replicas.leaseholders",
 					"replicas.leaders_not_leaseholders",
 					"replicas.leaders_invalid_lease",