kvserver: introduce GetLivenessesFromKV

Now that we always create a liveness record on start up (cockroachdb#53805), we can
simply fetch all liveness records from KV when wanting an up-to-date
view of all nodes in the cluster. We add a helper to do as much,
which we'll rely on in future commits. It's a bit unfortunate that we're
further adding on to the NodeLiveness API without changing the
underlying look-aside caching structure, but the methods fetching
records from KV is the world we're hoping to start moving towards over
time.

Release note: None

pkg/kv/kvserver/node_liveness.go

@@ -128,6 +128,8 @@ type LivenessMetrics struct {
 	HeartbeatLatency   *metric.Histogram
 }
 
+// TODO(irfansharif): We should/could cut out a pkg/nodeliveness here.
+
 // IsLiveCallback is invoked when a node's IsLive state changes to true.
 // Callbacks can be registered via NodeLiveness.RegisterCallback().
 type IsLiveCallback func(kvserverpb.Liveness)
@@ -1002,6 +1004,39 @@ func (nl *NodeLiveness) GetLivenesses() []kvserverpb.Liveness {
 	return livenesses
 }
 
+// GetLivenessesFromKV returns a slice containing the liveness status of every
+// node on the cluster. It's the improved version of GetLivenesses above, which
+// only consults the (possibly stale) in-memory cache.
+func (nl *NodeLiveness) GetLivenessesFromKV(ctx context.Context) ([]kvserverpb.Liveness, error) {
+	kvs, err := nl.db.Scan(ctx, keys.NodeLivenessPrefix, keys.NodeLivenessKeyMax, 0)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to get liveness")
+	}
+
+	var results []kvserverpb.Liveness
+	for _, kv := range kvs {
+		if kv.Value == nil {
+			return nil, errors.AssertionFailedf("missing liveness record")
+		}
+		var liveness kvserverpb.Liveness
+		if err := kv.Value.GetProto(&liveness); err != nil {
+			return nil, errors.Wrap(err, "invalid liveness record")
+		}
+
+		livenessRec := LivenessRecord{
+			Liveness: liveness,
+			raw:      kv.Value.TagAndDataBytes(),
+		}
+
+		// Update our cache with the liveness record we just found.
+		nl.maybeUpdate(ctx, livenessRec)
+
+		results = append(results, liveness)
+	}
+
+	return results, nil
+}
+
 // GetLiveness returns the liveness record for the specified nodeID. If the
 // liveness record is not found (due to gossip propagation delays or due to the
 // node not existing), we surface that to the caller. The record returned also