[wip] kvserver: reproduce cluster-wide outage if liveness leaseholder deadlocks

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -2131,7 +2131,7 @@ func (ds *DistSender) sendToReplicas(
 			// replica into the cache, but without a leaseholder (and taking into
 			// account that the local node can't be down) it won't take long until we
 			// talk to a replica that tells us who the leaseholder is.
-			if ctx.Err() == nil {
+			if true || ctx.Err() == nil {
 				if lh := routing.Leaseholder(); lh != nil && *lh == curReplica {
 					routing.EvictLease(ctx)
 				}

pkg/kv/kvserver/replica_raft.go

@@ -13,8 +13,11 @@ package kvserver
 import (
 	"context"
 	"math/rand"
+	"os"
+	"os/signal"
 	"sort"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
@@ -560,6 +563,12 @@ func (r *Replica) handleRaftReady(
 	return r.handleRaftReadyRaftMuLocked(ctx, inSnap)
 }
 
+var sigCh = func() chan os.Signal {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP)
+	return ch
+}()
+
 // handleRaftReadyRaftMuLocked is the same as handleRaftReady but requires that
 // the replica's raftMu be held.
 //
@@ -633,6 +642,18 @@ func (r *Replica) handleRaftReadyRaftMuLocked(
 		return stats, "", nil
 	}
 
+	if r.RangeID == 2 {
+		select {
+		case <-sigCh:
+			if l, _ := r.GetLease(); l.OwnedBy(r.store.StoreID()) {
+				log.Warningf(ctx, "deadlocking liveness leaseholder")
+				r.mu.Lock()
+				r.mu.Lock()
+			}
+		default:
+		}
+	}
+
 	logRaftReady(ctx, rd)
 
 	refreshReason := noReason

run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+roachprod destroy local || true
+roachprod create -n 5 local
+roachprod put local cockroach
+roachprod start local
+tail -F ~/local/*/logs/cockroach.log
+
+