roachtest: run kv/contention/nodes=4 with an extreme TxnLivenessThres…

…hold

Fixes #36089.

This commit bumps the TxnLivenessThreshold for clusters running
`kv/contention/nodes=4` to 10 minutes. This is sufficiently large
such that if at any point a transaction is abandoned then all other
transactions will begin waiting for it and the test will fail to
achieve its minimum QPS requirement.

Release note: None

pkg/cmd/roachtest/kv.go

@@ -148,7 +148,13 @@ func registerKVContention(r *registry) {
 		Run: func(ctx context.Context, t *test, c *cluster) {
 			c.Put(ctx, cockroach, "./cockroach", c.Range(1, nodes))
 			c.Put(ctx, workload, "./workload", c.Node(nodes+1))
-			c.Start(ctx, t, c.Range(1, nodes))
+
+			// Start the cluster with an extremely high txn liveness threshold.
+			// If requests ever get stuck on a transaction that was abandoned
+			// then it will take 10m for them to get unstuck, at which point the
+			// QPS threshold check in the test is guaranteed to fail.
+			args := startArgs("--env=COCKROACH_TXN_LIVENESS_HEARTBEAT_MULTIPLIER=600")
+			c.Start(ctx, t, args, c.Range(1, nodes))
 
 			// Enable request tracing, which is a good tool for understanding
 			// how different transactions are interacting.

pkg/storage/txnwait/txnqueue.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/internal/client"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
@@ -37,12 +38,13 @@ const maxWaitForQueryTxn = 50 * time.Millisecond
 
 // TxnLivenessHeartbeatMultiplier specifies what multiple the transaction
 // liveness threshold should be of the transaction heartbeat internval.
-const TxnLivenessHeartbeatMultiplier = 5
+var TxnLivenessHeartbeatMultiplier = envutil.EnvOrDefaultInt(
+	"COCKROACH_TXN_LIVENESS_HEARTBEAT_MULTIPLIER", 5)
 
 // TxnLivenessThreshold is the maximum duration between transaction heartbeats
 // before the transaction is considered expired by Queue. It is exposed and
 // mutable to allow tests to override it.
-var TxnLivenessThreshold = TxnLivenessHeartbeatMultiplier * base.DefaultHeartbeatInterval
+var TxnLivenessThreshold = time.Duration(TxnLivenessHeartbeatMultiplier) * base.DefaultHeartbeatInterval
 
 // ShouldPushImmediately returns whether the PushTxn request should
 // proceed without queueing. This is true for pushes which are neither