kvclient: add metrics for the txn over count after response

This commit adds a new metric txn.count_limit_on_response which tracks the
number of transactions that exceeded the max transaction count on their
response.

Epic: none

Release note (ops change): Add metric txn.count_limit_on_response.

docs/generated/metrics/metrics.html

@@ -1872,6 +1872,7 @@
 <tr><td>APPLICATION</td><td>txn.condensed_intent_spans</td><td>KV transactions that have exceeded their intent tracking memory budget (kv.transaction.max_intents_bytes). See also txn.condensed_intent_spans_gauge for a gauge of such transactions currently running.</td><td>KV Transactions</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>txn.condensed_intent_spans_gauge</td><td>KV transactions currently running that have exceeded their intent tracking memory budget (kv.transaction.max_intents_bytes). See also txn.condensed_intent_spans for a perpetual counter/rate.</td><td>KV Transactions</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>APPLICATION</td><td>txn.condensed_intent_spans_rejected</td><td>KV transactions that have been aborted because they exceeded their intent tracking memory budget (kv.transaction.max_intents_bytes). Rejection is caused by kv.transaction.reject_over_max_intents_budget.</td><td>KV Transactions</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>APPLICATION</td><td>txn.count_limit_on_response</td><td>KV transactions that have exceeded the count limit on a response</td><td>KV Transactions</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>txn.count_limit_rejected</td><td>KV transactions that have been aborted because they exceeded the max number of writes and locking reads allowed</td><td>KV Transactions</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>APPLICATION</td><td>txn.durations</td><td>KV transaction durations</td><td>KV Txn Duration</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>APPLICATION</td><td>txn.inflight_locks_over_tracking_budget</td><td>KV transactions whose in-flight writes and locking reads have exceeded the intent tracking memory budget (kv.transaction.max_intents_bytes).</td><td>KV Transactions</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>

pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go

@@ -740,6 +740,7 @@ func (tp *txnPipeliner) updateLockTracking(
 				"allowed by kv.transaction.max_intents_and_locks: "+
 				"count: %d, txn: %s, ba: %s", tp.writeCount, ba.Txn, ba.Summary())
 		}
+		tp.txnMetrics.TxnsResponseOverCountLimit.Inc(1)
 	}
 
 	// Deal with compacting the lock spans.

pkg/kv/kvclient/kvcoord/txn_metrics.go

@@ -35,6 +35,7 @@ type TxnMetrics struct {
 	TxnsWithCondensedIntentsGauge       *metric.Gauge
 	TxnsRejectedByLockSpanBudget        *metric.Counter
 	TxnsRejectedByCountLimit            *metric.Counter
+	TxnsResponseOverCountLimit          *metric.Counter
 	TxnsInFlightLocksOverTrackingBudget *metric.Counter
 
 	// Restarts is the number of times we had to restart the transaction.
@@ -181,6 +182,12 @@ var (
 		Measurement: "KV Transactions",
 		Unit:        metric.Unit_COUNT,
 	}
+	metaTxnsResponseOverCountLimit = metric.Metadata{
+		Name:        "txn.count_limit_on_response",
+		Help:        "KV transactions that have exceeded the count limit on a response",
+		Measurement: "KV Transactions",
+		Unit:        metric.Unit_COUNT,
+	}
 	metaTxnsInflightLocksOverTrackingBudget = metric.Metadata{
 		Name: "txn.inflight_locks_over_tracking_budget",
 		Help: "KV transactions whose in-flight writes and locking reads have exceeded " +
@@ -297,6 +304,7 @@ func MakeTxnMetrics(histogramWindow time.Duration) TxnMetrics {
 		TxnsWithCondensedIntentsGauge:       metric.NewGauge(metaTxnsWithCondensedIntentSpansGauge),
 		TxnsRejectedByLockSpanBudget:        metric.NewCounter(metaTxnsRejectedByLockSpanBudget),
 		TxnsRejectedByCountLimit:            metric.NewCounter(metaTxnsRejectedByCountLimit),
+		TxnsResponseOverCountLimit:          metric.NewCounter(metaTxnsResponseOverCountLimit),
 		TxnsInFlightLocksOverTrackingBudget: metric.NewCounter(metaTxnsInflightLocksOverTrackingBudget),
 		Restarts: metric.NewHistogram(metric.HistogramOptions{
 			Metadata:     metaRestartsHistogram,