Merge #99315 #126038

99315: txnwait: do not require vmodule to log deadlocks in a workload r=nvanbenschoten a=arulajmani

Deadlocks in a workload are rare (and bad) enough that they should be logged at the level of Info, instead of under vmodule.

Epic: none

Release note: None

126038: ui: add transaction deadlocks graph to the SQL dashboard r=nvanbenschoten a=arulajmani

Any time these are non-zero should be a cause for concern, just like full table scans. Show these on a graph in the DB console.

Epic: none

Release note: None

Co-authored-by: Arul Ajmani <[email protected]>

pkg/kv/kvserver/txnwait/queue.go

@@ -263,8 +263,9 @@ type TestingKnobs struct {
 //
 // Queue is thread safe.
 type Queue struct {
-	cfg Config
-	mu  struct {
+	cfg   Config
+	every log.EveryN // dictates logging for transaction aborts resulting from deadlock detection
+	mu    struct {
 		syncutil.RWMutex
 		txns    map[uuid.UUID]*pendingTxn
 		queries map[uuid.UUID]*waitingQueries
@@ -273,7 +274,10 @@ type Queue struct {
 
 // NewQueue instantiates a new Queue.
 func NewQueue(cfg Config) *Queue {
-	return &Queue{cfg: cfg}
+	return &Queue{
+		cfg:   cfg,
+		every: log.Every(1 * time.Second),
+	}
 }
 
 // Enable allows transactions to be enqueued and waiting pushers
@@ -754,9 +758,15 @@ func (q *Queue) waitForPush(
 				// Break the deadlock if the pusher has higher priority.
 				p1, p2 := pusheePriority, pusherPriority
 				if p1 < p2 || (p1 == p2 && bytes.Compare(req.PusheeTxn.ID.GetBytes(), req.PusherTxn.ID.GetBytes()) < 0) {
+					// NB: It's useful to have logs indicating the transactions involved
+					// in a deadlock, but we don't want these to be too spammy.
+					level := log.Level(1)
+					if q.every.ShouldLog() {
+						level = 0 // will behave like a log.Infof
+					}
 					log.VEventf(
 						ctx,
-						1,
+						level,
 						"%s breaking deadlock by force push of %s; dependencies=%s",
 						req.PusherTxn.ID.Short(),
 						req.PusheeTxn.ID.Short(),

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/sql.tsx

@@ -231,6 +231,27 @@ export default function (props: GraphDashboardProps) {
       </Axis>
     </LineGraph>,
 
+    <LineGraph
+      title="Transaction Deadlocks"
+      isKvGraph={false}
+      sources={nodeSources}
+      tenantSource={tenantSource}
+      tooltip={`The total number of transaction per second; typically, should be 0 ${tooltipSelection}.`}
+      showMetricsInTooltip={true}
+    >
+      <Axis label="transaction deadlocks per second">
+        {map(nodeIDs, node => (
+          <Metric
+            key={node}
+            name="cr.store.txnwaitqueue.deadlocks_total"
+            title={nodeDisplayName(nodeDisplayNameByID, node)}
+            sources={[node]}
+            nonNegativeRate
+          />
+        ))}
+      </Axis>
+    </LineGraph>,
+
     <LineGraph
       title="Active Flows for Distributed SQL Statements"
       isKvGraph={false}