release-22.1: sql: add sql.txn.contended.count metrics

pkg/server/status_test.go

@@ -3105,6 +3105,9 @@ func TestStatusAPIContentionEvents(t *testing.T) {
 	server1Conn := sqlutils.MakeSQLRunner(testCluster.ServerConn(0))
 	server2Conn := sqlutils.MakeSQLRunner(testCluster.ServerConn(1))
 
+	contentionCountBefore := testCluster.Server(1).SQLServer().(*sql.Server).
+		Metrics.EngineMetrics.SQLContendedTxns.Count()
+
 	sqlutils.CreateTable(
 		t,
 		testCluster.ServerConn(0),
@@ -3120,6 +3123,7 @@ func TestStatusAPIContentionEvents(t *testing.T) {
 
 	server1Conn.Exec(t, "USE test")
 	server2Conn.Exec(t, "USE test")
+	server2Conn.Exec(t, "SET application_name = 'contentionTest'")
 
 	server1Conn.Exec(t, `
 SET TRACING=on;
@@ -3164,6 +3168,29 @@ SET TRACING=off;
 
 	require.True(t, found,
 		"expect to find contention event for table %d, but found %+v", testTableID, resp)
+
+	server1Conn.CheckQueryResults(t, `
+  SELECT count(*)
+  FROM crdb_internal.statement_statistics
+  WHERE
+    (statistics -> 'execution_statistics' -> 'contentionTime' ->> 'mean')::FLOAT > 0
+    AND app_name = 'contentionTest'
+`, [][]string{{"1"}})
+
+	server1Conn.CheckQueryResults(t, `
+  SELECT count(*)
+  FROM crdb_internal.transaction_statistics
+  WHERE
+    (statistics -> 'execution_statistics' -> 'contentionTime' ->> 'mean')::FLOAT > 0
+    AND app_name = 'contentionTest'
+`, [][]string{{"1"}})
+
+	contentionCountNow := testCluster.Server(1).SQLServer().(*sql.Server).
+		Metrics.EngineMetrics.SQLContendedTxns.Count()
+
+	require.Greaterf(t, contentionCountNow, contentionCountBefore,
+		"expected txn contention count to be more than %d, but it is %d",
+		contentionCountBefore, contentionCountNow)
 }
 
 func TestStatusCancelSessionGatewayMetadataPropagation(t *testing.T) {

pkg/sql/conn_executor.go

@@ -425,6 +425,7 @@ func makeMetrics(internal bool) Metrics {
 				6*metricsSampleInterval),
 			SQLTxnsOpen:         metric.NewGauge(getMetricMeta(MetaSQLTxnsOpen, internal)),
 			SQLActiveStatements: metric.NewGauge(getMetricMeta(MetaSQLActiveQueries, internal)),
+			SQLContendedTxns:    metric.NewCounter(getMetricMeta(MetaSQLTxnContended, internal)),
 
 			TxnAbortCount:                     metric.NewCounter(getMetricMeta(MetaTxnAbort, internal)),
 			FailureCount:                      metric.NewCounter(getMetricMeta(MetaFailure, internal)),

pkg/sql/conn_executor_exec.go

@@ -470,7 +470,7 @@ func (ex *connExecutor) execStmtInOpenState(
 				ex.server.cfg,
 				ex.statsCollector,
 				&ex.extraTxnState.accumulatedStats,
-				ex.extraTxnState.shouldCollectTxnExecutionStats,
+				ih.collectExecStats,
 				p,
 				ast,
 				sql,
@@ -2214,6 +2214,10 @@ func (ex *connExecutor) recordTransactionFinish(
 	}
 	ex.metrics.EngineMetrics.SQLTxnLatency.RecordValue(txnTime.Nanoseconds())
 
+	if contentionDuration := ex.extraTxnState.accumulatedStats.ContentionTime.Nanoseconds(); contentionDuration > 0 {
+		ex.metrics.EngineMetrics.SQLContendedTxns.Inc(1)
+	}
+
 	ex.txnIDCacheWriter.Record(contentionpb.ResolvedTxnID{
 		TxnID:            ev.txnID,
 		TxnFingerprintID: transactionFingerprintID,
@@ -2241,7 +2245,7 @@ func (ex *connExecutor) recordTransactionFinish(
 		RetryLatency:            txnRetryLat,
 		CommitLatency:           commitLat,
 		RowsAffected:            ex.extraTxnState.numRows,
-		CollectedExecStats:      ex.extraTxnState.shouldCollectTxnExecutionStats,
+		CollectedExecStats:      ex.planner.instrumentation.collectExecStats,
 		ExecStats:               ex.extraTxnState.accumulatedStats,
 		RowsRead:                ex.extraTxnState.rowsRead,
 		RowsWritten:             ex.extraTxnState.rowsWritten,

pkg/sql/exec_util.go

@@ -845,6 +845,12 @@ var (
 		Measurement: "SQL Statements",
 		Unit:        metric.Unit_COUNT,
 	}
+	MetaSQLTxnContended = metric.Metadata{
+		Name:        "sql.txn.contended.count",
+		Help:        "Number of SQL transactions experienced contention",
+		Measurement: "Contention",
+		Unit:        metric.Unit_COUNT,
+	}
 	MetaSelectStarted = metric.Metadata{
 		Name:        "sql.select.started.count",
 		Help:        "Number of SQL SELECT statements started",

pkg/sql/executor_statement_metrics.go

@@ -40,6 +40,7 @@ type EngineMetrics struct {
 	SQLTxnLatency         *metric.Histogram
 	SQLTxnsOpen           *metric.Gauge
 	SQLActiveStatements   *metric.Gauge
+	SQLContendedTxns      *metric.Counter
 
 	// TxnAbortCount counts transactions that were aborted, either due
 	// to non-retriable errors, or retriable errors when the client-side

pkg/sql/instrumentation.go

@@ -196,11 +196,12 @@ func (ih *instrumentationHelper) Setup(
 			// collection is enabled so that stats are shown in the traces, but
 			// no extra work is needed by the instrumentationHelper.
 			ih.collectExecStats = true
-			// We still, however, want to finish the instrumentationHelper in
-			// case we're collecting a bundle. We also capture the span in order
-			// to fetch the trace from it, but the span won't be finished.
+			// We always want to finish the instrumentationHelper in order
+			// to record the execution statistics. Note that we capture the
+			// span in order to fetch the trace from it, but the span won't be
+			// finished.
 			ih.sp = sp
-			return ctx, ih.collectBundle
+			return ctx, true /* needFinish */
 		}
 	} else {
 		if buildutil.CrdbTestBuild {
@@ -244,7 +245,7 @@ func (ih *instrumentationHelper) Finish(
 	cfg *ExecutorConfig,
 	statsCollector sqlstats.StatsCollector,
 	txnStats *execstats.QueryLevelStats,
-	collectTxnExecStats bool,
+	collectExecStats bool,
 	p *planner,
 	ast tree.Statement,
 	stmtRawSQL string,
@@ -314,7 +315,7 @@ func (ih *instrumentationHelper) Finish(
 				log.Warningf(ctx, "unable to record statement exec stats: %s", err)
 			}
 		}
-		if collectTxnExecStats || ih.implicitTxn {
+		if collectExecStats || ih.implicitTxn {
 			txnStats.Accumulate(queryLevelStats)
 		}
 	}

pkg/ts/catalog/chart_catalog.go

@@ -2340,6 +2340,14 @@ var charts = []sectionDescription{
 				},
 				AxisLabel: "Count",
 			},
+			{
+				Title: "SQL Transaction Contention",
+				Metrics: []string{
+					"sql.txn.contended.count",
+					"sql.txn.contended.count.internal",
+				},
+				AxisLabel: "Transactions",
+			},
 		},
 	},
 	{