sql: keep track of statement error code and error message in

crdb_internal.execution_insights

Part of: cockroachdb#87785, cockroachdb#94381.

Previously, the insights subsystem did not keep track of error code and
error messages for failed executions and only contained a
`FailedExecution` value for the `problem` field. This commit adds
the `last_error_code` and `last_error_msg` columns to the
`crdb_internal.cluster_execution_insights` virtual table. The next PR
will focus on displaying these values in the UI.

Release note (sql change): Added `last_error_code` and `last_error_msg`
columns to the `crdb_internal.cluster_execution_insights` virtual table
which contain the error code and error message for a failed statement
execution, respectively.

pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant

@@ -309,6 +309,16 @@ SELECT * FROM crdb_internal.node_inflight_trace_spans WHERE span_id < 0
 ----
 trace_id  parent_span_id  span_id  goroutine_id  finished  start_time  duration  operation
 
+query TTTBTTTTTIITITTTTTTTTTTTTTTT colnames
+SELECT * FROM crdb_internal.cluster_execution_insights WHERE query = ''
+----
+session_id  txn_id  txn_fingerprint_id  stmt_id  stmt_fingerprint_id  problem  causes  query  status  start_time  end_time  full_scan  user_name  app_name  database_name  plan_gist  rows_read  rows_written  priority  retries  last_retry_reason  exec_node_ids  contention  contention_events  index_recommendations  implicit_txn  error_code  error_msg
+
+query TTTBTTTTTIITITTTTTTTTTTTTTTT colnames
+SELECT * FROM crdb_internal.node_execution_insights WHERE query = ''
+----
+session_id  txn_id  txn_fingerprint_id  stmt_id  stmt_fingerprint_id  problem  causes  query  status  start_time  end_time  full_scan  user_name  app_name  database_name  plan_gist  rows_read  rows_written  priority  retries  last_retry_reason  exec_node_ids  contention  contention_events  index_recommendations  implicit_txn  error_code  error_msg
+
 query TTTBTTTTTIITITTTTT colnames
 SELECT * FROM crdb_internal.cluster_txn_execution_insights WHERE query = ''
 ----

pkg/sql/crdb_internal.go

@@ -7119,7 +7119,9 @@ CREATE TABLE crdb_internal.%s (
 	contention                 INTERVAL,
 	contention_events          JSONB,
 	index_recommendations      STRING[] NOT NULL,
-	implicit_txn               BOOL NOT NULL
+	implicit_txn               BOOL NOT NULL,
+	error_code                 STRING,
+	error_msg                  STRING
 )`
 
 var crdbInternalClusterExecutionInsightsTable = virtualSchemaTable{
@@ -7249,6 +7251,8 @@ func populateStmtInsights(
 				contentionEvents,
 				indexRecommendations,
 				tree.MakeDBool(tree.DBool(insight.Transaction.ImplicitTxn)),
+				tree.NewDString(s.ErrorCode),
+				tree.NewDString(s.ErrorMsg),
 			))
 		}
 	}

pkg/sql/sqlstats/insights/detector.go

@@ -141,3 +141,7 @@ func (d *latencyThresholdDetector) enabled() bool {
 func (d *latencyThresholdDetector) isSlow(s *Statement) bool {
 	return d.enabled() && s.LatencyInSeconds >= LatencyThreshold.Get(&d.st.SV).Seconds()
 }
+
+func isFailed(s *Statement) bool {
+	return s.Status == Statement_Failed
+}

pkg/sql/sqlstats/insights/insights.proto

@@ -118,6 +118,8 @@ message Statement {
   repeated cockroach.roachpb.ContentionEvent contention_events = 20 [(gogoproto.nullable) = false];
   Problem problem = 21;
   repeated Cause causes = 22;
+  string error_code = 23;
+  string error_msg = 24;
 }
 
 

pkg/sql/sqlstats/insights/registry.go

@@ -97,10 +97,11 @@ func (r *lockingRegistry) ObserveTransaction(sessionID clusterunique.ID, transac
 	delete(r.statements, sessionID)
 	defer statements.release()
 
-	var slowStatements intsets.Fast
+	// Mark statements which are detected as slow or have a failed status.
+	var slowOrFailedStatements intsets.Fast
 	for i, s := range *statements {
-		if r.detector.isSlow(s) {
-			slowStatements.Add(i)
+		if r.detector.isSlow(s) || isFailed(s) {
+			slowOrFailedStatements.Add(i)
 		}
 	}
 
@@ -112,8 +113,8 @@ func (r *lockingRegistry) ObserveTransaction(sessionID clusterunique.ID, transac
 		highContention = transaction.Contention.Seconds() >= LatencyThreshold.Get(&r.causes.st.SV).Seconds()
 	}
 
-	if slowStatements.Empty() && !highContention {
-		// We only record an insight if we have slow statements or high txn contention.
+	if slowOrFailedStatements.Empty() && !highContention {
+		// We only record an insight if we have slow or failed statements or high txn contention.
 		return
 	}
 
@@ -127,14 +128,12 @@ func (r *lockingRegistry) ObserveTransaction(sessionID clusterunique.ID, transac
 	}
 
 	for i, s := range *statements {
-		if slowStatements.Contains(i) {
+		if slowOrFailedStatements.Contains(i) {
 			switch s.Status {
 			case Statement_Completed:
 				s.Problem = Problem_SlowExecution
 				s.Causes = r.causes.examine(s.Causes, s)
 			case Statement_Failed:
-				// Note that we'll be building better failure support for 23.1.
-				// For now, we only mark failed statements that were also slow.
 				s.Problem = Problem_FailedExecution
 			}
 

pkg/sql/sqlstats/ssmemstorage/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//pkg/settings",
         "//pkg/settings/cluster",
         "//pkg/sql/execstats",
+        "//pkg/sql/pgwire/pgerror",
         "//pkg/sql/sqlstats",
         "//pkg/sql/sqlstats/insights",
         "//pkg/util",

pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/execstats"
+	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlstats"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights"
 	"github.com/cockroachdb/cockroach/pkg/util"
@@ -185,6 +186,16 @@ func (s *Container) RecordStatement(
 		contentionEvents = value.ExecStats.ContentionEvents
 	}
 
+	var errorCode string
+	if value.StatementError != nil {
+		errorCode = pgerror.GetPGCode(value.StatementError).String()
+	}
+
+	var errorMsg string
+	if value.StatementError != nil {
+		errorMsg = value.StatementError.Error()
+	}
+
 	s.insights.ObserveStatement(value.SessionID, &insights.Statement{
 		ID:                   value.StatementID,
 		FingerprintID:        stmtFingerprintID,
@@ -204,6 +215,8 @@ func (s *Container) RecordStatement(
 		ContentionEvents:     contentionEvents,
 		IndexRecommendations: value.IndexRecommendations,
 		Database:             value.Database,
+		ErrorCode:            errorCode,
+		ErrorMsg:             errorMsg,
 	})
 
 	return stats.ID, nil