Merge pull request #84864 from THardy98/backport21.2-84182

release-21.2: sql: add statement fingerprint ID to sampled query

docs/generated/eventlog.md

@@ -2245,6 +2245,7 @@ contains common SQL event/execution details.
 | `StatementID` | Statement ID of the query. | no |
 | `TransactionID` | Transaction ID of the query. | no |
 | `DatabaseID` | Database ID of the query. | no |
+| `StatementFingerprintID` | Statement fingerprint ID of the query. | no |
 
 
 #### Common fields

pkg/sql/conn_executor_exec.go

@@ -984,6 +984,7 @@ func (ex *connExecutor) dispatchToExecutionEngine(
 		res.DisableBuffering()
 	}
 
+	var stmtFingerprintID roachpb.StmtFingerprintID
 	defer func() {
 		planner.maybeLogStatement(
 			ctx,
@@ -995,6 +996,7 @@ func (ex *connExecutor) dispatchToExecutionEngine(
 			ex.statsCollector.PhaseTimes().GetSessionPhaseTime(sessionphase.SessionQueryReceived),
 			&ex.extraTxnState.hasAdminRoleCache,
 			ex.server.TelemetryLoggingMetrics,
+			stmtFingerprintID,
 		)
 	}()
 
@@ -1081,7 +1083,7 @@ func (ex *connExecutor) dispatchToExecutionEngine(
 
 	// Record the statement summary. This also closes the plan if the
 	// plan has not been closed earlier.
-	ex.recordStatementSummary(
+	stmtFingerprintID = ex.recordStatementSummary(
 		ctx, planner,
 		ex.extraTxnState.autoRetryCounter, res.RowsAffected(), res.Err(), stats,
 	)

pkg/sql/exec_log.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
@@ -148,8 +149,9 @@ func (p *planner) maybeLogStatement(
 	queryReceived time.Time,
 	hasAdminRoleCache *HasAdminRoleCache,
 	telemetryLoggingMetrics *TelemetryLoggingMetrics,
+	stmtFingerprintID roachpb.StmtFingerprintID,
 ) {
-	p.maybeLogStatementInternal(ctx, execType, numRetries, txnCounter, rows, err, queryReceived, hasAdminRoleCache, telemetryLoggingMetrics)
+	p.maybeLogStatementInternal(ctx, execType, numRetries, txnCounter, rows, err, queryReceived, hasAdminRoleCache, telemetryLoggingMetrics, stmtFingerprintID)
 }
 
 func (p *planner) maybeLogStatementInternal(
@@ -160,6 +162,7 @@ func (p *planner) maybeLogStatementInternal(
 	startTime time.Time,
 	hasAdminRoleCache *HasAdminRoleCache,
 	telemetryMetrics *TelemetryLoggingMetrics,
+	stmtFingerprintID roachpb.StmtFingerprintID,
 ) {
 	// Note: if you find the code below crashing because p.execCfg == nil,
 	// do not add a test "if p.execCfg == nil { do nothing }" !
@@ -379,14 +382,15 @@ func (p *planner) maybeLogStatementInternal(
 			skippedQueries := telemetryMetrics.resetSkippedQueryCount()
 			databaseName := p.CurrentDatabase()
 			sampledQuery := eventpb.SampledQuery{
-				CommonSQLExecDetails: execDetails,
-				SkippedQueries:       skippedQueries,
-				CostEstimate:         p.curPlan.instrumentation.costEstimate,
-				Distribution:         p.curPlan.instrumentation.distribution.String(),
-				SessionID:            p.extendedEvalCtx.SessionID.String(),
-				Database:             p.CurrentDatabase(),
-				StatementID:          p.stmt.QueryID.String(),
-				TransactionID:        p.txn.ID().String(),
+				CommonSQLExecDetails:   execDetails,
+				SkippedQueries:         skippedQueries,
+				CostEstimate:           p.curPlan.instrumentation.costEstimate,
+				Distribution:           p.curPlan.instrumentation.distribution.String(),
+				SessionID:              p.extendedEvalCtx.SessionID.String(),
+				Database:               p.CurrentDatabase(),
+				StatementID:            p.stmt.QueryID.String(),
+				TransactionID:          p.txn.ID().String(),
+				StatementFingerprintID: uint64(stmtFingerprintID),
 			}
 			db, _ := p.Descriptors().GetImmutableDatabaseByName(ctx, p.txn, databaseName, tree.DatabaseLookupFlags{Required: true})
 			if db != nil {

pkg/sql/executor_statement_metrics.go

@@ -112,7 +112,7 @@ func (ex *connExecutor) recordStatementSummary(
 	rowsAffected int,
 	stmtErr error,
 	stats topLevelQueryStats,
-) {
+) roachpb.StmtFingerprintID {
 	phaseTimes := ex.statsCollector.PhaseTimes()
 
 	// Collect the statistics.
@@ -252,6 +252,7 @@ func (ex *connExecutor) recordStatementSummary(
 			sessionAge,
 		)
 	}
+	return stmtFingerprintID
 }
 
 func (ex *connExecutor) updateOptCounters(planFlags planFlags) {

pkg/sql/telemetry_logging_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -119,6 +120,7 @@ func TestTelemetryLogging(t *testing.T) {
 	testData := []struct {
 		name                    string
 		query                   string
+		queryNoConstants        string
 		execTimestampsSeconds   []float64 // Execute the query with the following timestamps.
 		expectedLogStatement    string
 		stubMaxEventFrequency   int64
@@ -133,6 +135,7 @@ func TestTelemetryLogging(t *testing.T) {
 			// logged since  we log all statements that are not of type DML.
 			"truncate-table-query",
 			"TRUNCATE t;",
+			"TRUNCATE TABLE t",
 			[]float64{1, 1.1, 1.2, 2},
 			`TRUNCATE TABLE`,
 			1,
@@ -145,6 +148,7 @@ func TestTelemetryLogging(t *testing.T) {
 			// The first statement should be logged.
 			"select-*-limit-1-query",
 			"SELECT * FROM t LIMIT 1;",
+			"SELECT * FROM t LIMIT _",
 			[]float64{3},
 			`SELECT * FROM \"\".\"\".t LIMIT ‹1›`,
 			1,
@@ -158,6 +162,7 @@ func TestTelemetryLogging(t *testing.T) {
 			// thus 2 log statements are expected, with 2 skipped queries.
 			"select-*-limit-2-query",
 			"SELECT * FROM t LIMIT 2;",
+			"SELECT * FROM t LIMIT _",
 			[]float64{4, 4.1, 4.2, 5},
 			`SELECT * FROM \"\".\"\".t LIMIT ‹2›`,
 			1,
@@ -170,6 +175,7 @@ func TestTelemetryLogging(t *testing.T) {
 			// Once required time has elapsed, the next statement should be logged.
 			"select-*-limit-3-query",
 			"SELECT * FROM t LIMIT 3;",
+			"SELECT * FROM t LIMIT _",
 			[]float64{6, 6.01, 6.05, 6.06, 6.1, 6.2},
 			`SELECT * FROM \"\".\"\".t LIMIT ‹3›`,
 			10,
@@ -272,6 +278,10 @@ func TestTelemetryLogging(t *testing.T) {
 				if !strings.Contains(e.Message, "\"DatabaseID\":"+strconv.Itoa(int(dbID))) {
 					t.Errorf("expected to find DatabaseID: %v", dbID)
 				}
+				stmtFingerprintID := roachpb.ConstructStatementFingerprintID(tc.queryNoConstants, false, true, databaseName)
+				if !strings.Contains(e.Message, "\"StatementFingerprintID\":"+strconv.FormatUint(uint64(stmtFingerprintID), 10)) {
+					t.Errorf("expected to find StatementFingerprintID: %v", stmtFingerprintID)
+				}
 			}
 		}
 		if logCount != expectedLogCount {

pkg/util/log/eventpb/telemetry.proto

@@ -58,6 +58,9 @@ message SampledQuery {
 
   // Database ID of the query.
   uint32 database_id = 12 [(gogoproto.customname) = "DatabaseID", (gogoproto.jsontag) = ",omitempty"];
+
+  // Statement fingerprint ID of the query.
+  uint64 statement_fingerprint_id = 13 [(gogoproto.customname) = "StatementFingerprintID", (gogoproto.jsontag) = ',omitempty'];
 }
 
 // CapturedIndexUsageStats