sql: add telemetry for statistics forecast usage

Add a few fields to the sampled_query telemetry events that will help us
measure how useful table statistics forecasting is in practice.

Fixes: cockroachdb#86356

Release note (ops change): Add five new fields to the sampled_query
telemetry events:
- `ScanCount`: Number of scans in the query plan.
- `ScanWithStatsCount`: Number of scans using statistics (including
  forecasted statistics) in the query plan.
- `ScanWithStatsForecastCount`: Number of scans using forecasted
  statistics in the query plan.
- `TotalScanRowsWithoutForecastsEstimate`: Total number of rows read by
  all scans in the query, as estimated by the optimizer without using
  forecasts.
- `NanosSinceStatsForecasted`: The greatest quantity of nanoseconds that
  have passed since the forecast time (or until the forecast time, if it
  is in the future, in which case it will be negative) for any table
  with forecasted stats scanned by this query.

docs/generated/eventlog.md

@@ -2816,6 +2816,11 @@ contains common SQL event/execution details.
 | `KVRowsRead` | The number of rows read at the KV layer for this query. | no |
 | `NetworkMessages` | The number of network messages sent by nodes for this query. | no |
 | `IndexRecommendations` | Generated index recommendations for this query. | no |
+| `ScanCount` | The number of scans in the query plan. | no |
+| `ScanWithStatsCount` | The number of scans using statistics (including forecasted statistics) in the query plan. | no |
+| `ScanWithStatsForecastCount` | The number of scans using forecasted statistics in the query plan. | no |
+| `TotalScanRowsWithoutForecastsEstimate` | Total number of rows read by all scans in the query, as estimated by the optimizer without using forecasts. | no |
+| `NanosSinceStatsForecasted` | The greatest quantity of nanoseconds that have passed since the forecast time (or until the forecast time, if it is in the future, in which case it will be negative) for any table with forecasted stats scanned by this query. | no |
 
 
 #### Common fields

pkg/sql/exec_log.go

@@ -457,6 +457,12 @@ func (p *planner) maybeLogStatementInternal(
 				KVRowsRead:               stats.KVRowsRead,
 				NetworkMessages:          stats.NetworkMessages,
 				IndexRecommendations:     indexRecs,
+
+				ScanCount:                             int64(p.curPlan.instrumentation.scanCounts[exec.ScanCount]),
+				ScanWithStatsCount:                    int64(p.curPlan.instrumentation.scanCounts[exec.ScanWithStatsCount]),
+				ScanWithStatsForecastCount:            int64(p.curPlan.instrumentation.scanCounts[exec.ScanWithStatsForecastCount]),
+				TotalScanRowsWithoutForecastsEstimate: p.curPlan.instrumentation.totalScanRowsWithoutForecasts,
+				NanosSinceStatsForecasted:             int64(p.curPlan.instrumentation.nanosSinceStatsForecasted),
 			}
 			p.logOperationalEventsOnlyExternally(ctx, isCopy, &sampledQuery)
 		} else {

pkg/sql/instrumentation.go

@@ -155,6 +155,11 @@ type instrumentationHelper struct {
 	// as estimated by the optimizer.
 	totalScanRows float64
 
+	// totalScanRowsWithoutForecasts is the total number of rows read by all scans
+	// in the query, as estimated by the optimizer without using forecasts. (If
+	// forecasts were not used, this should be the same as totalScanRows.)
+	totalScanRowsWithoutForecasts float64
+
 	// outputRows is the number of rows output by the query, as estimated by the
 	// optimizer.
 	outputRows float64
@@ -167,13 +172,22 @@ type instrumentationHelper struct {
 	// passed since stats were collected on any table scanned by this query.
 	nanosSinceStatsCollected time.Duration
 
+	// nanosSinceStatsForecasted is the greatest quantity of nanoseconds that have
+	// passed since the forecast time (or until the forecast time, if it is in the
+	// future, in which case it will be negative) for any table with forecasted
+	// stats scanned by this query.
+	nanosSinceStatsForecasted time.Duration
+
 	// joinTypeCounts records the number of times each type of logical join was
 	// used in the query.
 	joinTypeCounts map[descpb.JoinType]int
 
 	// joinAlgorithmCounts records the number of times each type of join algorithm
 	// was used in the query.
 	joinAlgorithmCounts map[exec.JoinAlgorithm]int
+
+	// scanCounts records the number of times scans were used in the query.
+	scanCounts [exec.NumScanCountTypes]int
 }
 
 // outputMode indicates how the statement output needs to be populated (for

pkg/sql/opt/exec/execbuilder/builder.go

@@ -140,10 +140,21 @@ type Builder struct {
 	// as estimated by the optimizer.
 	TotalScanRows float64
 
+	// TotalScanRowsWithoutForecasts is the total number of rows read by all scans
+	// in the query, as estimated by the optimizer without using forecasts. (If
+	// forecasts were not used, this should be the same as TotalScanRows.)
+	TotalScanRowsWithoutForecasts float64
+
 	// NanosSinceStatsCollected is the maximum number of nanoseconds that have
 	// passed since stats were collected on any table scanned by this query.
 	NanosSinceStatsCollected time.Duration
 
+	// NanosSinceStatsForecasted is the greatest quantity of nanoseconds that have
+	// passed since the forecast time (or until the forecast time, if the it is in
+	// the future, in which case it will be negative) for any table with
+	// forecasted stats scanned by this query.
+	NanosSinceStatsForecasted time.Duration
+
 	// JoinTypeCounts records the number of times each type of logical join was
 	// used in the query.
 	JoinTypeCounts map[descpb.JoinType]int
@@ -152,6 +163,9 @@ type Builder struct {
 	// was used in the query.
 	JoinAlgorithmCounts map[exec.JoinAlgorithm]int
 
+	// ScanCounts records the number of times scans were used in the query.
+	ScanCounts [exec.NumScanCountTypes]int
+
 	// wrapFunctionOverride overrides default implementation to return resolvable
 	// function reference for function with specified function name.
 	// The default can be overridden by calling SetBuiltinFuncWrapper method to provide

pkg/sql/opt/exec/execbuilder/relational.go

@@ -14,6 +14,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"math"
 	"strings"
 
 	"github.com/cockroachdb/cockroach/pkg/server/telemetry"
@@ -726,7 +727,8 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) {
 
 	// Save if we planned a full table/index scan on the builder so that the
 	// planner can be made aware later. We only do this for non-virtual tables.
-	stats := scan.Relational().Stats
+	relProps := scan.Relational()
+	stats := relProps.Stats
 	if !tab.IsVirtualTable() && isUnfiltered {
 		large := !stats.Available || stats.RowCount > b.evalCtx.SessionData().LargeFullScanRows
 		if scan.Index == cat.PrimaryIndex {
@@ -741,16 +743,50 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) {
 		}
 	}
 
-	// Save the total estimated number of rows scanned and the time since stats
-	// were collected.
+	// Save some instrumentation info.
+	b.ScanCounts[exec.ScanCount]++
 	if stats.Available {
 		b.TotalScanRows += stats.RowCount
-		if tab.StatisticCount() > 0 {
-			// The first stat is the most recent one.
-			nanosSinceStatsCollected := timeutil.Since(tab.Statistic(0).CreatedAt())
+		b.ScanCounts[exec.ScanWithStatsCount]++
+
+		// The first stat is the most recent one. Check if it was a forecast.
+		var first int
+		if first < tab.StatisticCount() && tab.Statistic(first).IsForecast() {
+			if b.evalCtx.SessionData().OptimizerUseForecasts {
+				b.ScanCounts[exec.ScanWithStatsForecastCount]++
+
+				// Calculate time since the forecast (or negative time until the forecast).
+				nanosSinceStatsForecasted := timeutil.Since(tab.Statistic(first).CreatedAt())
+				if nanosSinceStatsForecasted.Abs() > b.NanosSinceStatsForecasted.Abs() {
+					b.NanosSinceStatsForecasted = nanosSinceStatsForecasted
+				}
+			}
+			// Find the first non-forecast stat.
+			for first < tab.StatisticCount() && tab.Statistic(first).IsForecast() {
+				first++
+			}
+		}
+
+		if first < tab.StatisticCount() {
+			tabStat := tab.Statistic(first)
+
+			nanosSinceStatsCollected := timeutil.Since(tabStat.CreatedAt())
 			if nanosSinceStatsCollected > b.NanosSinceStatsCollected {
 				b.NanosSinceStatsCollected = nanosSinceStatsCollected
 			}
+
+			// Calculate another row count estimate using these (non-forecast)
+			// stats. If forecasts were not used, this should be the same as
+			// stats.RowCount.
+			rowCountWithoutForecast := float64(tabStat.RowCount())
+			rowCountWithoutForecast *= stats.Selectivity.AsFloat()
+			minCardinality, maxCardinality := relProps.Cardinality.Min, relProps.Cardinality.Max
+			if rowCountWithoutForecast > float64(maxCardinality) && maxCardinality != math.MaxUint32 {
+				rowCountWithoutForecast = float64(maxCardinality)
+			} else if rowCountWithoutForecast < float64(minCardinality) {
+				rowCountWithoutForecast = float64(minCardinality)
+			}
+			b.TotalScanRowsWithoutForecasts += rowCountWithoutForecast
 		}
 	}
 

pkg/sql/opt/exec/factory.go

@@ -386,3 +386,18 @@ const (
 	ZigZagJoin
 	NumJoinAlgorithms
 )
+
+// ScanCountType is the type of count of scan operations in a query.
+type ScanCountType int
+
+const (
+	// ScanCount is the count of all scans in a query.
+	ScanCount ScanCountType = iota
+	// ScanWithStatsCount is the count of scans with statistics in a query.
+	ScanWithStatsCount
+	// ScanWithStatsForecastCount is the count of scans which used forecasted
+	// statistics in a query.
+	ScanWithStatsForecastCount
+	// NumScanCountTypes is the total number of types of counts of scans.
+	NumScanCountTypes
+)

pkg/sql/plan_opt.go

@@ -656,9 +656,12 @@ func (opc *optPlanningCtx) runExecBuilder(
 		containsMutation = bld.ContainsMutation
 		planTop.instrumentation.maxFullScanRows = bld.MaxFullScanRows
 		planTop.instrumentation.totalScanRows = bld.TotalScanRows
+		planTop.instrumentation.totalScanRowsWithoutForecasts = bld.TotalScanRowsWithoutForecasts
 		planTop.instrumentation.nanosSinceStatsCollected = bld.NanosSinceStatsCollected
+		planTop.instrumentation.nanosSinceStatsForecasted = bld.NanosSinceStatsForecasted
 		planTop.instrumentation.joinTypeCounts = bld.JoinTypeCounts
 		planTop.instrumentation.joinAlgorithmCounts = bld.JoinAlgorithmCounts
+		planTop.instrumentation.scanCounts = bld.ScanCounts
 
 		planTop.instrumentation.RecordExplainPlan(explainPlan)
 	}