Merge cockroachdb#54425 cockroachdb#54453

54425: sql: add telemetry for partial indexes r=RaduBerinde a=mgartner

#### sql: add telemetry for partial index creation

This commit adds telemetry counters for the creation of partial index. A
counter is also added for the failed creation of a partial inverted
index.

Release note: None

#### opt: add telemetry for partial index usage

This commit adds telemetry counters for the usage of a partial index,
either in a scan or a lookup join.

Release note: None


54453: sql: periodically clear per app transaction statistics map r=otan a=arulajmani

For every app, we collect both statement and transaction statistics.
This datastructure is periodically reset to ensure it doesn't get
unboundedly large. This happens by manually resetting the
datastructures inside the app stats map instead of resetting the outer
map. Recently, we started collecting transaction statistics here in
addition to statement statistics, but we never added code to reset
the transaction statistics map. This patch corrects that oversight.

Fixes cockroachdb#54452

Release note (bug fix): There was a bug in transaction statistics
collection that could let the datastructure grow unboundedly large.
This is now fixed, and the resetting happens at the same cadence as
statement statistics.

Co-authored-by: Marcus Gartner <[email protected]>
Co-authored-by: arulajmani <[email protected]>

pkg/sql/app_stats.go

@@ -475,9 +475,9 @@ func (s *sqlStats) getStatsForApplication(appName string) *appStats {
 	return a
 }
 
-// resetAndMaybeDumpStats clears all the stored per-app and per-statement
-// statistics. If target s not nil, then the stats in s will be flushed
-// into target.
+// resetAndMaybeDumpStats clears all the stored per-app, per-statement and
+// per-transaction statistics. If target s not nil, then the stats in s will be
+// flushed into target.
 func (s *sqlStats) resetAndMaybeDumpStats(ctx context.Context, target *sqlStats) {
 	// Note: we do not clear the entire s.apps map here. We would need
 	// to do so to prevent problems with a runaway client running `SET
@@ -518,13 +518,14 @@ func (s *sqlStats) resetAndMaybeDumpStats(ctx context.Context, target *sqlStats)
 
 		// Only save a copy of a if we need to dump a copy of the stats.
 		if target != nil {
-			aCopy := &appStats{st: a.st, stmts: a.stmts}
+			aCopy := &appStats{st: a.st, stmts: a.stmts, txns: a.txns}
 			appStatsCopy[appName] = aCopy
 		}
 
 		// Clear the map, to release the memory; make the new map somewhat already
 		// large for the likely future workload.
 		a.stmts = make(map[stmtKey]*stmtStats, len(a.stmts)/2)
+		a.txns = make(map[txnKey]*txnStats, len(a.txns)/2)
 		a.Unlock()
 	}
 	s.lastReset = timeutil.Now()

pkg/sql/create_index.go

@@ -226,6 +226,7 @@ func MakeIndexDescriptor(
 
 	if n.Predicate != nil {
 		if n.Inverted {
+			telemetry.Inc(sqltelemetry.PartialInvertedIndexErrorCounter)
 			return nil, unimplemented.NewWithIssue(50952, "partial inverted indexes not supported")
 		}
 
@@ -235,6 +236,7 @@ func MakeIndexDescriptor(
 			return nil, err
 		}
 		indexDesc.Predicate = expr
+		telemetry.Inc(sqltelemetry.PartialIndexCounter)
 	}
 
 	if err := indexDesc.FillColumns(n.Columns); err != nil {

pkg/sql/create_table.go

@@ -1459,6 +1459,7 @@ func NewTableDesc(
 			}
 			if d.Predicate != nil {
 				if d.Inverted {
+					telemetry.Inc(sqltelemetry.PartialInvertedIndexErrorCounter)
 					return nil, unimplemented.NewWithIssue(50952, "partial inverted indexes not supported")
 				}
 
@@ -1467,6 +1468,7 @@ func NewTableDesc(
 					return nil, err
 				}
 				idx.Predicate = expr
+				telemetry.Inc(sqltelemetry.PartialIndexCounter)
 			}
 			if err := paramparse.ApplyStorageParameters(
 				ctx,
@@ -1515,6 +1517,7 @@ func NewTableDesc(
 					return nil, err
 				}
 				idx.Predicate = expr
+				telemetry.Inc(sqltelemetry.PartialIndexCounter)
 			}
 			if err := desc.AddIndex(idx, d.PrimaryKey); err != nil {
 				return nil, err

pkg/sql/opt/exec/execbuilder/relational.go

@@ -546,6 +546,11 @@ func (b *Builder) scanParams(
 func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) {
 	md := b.mem.Metadata()
 	tab := md.Table(scan.Table)
+
+	if !b.disableTelemetry && scan.UsesPartialIndex(md) {
+		telemetry.Inc(sqltelemetry.PartialIndexScanUseCounter)
+	}
+
 	params, outputCols, err := b.scanParams(tab, scan)
 	if err != nil {
 		return execPlan{}, err
@@ -1375,18 +1380,22 @@ func (b *Builder) buildIndexJoin(join *memo.IndexJoinExpr) (execPlan, error) {
 }
 
 func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) {
+	md := b.mem.Metadata()
+
 	if !b.disableTelemetry {
 		telemetry.Inc(sqltelemetry.JoinAlgoLookupUseCounter)
 		telemetry.Inc(opt.JoinTypeToUseCounter(join.JoinType))
+
+		if join.UsesPartialIndex(md) {
+			telemetry.Inc(sqltelemetry.PartialIndexLookupJoinUseCounter)
+		}
 	}
 
 	input, err := b.buildRelational(join.Input)
 	if err != nil {
 		return execPlan{}, err
 	}
 
-	md := b.mem.Metadata()
-
 	keyCols := make([]exec.NodeColumnOrdinal, len(join.KeyCols))
 	for i, c := range join.KeyCols {
 		keyCols[i] = input.getNodeColumnOrdinal(c)

pkg/sql/opt/memo/expr.go

@@ -649,6 +649,13 @@ func (s *ScanPrivate) PartialIndexPredicate(md *opt.Metadata) FiltersExpr {
 	return PartialIndexPredicate(tabMeta, s.Index)
 }
 
+// UsesPartialIndex returns true if the the LookupJoinPrivate looks-up via a
+// partial index.
+func (lj *LookupJoinPrivate) UsesPartialIndex(md *opt.Metadata) bool {
+	_, isPartialIndex := md.Table(lj.Table).Index(lj.Index).Predicate()
+	return isPartialIndex
+}
+
 // NeedResults returns true if the mutation operator can return the rows that
 // were mutated.
 func (m *MutationPrivate) NeedResults() bool {

pkg/sql/sqltelemetry/planning.go

@@ -150,6 +150,14 @@ var JoinTypeSemiUseCounter = telemetry.GetCounterOnce("sql.plan.opt.node.join.ty
 // planned.
 var JoinTypeAntiUseCounter = telemetry.GetCounterOnce("sql.plan.opt.node.join.type.anti")
 
+// PartialIndexScanUseCounter is to be incremented whenever a partial index scan
+// node is planned.
+var PartialIndexScanUseCounter = telemetry.GetCounterOnce("sql.plan.opt.partial-index.scan")
+
+// PartialIndexLookupJoinUseCounter is to be incremented whenever a lookup join
+// on a partial index is planned.
+var PartialIndexLookupJoinUseCounter = telemetry.GetCounterOnce("sql.plan.opt.partial-index.lookup-join")
+
 // CancelQueriesUseCounter is to be incremented whenever CANCEL QUERY or
 // CANCEL QUERIES is run.
 var CancelQueriesUseCounter = telemetry.GetCounterOnce("sql.session.cancel-queries")

pkg/sql/sqltelemetry/schema.go

@@ -66,6 +66,15 @@ var (
 	// geometry inverted index is created. These are a subset of the
 	// indexes counted in InvertedIndexCounter.
 	GeometryInvertedIndexCounter = telemetry.GetCounterOnce("sql.schema.geometry_inverted_index")
+
+	// PartialIndexCounter is to be incremented every time a partial index is
+	// created.
+	PartialIndexCounter = telemetry.GetCounterOnce("sql.schema.partial_index")
+
+	// PartialInvertedIndexErrorCounter is to be incremented every time a
+	// partial inverted index is attempted to be created, but fails because it
+	// is not yet supported.
+	PartialInvertedIndexErrorCounter = telemetry.GetCounterOnce("sql.schema.partial_inverted_index_error")
 )
 
 var (

pkg/sql/testdata/telemetry/partial_index

@@ -0,0 +1,38 @@
+# This file contains telemetry tests for sql.schema partial index creation
+# counters.
+
+feature-allowlist
+sql.schema.partial_index
+sql.schema.partial_inverted_index_error
+----
+
+feature-usage
+CREATE TABLE a (i INT, INDEX (i) WHERE i > 0)
+----
+sql.schema.partial_index
+
+feature-usage
+CREATE TABLE b (i INT, UNIQUE INDEX (i) WHERE i > 0)
+----
+sql.schema.partial_index
+
+feature-usage
+CREATE INDEX i ON b (i) WHERE i < 0
+----
+sql.schema.partial_index
+
+feature-usage
+CREATE TABLE c (i INT, j JSON, INVERTED INDEX (j) WHERE i > 0)
+----
+error: pq: unimplemented: partial inverted indexes not supported
+sql.schema.partial_inverted_index_error
+
+exec
+CREATE TABLE c (i INT, j JSON)
+----
+
+feature-usage
+CREATE INVERTED INDEX i ON c (j) WHERE i > 0
+----
+error: pq: unimplemented: partial inverted indexes not supported
+sql.schema.partial_inverted_index_error

pkg/sql/testdata/telemetry/planning

@@ -222,3 +222,25 @@ NATURAL INNER MERGE JOIN (SELECT * FROM x WHERE a > 0) AS x2
 ----
 sql.plan.opt.node.join.algo.merge
 sql.plan.opt.node.join.type.inner
+
+# Tests for partial index counters.
+
+feature-allowlist
+sql.plan.opt.partial-index.scan
+sql.plan.opt.partial-index.lookup-join
+----
+
+exec
+CREATE INDEX i ON x (a) WHERE a > 0
+----
+
+feature-usage
+SELECT a FROM x@i WHERE a > 0
+----
+sql.plan.opt.partial-index.scan
+
+feature-usage
+SELECT x1.a FROM x x1 INNER LOOKUP JOIN x x2 ON x1.a = x2.a WHERE x2.a > 0
+----
+sql.plan.opt.partial-index.lookup-join
+sql.plan.opt.partial-index.scan