stats: table-level setting to turn auto stats collection on/off

Fixes cockroachdb#40989

Previously, there was no way to enable or disable automatic statistics
collection at the table level. It could only be turned on or off via the
`sql.stats.automatic_collection.enabled` cluster setting.

This was inadequate because statistics collection can be expensive for
large tables, and it would be desirable to defer collection until after
data is finished loading, or in off hours. Also, small tables which are
frequently updated may trigger statistics collection leading to
unnecessary overhead and/or unpredictable query plan changes.

To address this, this patch adds support for setting of the following
cluster settings at the table level:
```
sql.stats.automatic_collection.enabled
sql.stats.automatic_collection.fraction_stale_rows
sql.stats.automatic_collection.min_stale_rows
```
for example:
```
ALTER TABLE t1 SET ("sql.stats.automatic_collection.enabled" = true);
ALTER TABLE t1
      SET ("sql.stats.automatic_collection.fraction_stale_rows" = 0.1,
           "sql.stats.automatic_collection.min_stale_rows" = 2000);
```
The table-level setting takes precedence over the cluster setting.

Release justification: Low risk fix for missing fine-grained control
over automatic statistics collection.

Release note (sql change): Automatic statistics collection can now be
enabled or disabled for individual tables, taking precedence over the
cluster setting, for example:
```
ALTER TABLE t1 SET ("sql.stats.automatic_collection.enabled" = true);
ALTER TABLE t1 SET ("sql.stats.automatic_collection.enabled" = false);
ALTER TABLE t1 RESET ("sql.stats.automatic_collection.enabled");
```
RESET removes the setting value entirely, in which case the cluster
setting of the same name is in effect for the table.

Cluster settings `sql.stats.automatic_collection.fraction_stale_rows`
and `sql.stats.automatic_collection.min_stale_rows` can now also be
set at the table level, either at table creation time, or later,
independent of whether auto stats is enabled:
```
ALTER TABLE t1
      SET ("sql.stats.automatic_collection.fraction_stale_rows" = 0.1,
           "sql.stats.automatic_collection.min_stale_rows" = 2000);
CREATE TABLE t1 (a INT, b INT)
       WITH ("sql.stats.automatic_collection.enabled" = true,
             "sql.stats.automatic_collection.min_stale_rows" = 1000000,
             "sql.stats.automatic_collection.fraction_stale_rows" = 0.05
             );
```
The current table-level cluster settings, along with storage parameters,
is shown in the `WITH` clause output of `SHOW CREATE TABLE`.

pkg/settings/cluster/cluster_settings.go

@@ -78,6 +78,30 @@ func TelemetryOptOut() bool {
 // (for example, a CLI subcommand that does not connect to a cluster).
 var NoSettings *Settings // = nil
 
+// BoolSetting represents a boolean setting value.
+type BoolSetting int
+
+// The values for BoolSetting.
+const (
+	NotSet = iota // NotSet is like a null. The value has not been set.
+	True
+	False
+)
+
+const (
+	// AutoStatsEnabledSettingName is the name of the automatic stats collection
+	// enabled cluster setting.
+	AutoStatsEnabledSettingName = "sql.stats.automatic_collection.enabled"
+
+	// AutoStatsMinStaleSettingName is the name of the automatic stats collection
+	// min stale rows cluster setting.
+	AutoStatsMinStaleSettingName = "sql.stats.automatic_collection.min_stale_rows"
+
+	// AutoStatsFractionStaleSettingName is the name of the automatic stats
+	// collection fraction stale rows cluster setting.
+	AutoStatsFractionStaleSettingName = "sql.stats.automatic_collection.fraction_stale_rows"
+)
+
 // CPUProfileType tracks whether a CPU profile is in progress.
 type CPUProfileType int32
 

pkg/sql/catalog/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//pkg/kv",
         "//pkg/roachpb",
         "//pkg/server/telemetry",
+        "//pkg/settings/cluster",
         "//pkg/sql/catalog/catpb",
         "//pkg/sql/catalog/descpb",
         "//pkg/sql/pgwire/pgcode",

pkg/sql/catalog/catpb/catalog.proto

@@ -217,3 +217,22 @@ message RowLevelTTL {
   // the relation name.
   optional bool label_metrics = 10 [(gogoproto.nullable) = false];
 }
+
+// ClusterSettingsForTable represents cluster settings specified at the table
+// level. Each setting is nullable so queries of the descriptor in JSON form
+// only list values which have been set.
+message ClusterSettingsForTable {
+  option (gogoproto.equal) = true;
+  optional bool sql_stats_automatic_collection_enabled = 1;
+  optional int64 sql_stats_automatic_collection_min_stale_rows = 2;
+  optional double sql_stats_automatic_collection_fraction_stale_rows = 3;
+}
+
+// SessionSettingsForTable represents session settings specified at the table
+// level. Each setting is nullable so queries of the descriptor in JSON form
+// only list values which have been set.
+// This is currently unused, but defined for backwards compatibility in
+// anticipation of expected use going forward.
+message SessionSettingsForTable {
+  option (gogoproto.equal) = true;
+}

pkg/sql/catalog/descpb/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/keys",
+        "//pkg/settings/cluster",
         "//pkg/sql/catalog/catconstants",
         "//pkg/sql/catalog/catpb",
         "//pkg/sql/parser",

pkg/sql/catalog/descpb/structured.go

@@ -12,6 +12,7 @@ package descpb
 
 import (
 	"github.com/cockroachdb/cockroach/pkg/keys"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catconstants"
 	"github.com/cockroachdb/cockroach/pkg/sql/protoreflect"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/catid"
@@ -273,6 +274,55 @@ func (desc *TableDescriptor) Persistence() tree.Persistence {
 	return tree.PersistencePermanent
 }
 
+// AutoStatsCollectionEnabled indicates if automatic statistics collection is
+// explicitly enabled or disabled for this table.
+func (desc *TableDescriptor) AutoStatsCollectionEnabled() cluster.BoolSetting {
+	if desc.NoClusterSettingOverrides() {
+		return cluster.NotSet
+	}
+	if desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionEnabled == nil {
+		return cluster.NotSet
+	}
+	if *desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionEnabled {
+		return cluster.True
+	}
+	return cluster.False
+}
+
+// AutoStatsMinStaleRows indicates the setting of
+// sql.stats.automatic_collection.min_stale_rows for this table.
+// If ok is true, then the minStaleRows value is valid, otherwise this has not
+// been set at the table level.
+func (desc *TableDescriptor) AutoStatsMinStaleRows() (minStaleRows int64, ok bool) {
+	if desc.NoClusterSettingOverrides() {
+		return 0, false
+	}
+	if desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionMinStaleRows == nil {
+		return 0, false
+	}
+	return *desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionMinStaleRows, true
+}
+
+// AutoStatsFractionStaleRows indicates the setting of
+// sql.stats.automatic_collection.fraction_stale_rows for this table.
+// If ok is true, then the fractionStaleRows value is valid, otherwise this has
+// not been set at the table level.
+func (desc *TableDescriptor) AutoStatsFractionStaleRows() (fractionStaleRows float64, ok bool) {
+	if desc.NoClusterSettingOverrides() {
+		return 0, false
+	}
+	if desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionFractionStaleRows == nil {
+		return 0, false
+	}
+	return *desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionFractionStaleRows, true
+}
+
+// NoClusterSettingOverrides is true if no cluster settings are set at the
+// table level for the given table.
+func (desc *TableDescriptor) NoClusterSettingOverrides() bool {
+	return desc.ClusterSettingsForTable == nil
+}
+
 // IsVirtualTable returns true if the TableDescriptor describes a
 // virtual Table (like the information_schema tables) and thus doesn't
 // need to be physically stored.

pkg/sql/catalog/descpb/structured.proto

@@ -1198,7 +1198,13 @@ message TableDescriptor {
   optional uint32 next_constraint_id = 49 [(gogoproto.nullable) = false,
     (gogoproto.customname) = "NextConstraintID", (gogoproto.casttype) = "ConstraintID"];
 
-  // Next ID: 51
+  // ClusterSettingsForTable are cluster settings specified at the table level.
+  optional cockroach.sql.catalog.catpb.ClusterSettingsForTable cluster_settings_for_table = 51 [(gogoproto.customname)="ClusterSettingsForTable"];
+
+  // SessionSettingsForTable are session settings specified at the table level.
+  optional cockroach.sql.catalog.catpb.SessionSettingsForTable session_settings_for_table = 52 [(gogoproto.customname)="SessionSettingsForTable"];
+
+  // Next ID: 53
 }
 
 // SurvivalGoal is the survival goal for a database.

pkg/sql/catalog/descriptor.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
@@ -678,6 +679,23 @@ type TableDescriptor interface {
 	GetExcludeDataFromBackup() bool
 	// GetStorageParams returns a list of storage parameters for the table.
 	GetStorageParams(spaceBetweenEqual bool) []string
+	// NoClusterSettingOverrides is true if no cluster settings are set at the
+	// table level for the given table, meaning no cluster settings are overridden
+	// for this table.
+	NoClusterSettingOverrides() bool
+	// AutoStatsCollectionEnabled indicates if automatic statistics collection is
+	// explicitly enabled or disabled for this table.
+	AutoStatsCollectionEnabled() cluster.BoolSetting
+	// AutoStatsMinStaleRows indicates the setting of
+	// sql.stats.automatic_collection.min_stale_rows for this table.
+	// If ok is true, then the minStaleRows value is valid, otherwise this has not
+	// been set at the table level.
+	AutoStatsMinStaleRows() (minStaleRows int64, ok bool)
+	// AutoStatsFractionStaleRows indicates the setting of
+	// sql.stats.automatic_collection.fraction_stale_rows for this table.
+	// If ok is true, then the fractionStaleRows value is valid, otherwise this has
+	// not been set at the table level.
+	AutoStatsFractionStaleRows() (fractionStaleRows float64, ok bool)
 }
 
 // TypeDescriptor will eventually be called typedesc.Descriptor.

pkg/sql/catalog/tabledesc/structured.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
@@ -2565,6 +2566,24 @@ func (desc *wrapper) GetStorageParams(spaceBetweenEqual bool) []string {
 	if exclude := desc.GetExcludeDataFromBackup(); exclude {
 		appendStorageParam(`exclude_data_from_backup`, `true`)
 	}
+	if settings := desc.ClusterSettingsForTable; settings != nil {
+		// These need to be wrapped in double-quotes because they contain '.' chars.
+		if settings.SqlStatsAutomaticCollectionEnabled != nil {
+			value := *settings.SqlStatsAutomaticCollectionEnabled
+			appendStorageParam(fmt.Sprintf(`"%s"`, cluster.AutoStatsEnabledSettingName),
+				fmt.Sprintf("%v", value))
+		}
+		if settings.SqlStatsAutomaticCollectionMinStaleRows != nil {
+			value := *settings.SqlStatsAutomaticCollectionMinStaleRows
+			appendStorageParam(fmt.Sprintf(`"%s"`, cluster.AutoStatsMinStaleSettingName),
+				strconv.FormatInt(value, 10))
+		}
+		if settings.SqlStatsAutomaticCollectionFractionStaleRows != nil {
+			value := *settings.SqlStatsAutomaticCollectionFractionStaleRows
+			appendStorageParam(fmt.Sprintf(`"%s"`, cluster.AutoStatsFractionStaleSettingName),
+				fmt.Sprintf("%g", value))
+		}
+	}
 	return storageParams
 }
 
@@ -2583,6 +2602,26 @@ func (desc *wrapper) GetMultiRegionEnumDependencyIfExists() bool {
 	return false
 }
 
+// NoClusterSettingOverrides implements the TableDescriptor interface.
+func (desc *wrapper) NoClusterSettingOverrides() bool {
+	return desc.TableDesc().NoClusterSettingOverrides()
+}
+
+// AutoStatsCollectionEnabled implements the TableDescriptor interface.
+func (desc *wrapper) AutoStatsCollectionEnabled() cluster.BoolSetting {
+	return desc.TableDesc().AutoStatsCollectionEnabled()
+}
+
+// AutoStatsMinStaleRows implements the TableDescriptor interface.
+func (desc *wrapper) AutoStatsMinStaleRows() (minStaleRows int64, ok bool) {
+	return desc.TableDesc().AutoStatsMinStaleRows()
+}
+
+// AutoStatsFractionStaleRows implements the TableDescriptor interface.
+func (desc *wrapper) AutoStatsFractionStaleRows() (fractionStaleRows float64, ok bool) {
+	return desc.TableDesc().AutoStatsFractionStaleRows()
+}
+
 // SetTableLocalityRegionalByTable sets the descriptor's locality config to
 // regional at the table level in the supplied region. An empty region name
 // (or its alias PrimaryRegionNotSpecifiedName) denotes that the table is homed in

pkg/sql/catalog/tabledesc/validate.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catprivilege"
@@ -520,6 +522,8 @@ func (desc *wrapper) ValidateSelf(vea catalog.ValidationErrorAccumulator) {
 		}
 	}
 
+	desc.validateClusterSettingsForTable(vea.Report)
+
 	if desc.IsSequence() {
 		return
 	}
@@ -1501,3 +1505,61 @@ func (desc *wrapper) validatePartitioning() error {
 		)
 	})
 }
+
+// validateClusterSettingsForTable validates that any new cluster settings at
+// the table level hold a valid value.
+func (desc *wrapper) validateClusterSettingsForTable(errReportFn func(err error)) {
+	if desc.NoClusterSettingOverrides() {
+		return
+	}
+	desc.validateBoolSetting(errReportFn, desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionEnabled,
+		cluster.AutoStatsEnabledSettingName)
+	desc.validateIntSetting(errReportFn, desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionMinStaleRows,
+		cluster.AutoStatsMinStaleSettingName, settings.NonNegativeInt)
+	desc.validateFloatSetting(errReportFn, desc.ClusterSettingsForTable.SqlStatsAutomaticCollectionFractionStaleRows,
+		cluster.AutoStatsFractionStaleSettingName, settings.NonNegativeFloat)
+}
+
+func (desc *wrapper) verifyProperTableForStatsSetting(
+	errReportFn func(err error), settingName string,
+) {
+	if desc.IsVirtualTable() {
+		errReportFn(errors.Newf("Setting %s may not be set on virtual table", settingName))
+	}
+	if !desc.IsTable() {
+		errReportFn(errors.Newf("Setting %s may not be set on a view or sequence", settingName))
+	}
+}
+
+func (desc *wrapper) validateBoolSetting(
+	errReportFn func(err error), value *bool, settingName string,
+) {
+	if value != nil {
+		desc.verifyProperTableForStatsSetting(errReportFn, settingName)
+	}
+}
+
+func (desc *wrapper) validateIntSetting(
+	errReportFn func(err error), value *int64, settingName string, validateFunc func(v int64) error,
+) {
+	if value != nil {
+		desc.verifyProperTableForStatsSetting(errReportFn, settingName)
+		if err := validateFunc(*value); err != nil {
+			errReportFn(errors.Wrapf(err, "invalid value for %s", settingName))
+		}
+	}
+}
+
+func (desc *wrapper) validateFloatSetting(
+	errReportFn func(err error),
+	value *float64,
+	settingName string,
+	validateFunc func(v float64) error,
+) {
+	if value != nil {
+		desc.verifyProperTableForStatsSetting(errReportFn, settingName)
+		if err := validateFunc(*value); err != nil {
+			errReportFn(errors.Wrapf(err, "invalid value for %s", settingName))
+		}
+	}
+}