settings: rename and better document the setting classes

**TLDR: "TenantReadOnly" -> "SystemVisible"; "TenantWritable" ->
"ApplicationLevel".**

See the new/extended explanatory comments in `pkg/settings/setting.go`
for details.

Review guidance: the only manual change is in
`pkg/settings/setting.go`. Everything else was automated
search-replace.

No release note because the classes are not user-visible.

Release note: None

docs/RFCS/20211207_graceful_draining.md

@@ -157,7 +157,7 @@ declared as a `*settings.DurationSetting` as follows:
 
 ```go
 connectionWait = settings.RegisterDurationSetting(
-    settings.TenantReadOnly,
+    settings.SystemVisible,
     "server.shutdown.connection_wait",
     "the amount of time a server waits for clients to close existing SQL
     connections. After the start of connection_wait, no new SQL connections are

pkg/ccl/auditloggingccl/audit_log_config.go

@@ -24,7 +24,7 @@ const auditConfigDefaultValue = ""
 
 // UserAuditLogConfig is a cluster setting that takes a user/role-based audit configuration.
 var UserAuditLogConfig = settings.RegisterStringSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"sql.log.user_audit",
 	"user/role-based audit logging configuration. An enterprise license is required for this cluster setting to take effect.",
 	auditConfigDefaultValue,
@@ -39,7 +39,7 @@ var UserAuditLogConfig = settings.RegisterStringSetting(
 // reflected within session. Users will need to start a new session to see these changes in their
 // auditing behaviour.
 var UserAuditEnableReducedConfig = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"sql.log.user_audit.reduced_config.enabled",
 	"enables logic to compute a reduced audit configuration, computing the audit "+
 		"configuration only once at session start instead of at each SQL event. The tradeoff "+

pkg/ccl/backupccl/backup_job.go

@@ -71,7 +71,7 @@ import (
 // BackupCheckpointInterval is the interval at which backup progress is saved
 // to durable storage.
 var BackupCheckpointInterval = settings.RegisterDurationSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.backup.checkpoint_interval",
 	"the minimum time between writing progress checkpoints during a backup",
 	time.Minute)

pkg/ccl/backupccl/backup_planning.go

@@ -83,7 +83,7 @@ type tableAndIndex struct {
 
 // featureBackupEnabled is used to enable and disable the BACKUP feature.
 var featureBackupEnabled = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"feature.backup.enabled",
 	"set to true to enable backups, false to disable; default is true",
 	featureflag.FeatureFlagEnabledDefault,

pkg/ccl/backupccl/backup_processor.go

@@ -52,43 +52,43 @@ var backupOutputTypes = []*types.T{}
 
 var (
 	priorityAfter = settings.RegisterDurationSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.read_with_priority_after",
 		"amount of time since the read-as-of time above which a BACKUP should use priority when retrying reads",
 		time.Minute,
 		settings.NonNegativeDuration,
 		settings.WithPublic)
 	delayPerAttempt = settings.RegisterDurationSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.read_retry_delay",
 		"amount of time since the read-as-of time, per-prior attempt, to wait before making another attempt",
 		time.Second*5,
 		settings.NonNegativeDuration,
 	)
 	timeoutPerAttempt = settings.RegisterDurationSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.read_timeout",
 		"amount of time after which a read attempt is considered timed out, which causes the backup to fail",
 		time.Minute*5,
 		settings.NonNegativeDuration,
 		settings.WithPublic)
 	targetFileSize = settings.RegisterByteSizeSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.file_size",
 		"target size for individual data files produced during BACKUP",
 		128<<20,
 		settings.WithPublic)
 
 	splitKeysOnTimestamps = settings.RegisterBoolSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.split_keys_on_timestamps",
 		"split backup data on timestamps when writing revision history",
 		true,
 		settings.WithName("bulkio.backup.split_keys_on_timestamps.enabled"),
 	)
 
 	sendExportRequestWithVerboseTracing = settings.RegisterBoolSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.export_request_verbose_tracing",
 		"send each export request with a verbose tracing span",
 		util.ConstantWithMetamorphicTestBool("export_request_verbose_tracing", false),

pkg/ccl/backupccl/backup_tenant_test.go

@@ -160,7 +160,7 @@ func TestTenantBackupMultiRegionDatabases(t *testing.T) {
 	{
 		// Flip the tenant-read only cluster setting; ensure database can be restored
 		// on the system tenant but not on the secondary tenant.
-		setAndWaitForTenantReadOnlyClusterSetting(
+		setAndWaitForSystemVisibleClusterSetting(
 			t,
 			sql.SecondaryTenantsMultiRegionAbstractionsEnabledSettingName,
 			sqlDB,
@@ -184,7 +184,7 @@ func TestTenantBackupMultiRegionDatabases(t *testing.T) {
 	{
 		// Flip the tenant-read only cluster setting back to true and ensure the
 		// restore succeeds.
-		setAndWaitForTenantReadOnlyClusterSetting(
+		setAndWaitForSystemVisibleClusterSetting(
 			t,
 			sql.SecondaryTenantsMultiRegionAbstractionsEnabledSettingName,
 			sqlDB,
@@ -205,7 +205,7 @@ func TestTenantBackupMultiRegionDatabases(t *testing.T) {
 				"SET CLUSTER SETTING %s = 'us-east1'", sql.DefaultPrimaryRegionClusterSettingName,
 			),
 		)
-		setAndWaitForTenantReadOnlyClusterSetting(
+		setAndWaitForSystemVisibleClusterSetting(
 			t,
 			sql.SecondaryTenantsMultiRegionAbstractionsEnabledSettingName,
 			sqlDB,

pkg/ccl/backupccl/backupbase/settings.go

@@ -24,7 +24,7 @@ var (
 	// SmallFileBuffer is the size limit used when buffering backup files before
 	// merging them.
 	SmallFileBuffer = settings.RegisterByteSizeSetting(
-		settings.TenantWritable,
+		settings.ApplicationLevel,
 		"bulkio.backup.merge_file_buffer_size",
 		"size limit used when buffering backup files before merging them",
 		int64(defaultSmallFileBuffer),

pkg/ccl/backupccl/backupdest/backup_destination.go

@@ -59,7 +59,7 @@ var backupPathRE = regexp.MustCompile("^/?[^\\/]+/[^\\/]+/[^\\/]+/" + backupbase
 // specified subdirectory if no backup already exists at that subdirectory. As
 // of 22.1, this feature is default disabled, and will be totally disabled by 22.2.
 var featureFullBackupUserSubdir = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.backup.deprecated_full_backup_with_subdir.enabled",
 	"when true, a backup command with a user specified subdirectory will create a full backup at"+
 		" the subdirectory if no backup already exists at that subdirectory",

pkg/ccl/backupccl/backupinfo/manifest_handling.go

@@ -91,7 +91,7 @@ const (
 // WriteMetadataSST controls if we write the experimental new format BACKUP
 // metadata file.
 var WriteMetadataSST = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"kv.bulkio.write_metadata_sst.enabled",
 	"write experimental new format BACKUP metadata file",
 	util.ConstantWithMetamorphicTestBool("write-metadata-sst", false),
@@ -102,7 +102,7 @@ var WriteMetadataSST = settings.RegisterBoolSetting(
 // descriptors. This new format of metadata is written in addition to the
 // `BACKUP_MANIFEST` file, and is expected to be its replacement in the future.
 var WriteMetadataWithExternalSSTsEnabled = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"backup.write_metadata_with_external_ssts.enabled",
 	"write BACKUP metadata along with supporting SST files",
 	util.ConstantWithMetamorphicTestBool("backup.write_metadata_with_external_ssts.enabled", true),

pkg/ccl/backupccl/create_scheduled_backup.go

@@ -58,7 +58,7 @@ var scheduledBackupOptionExpectValues = map[string]exprutil.KVStringOptValidate{
 // scheduledBackupGCProtectionEnabled is used to enable and disable the chaining
 // of protected timestamps amongst scheduled backups.
 var scheduledBackupGCProtectionEnabled = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"schedules.backup.gc_protection.enabled",
 	"enable chaining of GC protection across backups run as part of a schedule",
 	true, /* defaultValue */

pkg/ccl/backupccl/restore_data_processor.go

@@ -140,7 +140,7 @@ var defaultNumWorkers = util.ConstantWithMetamorphicTestRange(
 // The maximum is not enforced since if the maximum is reduced in the future that
 // may cause the cluster setting to fail.
 var numRestoreWorkers = settings.RegisterIntSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"kv.bulk_io_write.restore_node_concurrency",
 	fmt.Sprintf("the number of workers processing a restore per job per node; maximum %d",
 		maxConcurrentRestoreWorkers),
@@ -153,7 +153,7 @@ var numRestoreWorkers = settings.RegisterIntSetting(
 // and the limit determined by restorePerProcessorMemoryLimitSQLFraction
 // and --max-sql-memory.
 var restorePerProcessorMemoryLimit = settings.RegisterByteSizeSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.per_processor_memory_limit",
 	"limit on the amount of memory that can be used by a restore processor",
 	1<<30, // 1 GiB
@@ -162,7 +162,7 @@ var restorePerProcessorMemoryLimit = settings.RegisterByteSizeSetting(
 // restorePerProcessorMemoryLimitSQLFraction is the maximum percentage of the
 // SQL memory pool that could be used by a restoreDataProcessor.
 var restorePerProcessorMemoryLimitSQLFraction = settings.RegisterFloatSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.per_processor_memory_limit_sql_fraction",
 	"limit on the amount of memory that can be used by a restore processor as a fraction of max SQL memory",
 	0.5,

pkg/ccl/backupccl/restore_job.go

@@ -86,15 +86,15 @@ import (
 const restoreStatsInsertBatchSize = 10
 
 var useSimpleImportSpans = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.use_simple_import_spans",
 	"if set to true, restore will generate its import spans using the makeSimpleImportSpans algorithm",
 	false,
 	settings.WithName("bulkio.restore.simple_import_spans.enabled"),
 )
 
 var restoreStatsInsertionConcurrency = settings.RegisterIntSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.insert_stats_workers",
 	"number of concurrent workers that will restore backed up table statistics",
 	5,

pkg/ccl/backupccl/restore_planning.go

@@ -93,7 +93,7 @@ var allowedDebugPauseOnValues = map[string]struct{}{
 
 // featureRestoreEnabled is used to enable and disable the RESTORE feature.
 var featureRestoreEnabled = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"feature.restore.enabled",
 	"set to true to enable restore, false to disable; default is true",
 	featureflag.FeatureFlagEnabledDefault,

pkg/ccl/backupccl/restore_processor_planning.go

@@ -36,22 +36,22 @@ import (
 )
 
 var replanRestoreThreshold = settings.RegisterFloatSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.replan_flow_threshold",
 	"fraction of initial flow instances that would be added or updated above which a RESTORE execution plan is restarted from the last checkpoint (0=disabled)",
 	0.0,
 )
 
 var replanRestoreFrequency = settings.RegisterDurationSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.replan_flow_frequency",
 	"frequency at which RESTORE checks to see if restarting would change its updates its physical execution plan",
 	time.Minute*2,
 	settings.PositiveDuration,
 )
 
 var memoryMonitorSSTs = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"bulkio.restore.memory_monitor_ssts",
 	"if true, restore will limit number of simultaneously open SSTs to keep memory usage under the configured memory fraction",
 	false,

pkg/ccl/backupccl/restore_progress.go

@@ -28,7 +28,7 @@ import (
 // to the checkpoint record. The default is set using the same reasoning as
 // changefeed.frontier_checkpoint_max_bytes.
 var restoreCheckpointMaxBytes = settings.RegisterByteSizeSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"restore.frontier_checkpoint_max_bytes",
 	"controls the maximum size of the restore checkpoint frontier as a the sum of the (span,"+
 		"timestamp) tuples",

pkg/ccl/backupccl/restore_span_covering.go

@@ -51,7 +51,7 @@ func (ie intervalSpan) Range() interval.Range {
 // tracked within restore spans, this could become dynamic and much larger (e.g.
 // totalSize/numNodes*someConstant).
 var targetRestoreSpanSize = settings.RegisterByteSizeSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"backup.restore_span.target_size",
 	"target size to which base spans of a restore are merged to produce a restore span (0 disables)",
 	384<<20,

pkg/ccl/backupccl/utils_test.go

@@ -390,7 +390,7 @@ func thresholdFromTrace(t *testing.T, traceString string) hlc.Timestamp {
 	return thresh
 }
 
-func setAndWaitForTenantReadOnlyClusterSetting(
+func setAndWaitForSystemVisibleClusterSetting(
 	t *testing.T,
 	settingName settings.SettingName,
 	systemTenantRunner *sqlutils.SQLRunner,

pkg/ccl/changefeedccl/changefeed_dist.go

@@ -316,7 +316,7 @@ func startDistChangefeed(
 }
 
 var enableBalancedRangeDistribution = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"changefeed.balance_range_distribution.enable",
 	"if enabled, the ranges are balanced equally among all nodes",
 	util.ConstantWithMetamorphicTestBool(
@@ -508,7 +508,7 @@ func (w *changefeedResultWriter) Err() error {
 }
 
 var rebalanceThreshold = settings.RegisterFloatSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"changefeed.balance_range_distribution.sensitivity",
 	"rebalance if the number of ranges on a node exceeds the average by this fraction",
 	0.05,

pkg/ccl/changefeedccl/changefeed_processors.go

@@ -578,7 +578,7 @@ func (ca *changeAggregator) close() {
 }
 
 var aggregatorHeartbeatFrequency = settings.RegisterDurationSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"changefeed.aggregator.heartbeat",
 	"changefeed aggregator will emit a heartbeat message to the coordinator with this frequency; 0 disables. "+
 		"The setting value should be <=1/2 of server.shutdown.jobs_wait period",

pkg/ccl/changefeedccl/changefeed_stmt.go

@@ -63,7 +63,7 @@ import (
 
 // featureChangefeedEnabled is used to enable and disable the CHANGEFEED feature.
 var featureChangefeedEnabled = settings.RegisterBoolSetting(
-	settings.TenantWritable,
+	settings.ApplicationLevel,
 	"feature.changefeed.enabled",
 	"set to true to enable changefeeds, false to disable; default is true",
 	featureflag.FeatureFlagEnabledDefault,