From bea6fb6b469a384a07bf8bc230e4fb100eec4a77 Mon Sep 17 00:00:00 2001
From: Jeff <swenson@cockroachlabs.com>
Date: Fri, 3 Mar 2023 18:46:50 +0000
Subject: [PATCH 1/2] server: move settingswatcher initialization to the start

Previously, the settingswatcher was one of the last services initialized
during sql server start up. Now, it is one of the first services to
start up. The settings do not have a well defined value until after
settingwatcher initializes. Some settings, like cluster version, must
have valid values during initialization.

Part of #94843
---
 pkg/server/server_sql.go | 38 +++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/pkg/server/server_sql.go b/pkg/server/server_sql.go
index 0fdc9c4efd9e..097fb2e02bca 100644
--- a/pkg/server/server_sql.go
+++ b/pkg/server/server_sql.go
@@ -520,6 +520,19 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 	// SQL pod server.
 	_, isMixedSQLAndKVNode := cfg.nodeIDContainer.OptionalNodeID()
 
+	var settingsWatcher *settingswatcher.SettingsWatcher
+	if codec.ForSystemTenant() {
+		settingsWatcher = settingswatcher.New(
+			cfg.clock, codec, cfg.Settings, cfg.rangeFeedFactory, cfg.stopper, cfg.settingsStorage,
+		)
+	} else {
+		// Create the tenant settings watcher, using the tenant connector as the
+		// overrides monitor.
+		settingsWatcher = settingswatcher.NewWithOverrides(
+			cfg.clock, codec, cfg.Settings, cfg.rangeFeedFactory, cfg.stopper, cfg.tenantConnect, cfg.settingsStorage,
+		)
+	}
+
 	sqllivenessKnobs, _ := cfg.TestingKnobs.SQLLivenessKnobs.(*sqlliveness.TestingKnobs)
 	var sessionEventsConsumer slinstance.SessionEventListener
 	if !isMixedSQLAndKVNode {
@@ -1297,19 +1310,6 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 	vmoduleSetting.SetOnChange(&cfg.Settings.SV, fn)
 	fn(ctx)
 
-	var settingsWatcher *settingswatcher.SettingsWatcher
-	if codec.ForSystemTenant() {
-		settingsWatcher = settingswatcher.New(
-			cfg.clock, codec, cfg.Settings, cfg.rangeFeedFactory, cfg.stopper, cfg.settingsStorage,
-		)
-	} else {
-		// Create the tenant settings watcher, using the tenant connector as the
-		// overrides monitor.
-		settingsWatcher = settingswatcher.NewWithOverrides(
-			cfg.clock, codec, cfg.Settings, cfg.rangeFeedFactory, cfg.stopper, cfg.tenantConnect, cfg.settingsStorage,
-		)
-	}
-
 	return &SQLServer{
 		ambientCtx:                     cfg.BaseConfig.AmbientCtx,
 		stopper:                        cfg.stopper,
@@ -1365,6 +1365,13 @@ func (s *SQLServer) preStart(
 		}
 	}
 
+	// Initialize the settings watcher early in sql server startup. Settings
+	// values are meaningless before the watcher is initialized and most sub
+	// systems depend on system settings.
+	if err := s.settingsWatcher.Start(ctx); err != nil {
+		return errors.Wrap(err, "initializing settings")
+	}
+
 	// Load the multi-region enum by reading the system database's descriptor.
 	// This also serves as a simple check to see if a tenant exist (i.e. by
 	// checking whether the system db has been bootstrapped).
@@ -1499,11 +1506,8 @@ func (s *SQLServer) preStart(
 		bootstrapVersion = roachpb.Version{Major: 20, Minor: 1, Internal: 1}
 	}
 
-	if err := s.settingsWatcher.Start(ctx); err != nil {
-		return errors.Wrap(err, "initializing settings")
-	}
 	if err := s.systemConfigWatcher.Start(ctx, s.stopper); err != nil {
-		return errors.Wrap(err, "initializing settings")
+		return errors.Wrap(err, "initializing system config watcher")
 	}
 
 	clusterVersionMetrics := clusterversion.MakeMetricsAndRegisterOnVersionChangeCallback(&s.cfg.Settings.SV)

From 420a0aa1c260db2d48e075d2be7dd6d598a31c65 Mon Sep 17 00:00:00 2001
From: Faizan Qazi <faizan@cockroachlabs.com>
Date: Mon, 6 Mar 2023 19:37:05 -0500
Subject: [PATCH 2/2] sql/schmachanger: use more friendly language for fallback

Previously, the declarative schema changer used slightly
alarming language when falling back. To address this,
this patch removes the word failure and uses softer
language. Also the logging level is reduced to remove
this message by default.

Fixes: #97922

Release note: None
---
 pkg/sql/schemachanger/scerrors/errors.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/sql/schemachanger/scerrors/errors.go b/pkg/sql/schemachanger/scerrors/errors.go
index db4c49a251e8..3c70ae965646 100644
--- a/pkg/sql/schemachanger/scerrors/errors.go
+++ b/pkg/sql/schemachanger/scerrors/errors.go
@@ -75,7 +75,7 @@ func (el EventLogger) HandlePanicAndLogError(ctx context.Context, err *error) {
 			log.InfofDepth(ctx, depth, "done %s in %s", el.msg, redact.Safe(timeutil.Since(el.start)))
 		}
 	case HasNotImplemented(*err):
-		log.InfofDepth(ctx, depth, "failed %s with error: %v", el.msg, *err)
+		log.VEventfDepth(ctx, depth, 1, "declarative schema changer does not support %s: %v", el.msg, *err)
 	case errors.HasAssertionFailure(*err):
 		*err = errors.Wrapf(*err, "%s", el.msg)
 		fallthrough