cli: support SQL disk spilling in tenants

Previously, temp storage for SQL tenants was configured to point to memory
with a limit of 100MB. As a result, all ephemeral data when processing large
queries end up going to memory, and there was no way to configure this to
point to disk. This commit changes that behavior, and the default temp storage
for SQL tenants is now the same as SQL for dedicated, i.e. disk, with a limit
of 32GB. The operator can configure this through the `--max-disk-temp-storage`
flag.

Release note (cli change): `cockroach mt start-sql` will now support the
following flags to configure ephemeral storage for SQL when processing large
queries: `--store`, `--temp-dir`, and `--max-disk-temp-storage`.

Release note (sql change): SQL tenants will now spill to disk by default
when processing large queries, instead of memory.

pkg/cli/flags.go

@@ -949,6 +949,7 @@ func init() {
 		varFlag(f, addrSetter{&serverHTTPAddr, &serverHTTPPort}, cliflags.ListenHTTPAddr)
 		varFlag(f, addrSetter{&serverAdvertiseAddr, &serverAdvertisePort}, cliflags.AdvertiseAddr)
 
+		varFlag(f, &serverCfg.Stores, cliflags.Store)
 		stringFlag(f, &startCtx.geoLibsDir, cliflags.GeoLibsDir)
 
 		stringSliceFlag(f, &serverCfg.SQLConfig.TenantKVAddrs, cliflags.KVAddrs)
@@ -957,6 +958,11 @@ func init() {
 		boolFlag(f, &serverCfg.ExternalIODirConfig.DisableOutbound, cliflags.ExternalIODisabled)
 		boolFlag(f, &serverCfg.ExternalIODirConfig.DisableImplicitCredentials, cliflags.ExternalIODisableImplicitCredentials)
 
+		// N.B. diskTempStorageSizeValue.ResolvePercentage() will be called after
+		// the stores flag has been parsed and the storage device that a percentage
+		// refers to becomes known.
+		varFlag(f, diskTempStorageSizeValue, cliflags.SQLTempStorage)
+		stringFlag(f, &startCtx.tempDir, cliflags.TempDir)
 	}
 
 	// Multi-tenancy proxy command flags.

pkg/cli/mt_start_sql.go

@@ -15,7 +15,6 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/cli/clierrorplus"
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/server"
@@ -63,13 +62,6 @@ func runStartSQL(cmd *cobra.Command, args []string) error {
 	ctx := context.Background()
 	const clusterName = ""
 
-	// Remove the default store, which avoids using it to set up logging.
-	// Instead, we'll default to logging to stderr unless --log-dir is
-	// specified. This makes sense since the standalone SQL server is
-	// at the time of writing stateless and may not be provisioned with
-	// suitable storage.
-	serverCfg.Stores.Specs = nil
-
 	stopper, err := setupAndInitializeLoggingAndProfiling(ctx, cmd, false /* isServerCmd */)
 	if err != nil {
 		return err
@@ -94,21 +86,12 @@ func runStartSQL(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	tempStorageMaxSizeBytes := int64(base.DefaultInMemTempStorageMaxSizeBytes)
-	if err := diskTempStorageSizeValue.Resolve(
-		&tempStorageMaxSizeBytes, memoryPercentResolver,
+	if serverCfg.SQLConfig.TempStorageConfig, err = initTempStorageConfig(
+		ctx, serverCfg.Settings, stopper, serverCfg.Stores,
 	); err != nil {
 		return err
 	}
 
-	serverCfg.SQLConfig.TempStorageConfig = base.TempStorageConfigFromEnv(
-		ctx,
-		st,
-		base.StoreSpec{InMemory: true},
-		"", // parentDir
-		tempStorageMaxSizeBytes,
-	)
-
 	initGEOS(ctx)
 
 	sqlServer, addr, httpAddr, err := server.StartTenant(

pkg/cli/start.go

@@ -198,8 +198,37 @@ func initExternalIODir(ctx context.Context, firstStore base.StoreSpec) (string,
 }
 
 func initTempStorageConfig(
-	ctx context.Context, st *cluster.Settings, stopper *stop.Stopper, useStore base.StoreSpec,
+	ctx context.Context, st *cluster.Settings, stopper *stop.Stopper, stores base.StoreSpecList,
 ) (base.TempStorageConfig, error) {
+	// Initialize the target directory for temporary storage. If encryption at
+	// rest is enabled in any fashion, we'll want temp storage to be encrypted
+	// too. To achieve this, we use the first encrypted store as temp dir
+	// target, if any. If we can't find one, we use the first StoreSpec in the
+	// list.
+	//
+	// While we look, we also clean up any abandoned temporary directories. We
+	// don't know which store spec was used previously—and it may change if
+	// encryption gets enabled after the fact—so we check each store.
+	var specIdx = 0
+	for i, spec := range stores.Specs {
+		if spec.IsEncrypted() {
+			// TODO(jackson): One store's EncryptionOptions may say to encrypt
+			// with a real key, while another store's say to use key=plain.
+			// This provides no guarantee that we'll use the encrypted one's.
+			specIdx = i
+		}
+		if spec.InMemory {
+			continue
+		}
+		recordPath := filepath.Join(spec.Path, server.TempDirsRecordFilename)
+		if err := storage.CleanupTempDirs(recordPath); err != nil {
+			return base.TempStorageConfig{}, errors.Wrap(err,
+				"could not cleanup temporary directories from record file")
+		}
+	}
+
+	useStore := stores.Specs[specIdx]
+
 	var recordPath string
 	if !useStore.InMemory {
 		recordPath = filepath.Join(useStore.Path, server.TempDirsRecordFilename)
@@ -439,34 +468,8 @@ func runStart(cmd *cobra.Command, args []string, startSingleNode bool) (returnEr
 		return err
 	}
 
-	// Next we initialize the target directory for temporary storage.
-	// If encryption at rest is enabled in any fashion, we'll want temp
-	// storage to be encrypted too. To achieve this, we use
-	// the first encrypted store as temp dir target, if any.
-	// If we can't find one, we use the first StoreSpec in the list.
-	//
-	// While we look, we also clean up any abandoned temporary directories. We
-	// don't know which store spec was used previously—and it may change if
-	// encryption gets enabled after the fact—so we check each store.
-	var specIdx = 0
-	for i, spec := range serverCfg.Stores.Specs {
-		if spec.IsEncrypted() {
-			// TODO(jackson): One store's EncryptionOptions may say to encrypt
-			// with a real key, while another store's say to use key=plain.
-			// This provides no guarantee that we'll use the encrypted one's.
-			specIdx = i
-		}
-		if spec.InMemory {
-			continue
-		}
-		recordPath := filepath.Join(spec.Path, server.TempDirsRecordFilename)
-		if err := storage.CleanupTempDirs(recordPath); err != nil {
-			return errors.Wrap(err, "could not cleanup temporary directories from record file")
-		}
-	}
-
 	if serverCfg.TempStorageConfig, err = initTempStorageConfig(
-		ctx, serverCfg.Settings, stopper, serverCfg.Stores.Specs[specIdx],
+		ctx, serverCfg.Settings, stopper, serverCfg.Stores,
 	); err != nil {
 		return err
 	}