sql,ccl: new tenant info field ServiceMode

Summary of changes:
- the new TenantInfo.ServiceMode field indicates how to run servers.
- new syntax: `ALTER TENANT ... START SERVICE EXTERNAL/SHARED`,
  `ALTER TENANT ... STOP SERVICE`.
- tenants created via `create_tenant(<id>)`
  (via CC serverless control plane) start in service mode EXTERNAL.
- other tenants start in service mode NONE.
- need ALTER TENANT STOP SERVICE before dropping a tenant.
  - except in the case of `crdb_internal.destroy_tenant`
    for compat with CC serverless control plane.

This commit does not contain the logic to start/stop
servers yet.

Release note: None

docs/generated/sql/bnf/stmt_block.bnf

@@ -1338,11 +1338,13 @@ unreserved_keyword ::=
 	| 'SEQUENCE'
 	| 'SEQUENCES'
 	| 'SERVER'
+	| 'SERVICE'
 	| 'SESSION'
 	| 'SESSIONS'
 	| 'SET'
 	| 'SETS'
 	| 'SHARE'
+	| 'SHARED'
 	| 'SHOW'
 	| 'SIMPLE'
 	| 'SKIP'
@@ -1361,6 +1363,7 @@ unreserved_keyword ::=
 	| 'STATEMENTS'
 	| 'STATISTICS'
 	| 'STDIN'
+	| 'STOP'
 	| 'STORAGE'
 	| 'STORE'
 	| 'STORED'

pkg/ccl/backupccl/backup_test.go

@@ -6885,7 +6885,7 @@ func TestBackupRestoreTenant(t *testing.T) {
 			{`1`,
 				`true`,
 				`system`,
-				`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+				`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 			},
 		})
 		restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/t10'`)
@@ -6896,13 +6896,13 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`10`,
 					`true`,
 					`tenant-10`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 			},
 		)
@@ -6936,13 +6936,13 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`10`,
 					`false`,
 					`NULL`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-10", "id": "10", "name": "", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-10", "id": "10", "name": "", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 			},
 		)
@@ -6971,13 +6971,13 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`10`,
 					`true`,
 					`tenant-10`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 			},
 		)
@@ -7006,7 +7006,7 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 			})
 		restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/t10'`)
@@ -7017,13 +7017,13 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`10`,
 					`true`,
 					`tenant-10`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 			},
 		)
@@ -7050,7 +7050,7 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 			})
 		restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/clusterwide'`)
@@ -7061,13 +7061,13 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`10`,
 					`true`,
 					`tenant-10`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 			},
 		)
@@ -7105,7 +7105,7 @@ func TestBackupRestoreTenant(t *testing.T) {
 					`1`,
 					`true`,
 					`system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 			})
 		restoreDB.Exec(t, `RESTORE FROM 'nodelocal://1/clusterwide'`)
@@ -7115,25 +7115,25 @@ func TestBackupRestoreTenant(t *testing.T) {
 				{
 					`1`,
 					`true`, `system`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`10`,
 					`true`,
 					`tenant-10`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "10", "name": "tenant-10", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`11`,
 					`true`,
 					`tenant-11`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "11", "name": "tenant-11", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "11", "name": "tenant-11", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 				{
 					`20`,
 					`true`,
 					`tenant-20`,
-					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "20", "name": "tenant-20", "tenantReplicationJobId": "0"}`,
+					`{"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "20", "name": "tenant-20", "serviceMode": "NONE", "tenantReplicationJobId": "0"}`,
 				},
 			},
 		)

pkg/ccl/backupccl/restore_job.go

@@ -1198,18 +1198,19 @@ func createImportingDescriptors(
 				if err != nil {
 					return err
 				}
-				for _, tenant := range details.Tenants {
-					switch tenant.DataState {
+				for _, tenantInfoCopy := range details.Tenants {
+					switch tenantInfoCopy.DataState {
 					case descpb.TenantInfo_READY:
 						// If the tenant was backed up in the `READY` state then we create
 						// the restored record in an `ADD` state and mark it `READY` at
 						// the end of the restore.
-						tenant.DataState = descpb.TenantInfo_ADD
+						tenantInfoCopy.ServiceMode = descpb.TenantInfo_NONE
+						tenantInfoCopy.DataState = descpb.TenantInfo_ADD
 					case descpb.TenantInfo_DROP, descpb.TenantInfo_ADD:
 					// If the tenant was backed up in a `DROP` or `ADD` state then we must
 					// create the restored tenant record in that state as well.
 					default:
-						return errors.AssertionFailedf("unknown tenant data state %v", tenant)
+						return errors.AssertionFailedf("unknown tenant data state %v", tenantInfoCopy)
 					}
 					spanConfigs := p.ExecCfg().SpanConfigKVAccessor.WithTxn(ctx, txn.KV())
 					if _, err := sql.CreateTenantRecord(
@@ -1218,7 +1219,7 @@ func createImportingDescriptors(
 						p.ExecCfg().Settings,
 						txn,
 						spanConfigs,
-						&tenant,
+						&tenantInfoCopy,
 						initialTenantZoneConfig,
 					); err != nil {
 						return err
@@ -2197,7 +2198,7 @@ func (r *restoreResumer) publishDescriptors(
 			// the tenant as the final step of the restore. The tenant has already
 			// been created at an earlier stage in the restore in an `ADD` state.
 			if err := sql.ActivateTenant(
-				ctx, r.execCfg.Settings, r.execCfg.Codec, txn, tenant.ID,
+				ctx, r.execCfg.Settings, r.execCfg.Codec, txn, tenant.ID, tenant.ServiceMode,
 			); err != nil {
 				return err
 			}

pkg/ccl/backupccl/testdata/backup-restore/restore-tenants

@@ -15,15 +15,15 @@ SELECT crdb_internal.create_tenant(6);
 
 # Drop one of them.
 exec-sql
-DROP TENANT [5]
+ALTER TENANT [5] STOP SERVICE; DROP TENANT [5]
 ----
 
 query-sql
 SELECT id,active,crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants;
 ----
-1 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}
-5 false {"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-5", "id": "5", "name": "", "tenantReplicationJobId": "0"}
-6 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "6", "name": "tenant-6", "tenantReplicationJobId": "0"}
+1 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}
+5 false {"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-5", "id": "5", "name": "", "serviceMode": "NONE", "tenantReplicationJobId": "0"}
+6 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "6", "name": "tenant-6", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}
 
 exec-sql
 BACKUP INTO 'nodelocal://1/cluster'
@@ -49,9 +49,9 @@ RESTORE FROM LATEST IN 'nodelocal://1/cluster'
 query-sql
 SELECT id,active,crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants;
 ----
-1 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}
-5 false {"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-5", "id": "5", "name": "tenant-5", "tenantReplicationJobId": "0"}
-6 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "6", "name": "tenant-6", "tenantReplicationJobId": "0"}
+1 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}
+5 false {"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-5", "id": "5", "name": "tenant-5", "serviceMode": "NONE", "tenantReplicationJobId": "0"}
+6 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "6", "name": "tenant-6", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}
 
 exec-sql expect-error-regex=(tenant 6 already exists)
 RESTORE TENANT 6 FROM LATEST IN 'nodelocal://1/tenant6';
@@ -75,7 +75,7 @@ RESTORE TENANT 6 FROM LATEST IN 'nodelocal://1/tenant6' WITH tenant_name = 'newn
 query-sql
 SELECT id,active,crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants;
 ----
-1 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "tenantReplicationJobId": "0"}
-2 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "2", "name": "newname", "tenantReplicationJobId": "0"}
-5 false {"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-5", "id": "5", "name": "tenant-5", "tenantReplicationJobId": "0"}
-6 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "6", "name": "tenant-6", "tenantReplicationJobId": "0"}
+1 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "1", "name": "system", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}
+2 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "2", "name": "newname", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}
+5 false {"capabilities": {"canAdminSplit": false}, "dataState": "DROP", "droppedName": "tenant-5", "id": "5", "name": "tenant-5", "serviceMode": "NONE", "tenantReplicationJobId": "0"}
+6 true {"capabilities": {"canAdminSplit": false}, "dataState": "READY", "droppedName": "", "id": "6", "name": "tenant-6", "serviceMode": "EXTERNAL", "tenantReplicationJobId": "0"}

pkg/ccl/logictestccl/testdata/logic_test/tenant_usage

@@ -20,6 +20,7 @@ SELECT crdb_internal.update_tenant_resource_limits(5, 1000, 100, 0, now(), 0)
 # Note this marks the tenant as dropped. The GC will not delete the tenant
 # until after the ttl expires.
 statement ok
+ALTER TENANT [5] STOP SERVICE;
 DROP TENANT [5]
 
 query error tenant "5" is not active

pkg/ccl/sqlproxyccl/tenant/directory_cache_test.go

@@ -405,14 +405,19 @@ func createTenant(tc serverutils.TestClusterInterface, id roachpb.TenantID) erro
 func destroyTenant(tc serverutils.TestClusterInterface, id roachpb.TenantID) error {
 	srv := tc.Server(0)
 	conn := srv.InternalExecutor().(*sql.InternalExecutor)
-	if _, err := conn.Exec(
-		context.Background(),
-		"testserver-destroy-tenant",
-		nil, /* txn */
-		"DROP TENANT [$1] IMMEDIATE",
-		id.ToUint64(),
-	); err != nil {
-		return err
+	for _, stmt := range []string{
+		`ALTER TENANT [$1] STOP SERVICE`,
+		`DROP TENANT [$1] IMMEDIATE`,
+	} {
+		if _, err := conn.Exec(
+			context.Background(),
+			"testserver-destroy-tenant",
+			nil, /* txn */
+			stmt,
+			id.ToUint64(),
+		); err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/ccl/streamingccl/streamingest/replication_stream_e2e_test.go

@@ -925,6 +925,7 @@ func TestTenantStreamingShowTenant(t *testing.T) {
 		id            int
 		dest          string
 		status        string
+		serviceMode   string
 		source        string
 		sourceUri     string
 		jobId         int
@@ -933,10 +934,11 @@ func TestTenantStreamingShowTenant(t *testing.T) {
 		cutoverTime   []byte // should be nil
 	)
 	row := c.DestSysSQL.QueryRow(t, fmt.Sprintf("SHOW TENANT %s WITH REPLICATION STATUS", args.DestTenantName))
-	row.Scan(&id, &dest, &status, &source, &sourceUri, &jobId, &maxReplTime, &protectedTime, &cutoverTime)
+	row.Scan(&id, &dest, &status, &serviceMode, &source, &sourceUri, &jobId, &maxReplTime, &protectedTime, &cutoverTime)
 	require.Equal(t, 2, id)
 	require.Equal(t, "destination", dest)
 	require.Equal(t, "REPLICATING", status)
+	require.Equal(t, "NONE", serviceMode)
 	require.Equal(t, "source", source)
 	require.Equal(t, c.SrcURL.String(), sourceUri)
 	require.Equal(t, ingestionJobID, jobId)

pkg/ccl/streamingccl/streamingest/stream_ingestion_planning.go

@@ -149,13 +149,14 @@ func ingestionPlanHook(
 				sourceTenant, dstTenantName, dstTenantID)
 		}
 
-		// Create a new tenant for the replication stream
+		// Create a new tenant for the replication stream.
 		jobID := p.ExecCfg().JobRegistry.MakeJobID()
 		tenantInfo := &descpb.TenantInfoWithUsage{
 			TenantInfo: descpb.TenantInfo{
 				// dstTenantID may be zero which will cause auto-allocation.
 				ID:                     dstTenantID,
 				DataState:              descpb.TenantInfo_ADD,
+				ServiceMode:            descpb.TenantInfo_NONE,
 				Name:                   roachpb.TenantName(dstTenantName),
 				TenantReplicationJobID: jobID,
 			},

pkg/ccl/streamingccl/streamingest/testdata/simple

@@ -28,14 +28,14 @@ IMPORT INTO d.x CSV DATA ('userfile:///dx/export*-n*.0.csv');
 query-sql as=source-system
 SHOW TENANTS
 ----
-1 system READY
-10 source READY
+1 system READY EXTERNAL
+10 source READY NONE
 
 query-sql as=destination-system
 SHOW TENANTS
 ----
-1 system READY
-2 destination REPLICATING
+1 system READY EXTERNAL
+2 destination REPLICATING NONE
 
 let $ts as=source-system
 SELECT clock_timestamp()::timestamp::string

pkg/ccl/testccl/sqlccl/tenant_gc_test.go

@@ -121,7 +121,9 @@ func TestGCTenantRemovesSpanConfigs(t *testing.T) {
 	) error {
 		return sql.TestingUpdateTenantRecord(
 			ctx, ts.ClusterSettings(), txn,
-			&descpb.TenantInfo{ID: tenantID.ToUint64(), DataState: descpb.TenantInfo_DROP},
+			&descpb.TenantInfo{ID: tenantID.ToUint64(),
+				ServiceMode: descpb.TenantInfo_NONE,
+				DataState:   descpb.TenantInfo_DROP},
 		)
 	}))
 

pkg/sql/BUILD.bazel

@@ -244,6 +244,7 @@ go_library(
         "tenant_creation.go",
         "tenant_deletion.go",
         "tenant_gc.go",
+        "tenant_service.go",
         "tenant_settings.go",
         "tenant_spec.go",
         "tenant_update.go",

pkg/sql/catalog/colinfo/result_columns.go

@@ -279,7 +279,8 @@ var ExportColumns = ResultColumns{
 var TenantColumns = ResultColumns{
 	{Name: "id", Typ: types.Int},
 	{Name: "name", Typ: types.String},
-	{Name: "status", Typ: types.String},
+	{Name: "data_status", Typ: types.String},
+	{Name: "service_mode", Typ: types.String},
 }
 
 // TenantColumnsWithReplication is appended to TenantColumns for