server: add ./cockroach mt start-sql

pkg/cli/cliflags/flags_mt.go

@@ -0,0 +1,26 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cliflags
+
+// Flags specific to multi-tenancy commands.
+var (
+	TenantID = FlagInfo{
+		Name:        "tenant-id",
+		EnvVar:      "COCKROACH_TENANT_ID",
+		Description: `The tenant ID under which to start the SQL server.`,
+	}
+
+	KVAddrs = FlagInfo{
+		Name:        "kv-addrs",
+		EnvVar:      "COCKROACH_KV_ADDRS",
+		Description: `A comma-separated list of KV endpoints (load balancers allowed).`,
+	}
+)

pkg/cli/context.go

@@ -109,18 +109,19 @@ func initCLIDefaults() {
 	debugCtx.maxResults = 0
 	debugCtx.ballastSize = base.SizeSpec{InBytes: 1000000000}
 
-	serverCfg.GoroutineDumpDirName = ""
-	serverCfg.HeapProfileDirName = ""
-	serverCfg.ReadyFn = nil
-	serverCfg.DelayedBootstrapFn = nil
-	serverCfg.SocketFile = ""
-	serverCfg.JoinList = nil
-	serverCfg.JoinPreferSRVRecords = false
-	serverCfg.DefaultZoneConfig = zonepb.DefaultZoneConfig()
-	serverCfg.DefaultSystemZoneConfig = zonepb.DefaultSystemZoneConfig()
+	serverCfg.KVConfig.GoroutineDumpDirName = ""
+	serverCfg.KVConfig.HeapProfileDirName = ""
+	serverCfg.KVConfig.ReadyFn = nil
+	serverCfg.KVConfig.DelayedBootstrapFn = nil
+	serverCfg.SQLConfig.SocketFile = ""
+	serverCfg.KVConfig.JoinList = nil
+	serverCfg.TenantKVAddrs = []string{"127.0.0.1:26257"}
+	serverCfg.KVConfig.JoinPreferSRVRecords = false
+	serverCfg.BaseConfig.DefaultZoneConfig = zonepb.DefaultZoneConfig()
+	serverCfg.KVConfig.DefaultSystemZoneConfig = zonepb.DefaultSystemZoneConfig()
 	// Attempt to default serverCfg.MemoryPoolSize to 25% if possible.
 	if bytes, _ := memoryPercentResolver(25); bytes != 0 {
-		serverCfg.MemoryPoolSize = bytes
+		serverCfg.SQLConfig.MemoryPoolSize = bytes
 	}
 
 	startCtx.serverInsecure = baseCfg.Insecure

pkg/cli/flags.go

@@ -15,11 +15,13 @@ import (
 	"net"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/cli/cliflags"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/security"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
@@ -246,8 +248,11 @@ func init() {
 		return setDefaultStderrVerbosity(cmd, log.Severity_WARNING)
 	})
 
-	// Add a pre-run command for `start` and `start-single-node`.
-	for _, cmd := range StartCmds {
+	// Add a pre-run command for `start` and `start-single-node`, as well as the
+	// multi-tenancy related commands that start long-running servers.
+	allStartCmds := append([]*cobra.Command(nil), StartCmds...)
+	allStartCmds = append(allStartCmds, mtStartSQLCmd)
+	for _, cmd := range allStartCmds {
 		AddPersistentPreRunE(cmd, func(cmd *cobra.Command, _ []string) error {
 			// Finalize the configuration of network and logging settings.
 			if err := extraServerFlagInit(cmd); err != nil {
@@ -704,6 +709,44 @@ func init() {
 		f := debugBallastCmd.Flags()
 		VarFlag(f, &debugCtx.ballastSize, cliflags.Size)
 	}
+
+	// Multi-tenancy commands.
+	{
+		f := mtStartSQLCmd.Flags()
+		VarFlag(f, &tenantIDWrapper{&serverCfg.SQLConfig.TenantID}, cliflags.TenantID)
+		// NB: serverInsecure populates baseCfg.{Insecure,SSLCertsDir} in this the following method
+		// (which is a PreRun for this command):
+		_ = extraServerFlagInit // guru assignment
+		BoolFlag(f, &startCtx.serverInsecure, cliflags.ServerInsecure, startCtx.serverInsecure)
+		StringFlag(f, &startCtx.serverSSLCertsDir, cliflags.ServerCertsDir, startCtx.serverSSLCertsDir)
+		// NB: this also gets PreRun treatment via extraServerFlagInit to populate BaseCfg.SQLAddr.
+		VarFlag(f, addrSetter{&serverSQLAddr, &serverSQLPort}, cliflags.ListenSQLAddr)
+
+		StringSlice(f, &serverCfg.SQLConfig.TenantKVAddrs, cliflags.KVAddrs, serverCfg.SQLConfig.TenantKVAddrs)
+	}
+}
+
+type tenantIDWrapper struct {
+	tenID *roachpb.TenantID
+}
+
+func (w *tenantIDWrapper) String() string {
+	return w.tenID.String()
+}
+func (w *tenantIDWrapper) Set(s string) error {
+	tenID, err := strconv.ParseUint(s, 10, 64)
+	if err != nil {
+		return errors.Wrap(err, "invalid tenant ID")
+	}
+	if tenID == 0 {
+		return errors.New("invalid tenant ID")
+	}
+	*w.tenID = roachpb.MakeTenantID(tenID)
+	return nil
+}
+
+func (w *tenantIDWrapper) Type() string {
+	return "number"
 }
 
 // processEnvVarDefaults injects the current value of flag-related
@@ -786,14 +829,24 @@ func extraServerFlagInit(cmd *cobra.Command) error {
 
 	fs := flagSetForCmd(cmd)
 
-	// Construct the socket name, if requested.
-	if !fs.Lookup(cliflags.Socket.Name).Changed && fs.Lookup(cliflags.SocketDir.Name).Changed {
-		// If --socket (DEPRECATED) was set, then serverCfg.SocketFile is
-		// already set and we don't want to change it.
-		// However, if --socket-dir is set, then we'll use that.
-		// There are two cases:
-		// --socket-dir is set and is empty; in this case the user is telling us "disable the socket".
-		// is set and non-empty. Then it should be used as specified.
+	// Helper for .Changed that is nil-aware as not all of the `cmd`s may have
+	// all of the flags.
+	changed := func(set *pflag.FlagSet, name string) bool {
+		f := set.Lookup(name)
+		return f != nil && f.Changed
+	}
+
+	// Construct the socket name, if requested. The flags may not be defined for
+	// `cmd` so be cognizant of that.
+	//
+	// If --socket (DEPRECATED) was set, then serverCfg.SocketFile is
+	// already set and we don't want to change it.
+	// However, if --socket-dir is set, then we'll use that.
+	// There are two cases:
+	// 1. --socket-dir is set and is empty; in this case the user is telling us
+	//    "disable the socket".
+	// 2. is set and non-empty. Then it should be used as specified.
+	if !changed(fs, cliflags.Socket.Name) && changed(fs, cliflags.SocketDir.Name) {
 		if serverSocketDir == "" {
 			serverCfg.SocketFile = ""
 		} else {
@@ -820,9 +873,9 @@ func extraServerFlagInit(cmd *cobra.Command) error {
 	serverCfg.SQLAddr = net.JoinHostPort(serverSQLAddr, serverSQLPort)
 	serverCfg.SplitListenSQL = fs.Lookup(cliflags.ListenSQLAddr.Name).Changed
 
-	// Fill in the defaults for --advertise-sql-addr.
-	advSpecified := fs.Lookup(cliflags.AdvertiseAddr.Name).Changed ||
-		fs.Lookup(cliflags.AdvertiseHost.Name).Changed
+	// Fill in the defaults for --advertise-sql-addr, if the flag exists on `cmd`.
+	advSpecified := changed(fs, cliflags.AdvertiseAddr.Name) ||
+		changed(fs, cliflags.AdvertiseHost.Name)
 	if serverSQLAdvertiseAddr == "" {
 		if advSpecified {
 			serverSQLAdvertiseAddr = serverAdvertiseAddr
@@ -851,8 +904,8 @@ func extraServerFlagInit(cmd *cobra.Command) error {
 		// Before we do so, we'll check whether the user explicitly
 		// specified something contradictory, and tell them that's no
 		// good.
-		if (fs.Lookup(cliflags.ListenHTTPAddr.Name).Changed ||
-			fs.Lookup(cliflags.ListenHTTPAddrAlias.Name).Changed) &&
+		if (changed(fs, cliflags.ListenHTTPAddr.Name) ||
+			changed(fs, cliflags.ListenHTTPAddrAlias.Name)) &&
 			(serverHTTPAddr != "" && serverHTTPAddr != "localhost") {
 			return errors.WithHintf(
 				errors.Newf("--unencrypted-localhost-http is incompatible with --http-addr=%s:%s",

pkg/cli/mt.go

@@ -0,0 +1,31 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cli
+
+import "github.com/spf13/cobra"
+
+func init() {
+	cockroachCmd.AddCommand(mtCmd)
+	mtCmd.AddCommand(mtStartSQLCmd)
+}
+
+// mtCmd is the base command for functionality related to multi-tenancy.
+var mtCmd = &cobra.Command{
+	Use:   "mt [command]",
+	Short: "commands related to multi-tenancy",
+	Long: `
+Commands related to multi-tenancy.
+
+This functionality is **experimental** and for internal use only.
+`,
+	RunE:   usageAndErr,
+	Hidden: true,
+}

pkg/cli/mt_start_sql.go

@@ -0,0 +1,88 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+
+	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
+	"github.com/cockroachdb/cockroach/pkg/server"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/stop"
+	"github.com/cockroachdb/errors"
+	"github.com/spf13/cobra"
+)
+
+var mtStartSQLCmd = &cobra.Command{
+	Use:   "start-sql",
+	Short: "start a standalone SQL server",
+	Long: `
+Start a standalone SQL server.
+
+This functionality is **experimental** and for internal use only.
+`,
+	Args: cobra.NoArgs,
+	RunE: MaybeDecorateGRPCError(runStartSQL),
+}
+
+func runStartSQL(cmd *cobra.Command, args []string) error {
+	ctx := context.Background()
+	const clusterName = ""
+	stopper := stop.NewStopper()
+	defer stopper.Stop(ctx)
+
+	st := serverCfg.BaseConfig.Settings
+
+	// TODO(tbg): this has to be passed in. See the upgrade strategy in:
+	// https://github.com/cockroachdb/cockroach/issues/47919
+	if err := clusterversion.Initialize(ctx, st.Version.BinaryVersion(), &st.SV); err != nil {
+		return err
+	}
+
+	tempStorageMaxSizeBytes := int64(base.DefaultInMemTempStorageMaxSizeBytes)
+	if err := diskTempStorageSizeValue.Resolve(
+		&tempStorageMaxSizeBytes, memoryPercentResolver,
+	); err != nil {
+		return err
+	}
+
+	serverCfg.SQLConfig.TempStorageConfig = base.TempStorageConfigFromEnv(
+		ctx,
+		st,
+		base.StoreSpec{InMemory: true},
+		"", // parentDir
+		tempStorageMaxSizeBytes,
+	)
+
+	addr, err := server.StartTenant(
+		ctx,
+		stopper,
+		clusterName,
+		serverCfg.BaseConfig,
+		serverCfg.SQLConfig,
+	)
+	if err != nil {
+		return err
+	}
+	log.Infof(ctx, "SQL server for tenant %s listening at %s", serverCfg.SQLConfig.TenantID, addr)
+
+	// TODO(tbg): make the other goodies in `./cockroach start` reusable, such as
+	// logging to files, periodic memory output, heap and goroutine dumps, debug
+	// server, graceful drain. Then use them here.
+
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, drainSignals...)
+	sig := <-ch
+	return errors.Newf("received signal %v", sig)
+}

pkg/cmd/roachtest/acceptance.go

@@ -42,6 +42,11 @@ func registerAcceptance(r *testRegistry) {
 		{name: "gossip/restart", fn: runGossipRestart},
 		{name: "gossip/restart-node-one", fn: runGossipRestartNodeOne},
 		{name: "gossip/locality-address", fn: runCheckLocalityIPAddress},
+		{
+			name:       "multitenant",
+			minVersion: "v20.2.0", // multitenancy is introduced in this cycle
+			fn:         runAcceptanceMultitenant,
+		},
 		{name: "rapid-restart", fn: runRapidRestart},
 		{
 			name: "many-splits", fn: runManySplits,

pkg/cmd/roachtest/multitenant.go

@@ -0,0 +1,75 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package main
+
+import (
+	"context"
+	gosql "database/sql"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func runAcceptanceMultitenant(ctx context.Context, t *test, c *cluster) {
+	c.Put(ctx, cockroach, "./cockroach")
+	c.Start(ctx, t, c.All())
+
+	_, err := c.Conn(ctx, 1).Exec(`SELECT crdb_internal.create_tenant(123)`)
+	require.NoError(t, err)
+
+	kvAddrs := c.ExternalAddr(ctx, c.All())
+
+	errCh := make(chan error)
+	go func() {
+		errCh <- c.RunE(ctx, c.Node(1),
+			"./cockroach", "mt", "start-sql",
+			// TODO(tbg): make this test secure.
+			// "--certs-dir", "certs",
+			"--insecure",
+			"--tenant-id", "123",
+			"--kv-addrs", strings.Join(kvAddrs, ","),
+			// Don't bind to external interfaces when running locally.
+			"--sql-addr", ifLocal("127.0.0.1", "0.0.0.0")+":36257",
+		)
+	}()
+	u, err := url.Parse(c.ExternalPGUrl(ctx, c.Node(1))[0])
+	require.NoError(t, err)
+	u.Host = c.ExternalIP(ctx, c.Node(1))[0] + ":36257"
+	url := u.String()
+	c.l.Printf("sql server should be running at %s", url)
+
+	time.Sleep(time.Second)
+
+	select {
+	case err := <-errCh:
+		t.Fatal(err)
+	default:
+	}
+
+	db, err := gosql.Open("postgres", url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	_, err = db.Exec(`CREATE TABLE foo (id INT PRIMARY KEY, v STRING)`)
+	require.NoError(t, err)
+
+	_, err = db.Exec(`INSERT INTO foo VALUES($1, $2)`, 1, "bar")
+	require.NoError(t, err)
+
+	var id int
+	var v string
+	require.NoError(t, db.QueryRow(`SELECT * FROM foo LIMIT 1`).Scan(&id, &v))
+	require.Equal(t, 1, id)
+	require.Equal(t, "bar", v)
+}