diff --git a/pkg/cli/cliflags/flags_mt.go b/pkg/cli/cliflags/flags_mt.go new file mode 100644 index 000000000000..b3bcc9845d54 --- /dev/null +++ b/pkg/cli/cliflags/flags_mt.go @@ -0,0 +1,26 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cliflags + +// Flags specific to multi-tenancy commands. +var ( + TenantID = FlagInfo{ + Name: "tenant-id", + EnvVar: "COCKROACH_TENANT_ID", + Description: `The tenant ID under which to start the SQL server.`, + } + + KVAddrs = FlagInfo{ + Name: "kv-addrs", + EnvVar: "COCKROACH_KV_ADDRS", + Description: `A comma-separated list of KV endpoints (load balancers allowed).`, + } +) diff --git a/pkg/cli/context.go b/pkg/cli/context.go index c56c2567c47f..26ac53fb43f4 100644 --- a/pkg/cli/context.go +++ b/pkg/cli/context.go @@ -109,18 +109,19 @@ func initCLIDefaults() { debugCtx.maxResults = 0 debugCtx.ballastSize = base.SizeSpec{InBytes: 1000000000} - serverCfg.GoroutineDumpDirName = "" - serverCfg.HeapProfileDirName = "" - serverCfg.ReadyFn = nil - serverCfg.DelayedBootstrapFn = nil - serverCfg.SocketFile = "" - serverCfg.JoinList = nil - serverCfg.JoinPreferSRVRecords = false - serverCfg.DefaultZoneConfig = zonepb.DefaultZoneConfig() - serverCfg.DefaultSystemZoneConfig = zonepb.DefaultSystemZoneConfig() + serverCfg.KVConfig.GoroutineDumpDirName = "" + serverCfg.KVConfig.HeapProfileDirName = "" + serverCfg.KVConfig.ReadyFn = nil + serverCfg.KVConfig.DelayedBootstrapFn = nil + serverCfg.SQLConfig.SocketFile = "" + serverCfg.KVConfig.JoinList = nil + serverCfg.TenantKVAddrs = []string{"127.0.0.1:26257"} + serverCfg.KVConfig.JoinPreferSRVRecords = false + serverCfg.BaseConfig.DefaultZoneConfig = zonepb.DefaultZoneConfig() + serverCfg.KVConfig.DefaultSystemZoneConfig = zonepb.DefaultSystemZoneConfig() // Attempt to default serverCfg.MemoryPoolSize to 25% if possible. if bytes, _ := memoryPercentResolver(25); bytes != 0 { - serverCfg.MemoryPoolSize = bytes + serverCfg.SQLConfig.MemoryPoolSize = bytes } startCtx.serverInsecure = baseCfg.Insecure diff --git a/pkg/cli/flags.go b/pkg/cli/flags.go index dc942d5c8fe9..fb2fde578c37 100644 --- a/pkg/cli/flags.go +++ b/pkg/cli/flags.go @@ -15,11 +15,13 @@ import ( "net" "path/filepath" "regexp" + "strconv" "strings" "time" "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/cli/cliflags" + "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/util/envutil" @@ -246,8 +248,11 @@ func init() { return setDefaultStderrVerbosity(cmd, log.Severity_WARNING) }) - // Add a pre-run command for `start` and `start-single-node`. - for _, cmd := range StartCmds { + // Add a pre-run command for `start` and `start-single-node`, as well as the + // multi-tenancy related commands that start long-running servers. + allStartCmds := append([]*cobra.Command(nil), StartCmds...) + allStartCmds = append(allStartCmds, mtStartSQLCmd) + for _, cmd := range allStartCmds { AddPersistentPreRunE(cmd, func(cmd *cobra.Command, _ []string) error { // Finalize the configuration of network and logging settings. if err := extraServerFlagInit(cmd); err != nil { @@ -704,6 +709,44 @@ func init() { f := debugBallastCmd.Flags() VarFlag(f, &debugCtx.ballastSize, cliflags.Size) } + + // Multi-tenancy commands. + { + f := mtStartSQLCmd.Flags() + VarFlag(f, &tenantIDWrapper{&serverCfg.SQLConfig.TenantID}, cliflags.TenantID) + // NB: serverInsecure populates baseCfg.{Insecure,SSLCertsDir} in this the following method + // (which is a PreRun for this command): + _ = extraServerFlagInit // guru assignment + BoolFlag(f, &startCtx.serverInsecure, cliflags.ServerInsecure, startCtx.serverInsecure) + StringFlag(f, &startCtx.serverSSLCertsDir, cliflags.ServerCertsDir, startCtx.serverSSLCertsDir) + // NB: this also gets PreRun treatment via extraServerFlagInit to populate BaseCfg.SQLAddr. + VarFlag(f, addrSetter{&serverSQLAddr, &serverSQLPort}, cliflags.ListenSQLAddr) + + StringSlice(f, &serverCfg.SQLConfig.TenantKVAddrs, cliflags.KVAddrs, serverCfg.SQLConfig.TenantKVAddrs) + } +} + +type tenantIDWrapper struct { + tenID *roachpb.TenantID +} + +func (w *tenantIDWrapper) String() string { + return w.tenID.String() +} +func (w *tenantIDWrapper) Set(s string) error { + tenID, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return errors.Wrap(err, "invalid tenant ID") + } + if tenID == 0 { + return errors.New("invalid tenant ID") + } + *w.tenID = roachpb.MakeTenantID(tenID) + return nil +} + +func (w *tenantIDWrapper) Type() string { + return "number" } // processEnvVarDefaults injects the current value of flag-related @@ -786,14 +829,24 @@ func extraServerFlagInit(cmd *cobra.Command) error { fs := flagSetForCmd(cmd) - // Construct the socket name, if requested. - if !fs.Lookup(cliflags.Socket.Name).Changed && fs.Lookup(cliflags.SocketDir.Name).Changed { - // If --socket (DEPRECATED) was set, then serverCfg.SocketFile is - // already set and we don't want to change it. - // However, if --socket-dir is set, then we'll use that. - // There are two cases: - // --socket-dir is set and is empty; in this case the user is telling us "disable the socket". - // is set and non-empty. Then it should be used as specified. + // Helper for .Changed that is nil-aware as not all of the `cmd`s may have + // all of the flags. + changed := func(set *pflag.FlagSet, name string) bool { + f := set.Lookup(name) + return f != nil && f.Changed + } + + // Construct the socket name, if requested. The flags may not be defined for + // `cmd` so be cognizant of that. + // + // If --socket (DEPRECATED) was set, then serverCfg.SocketFile is + // already set and we don't want to change it. + // However, if --socket-dir is set, then we'll use that. + // There are two cases: + // 1. --socket-dir is set and is empty; in this case the user is telling us + // "disable the socket". + // 2. is set and non-empty. Then it should be used as specified. + if !changed(fs, cliflags.Socket.Name) && changed(fs, cliflags.SocketDir.Name) { if serverSocketDir == "" { serverCfg.SocketFile = "" } else { @@ -820,9 +873,9 @@ func extraServerFlagInit(cmd *cobra.Command) error { serverCfg.SQLAddr = net.JoinHostPort(serverSQLAddr, serverSQLPort) serverCfg.SplitListenSQL = fs.Lookup(cliflags.ListenSQLAddr.Name).Changed - // Fill in the defaults for --advertise-sql-addr. - advSpecified := fs.Lookup(cliflags.AdvertiseAddr.Name).Changed || - fs.Lookup(cliflags.AdvertiseHost.Name).Changed + // Fill in the defaults for --advertise-sql-addr, if the flag exists on `cmd`. + advSpecified := changed(fs, cliflags.AdvertiseAddr.Name) || + changed(fs, cliflags.AdvertiseHost.Name) if serverSQLAdvertiseAddr == "" { if advSpecified { serverSQLAdvertiseAddr = serverAdvertiseAddr @@ -851,8 +904,8 @@ func extraServerFlagInit(cmd *cobra.Command) error { // Before we do so, we'll check whether the user explicitly // specified something contradictory, and tell them that's no // good. - if (fs.Lookup(cliflags.ListenHTTPAddr.Name).Changed || - fs.Lookup(cliflags.ListenHTTPAddrAlias.Name).Changed) && + if (changed(fs, cliflags.ListenHTTPAddr.Name) || + changed(fs, cliflags.ListenHTTPAddrAlias.Name)) && (serverHTTPAddr != "" && serverHTTPAddr != "localhost") { return errors.WithHintf( errors.Newf("--unencrypted-localhost-http is incompatible with --http-addr=%s:%s", diff --git a/pkg/cli/mt.go b/pkg/cli/mt.go new file mode 100644 index 000000000000..fab067ce5094 --- /dev/null +++ b/pkg/cli/mt.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import "github.com/spf13/cobra" + +func init() { + cockroachCmd.AddCommand(mtCmd) + mtCmd.AddCommand(mtStartSQLCmd) +} + +// mtCmd is the base command for functionality related to multi-tenancy. +var mtCmd = &cobra.Command{ + Use: "mt [command]", + Short: "commands related to multi-tenancy", + Long: ` +Commands related to multi-tenancy. + +This functionality is **experimental** and for internal use only. +`, + RunE: usageAndErr, + Hidden: true, +} diff --git a/pkg/cli/mt_start_sql.go b/pkg/cli/mt_start_sql.go new file mode 100644 index 000000000000..bc4334fd1e39 --- /dev/null +++ b/pkg/cli/mt_start_sql.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "context" + "os" + "os/signal" + + "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/stop" + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" +) + +var mtStartSQLCmd = &cobra.Command{ + Use: "start-sql", + Short: "start a standalone SQL server", + Long: ` +Start a standalone SQL server. + +This functionality is **experimental** and for internal use only. +`, + Args: cobra.NoArgs, + RunE: MaybeDecorateGRPCError(runStartSQL), +} + +func runStartSQL(cmd *cobra.Command, args []string) error { + ctx := context.Background() + const clusterName = "" + stopper := stop.NewStopper() + defer stopper.Stop(ctx) + + st := serverCfg.BaseConfig.Settings + + // TODO(tbg): this has to be passed in. See the upgrade strategy in: + // https://github.com/cockroachdb/cockroach/issues/47919 + if err := clusterversion.Initialize(ctx, st.Version.BinaryVersion(), &st.SV); err != nil { + return err + } + + tempStorageMaxSizeBytes := int64(base.DefaultInMemTempStorageMaxSizeBytes) + if err := diskTempStorageSizeValue.Resolve( + &tempStorageMaxSizeBytes, memoryPercentResolver, + ); err != nil { + return err + } + + serverCfg.SQLConfig.TempStorageConfig = base.TempStorageConfigFromEnv( + ctx, + st, + base.StoreSpec{InMemory: true}, + "", // parentDir + tempStorageMaxSizeBytes, + ) + + addr, err := server.StartTenant( + ctx, + stopper, + clusterName, + serverCfg.BaseConfig, + serverCfg.SQLConfig, + ) + if err != nil { + return err + } + log.Infof(ctx, "SQL server for tenant %s listening at %s", serverCfg.SQLConfig.TenantID, addr) + + // TODO(tbg): make the other goodies in `./cockroach start` reusable, such as + // logging to files, periodic memory output, heap and goroutine dumps, debug + // server, graceful drain. Then use them here. + + ch := make(chan os.Signal, 1) + signal.Notify(ch, drainSignals...) + sig := <-ch + return errors.Newf("received signal %v", sig) +} diff --git a/pkg/cmd/roachtest/acceptance.go b/pkg/cmd/roachtest/acceptance.go index 5f4229383176..4828d5bbaecf 100644 --- a/pkg/cmd/roachtest/acceptance.go +++ b/pkg/cmd/roachtest/acceptance.go @@ -42,6 +42,11 @@ func registerAcceptance(r *testRegistry) { {name: "gossip/restart", fn: runGossipRestart}, {name: "gossip/restart-node-one", fn: runGossipRestartNodeOne}, {name: "gossip/locality-address", fn: runCheckLocalityIPAddress}, + { + name: "multitenant", + minVersion: "v20.2.0", // multitenancy is introduced in this cycle + fn: runAcceptanceMultitenant, + }, {name: "rapid-restart", fn: runRapidRestart}, { name: "many-splits", fn: runManySplits, diff --git a/pkg/cmd/roachtest/multitenant.go b/pkg/cmd/roachtest/multitenant.go new file mode 100644 index 000000000000..ba577c8909b2 --- /dev/null +++ b/pkg/cmd/roachtest/multitenant.go @@ -0,0 +1,75 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "context" + gosql "database/sql" + "net/url" + "strings" + "time" + + "github.com/stretchr/testify/require" +) + +func runAcceptanceMultitenant(ctx context.Context, t *test, c *cluster) { + c.Put(ctx, cockroach, "./cockroach") + c.Start(ctx, t, c.All()) + + _, err := c.Conn(ctx, 1).Exec(`SELECT crdb_internal.create_tenant(123)`) + require.NoError(t, err) + + kvAddrs := c.ExternalAddr(ctx, c.All()) + + errCh := make(chan error) + go func() { + errCh <- c.RunE(ctx, c.Node(1), + "./cockroach", "mt", "start-sql", + // TODO(tbg): make this test secure. + // "--certs-dir", "certs", + "--insecure", + "--tenant-id", "123", + "--kv-addrs", strings.Join(kvAddrs, ","), + // Don't bind to external interfaces when running locally. + "--sql-addr", ifLocal("127.0.0.1", "0.0.0.0")+":36257", + ) + }() + u, err := url.Parse(c.ExternalPGUrl(ctx, c.Node(1))[0]) + require.NoError(t, err) + u.Host = c.ExternalIP(ctx, c.Node(1))[0] + ":36257" + url := u.String() + c.l.Printf("sql server should be running at %s", url) + + time.Sleep(time.Second) + + select { + case err := <-errCh: + t.Fatal(err) + default: + } + + db, err := gosql.Open("postgres", url) + if err != nil { + t.Fatal(err) + } + defer db.Close() + _, err = db.Exec(`CREATE TABLE foo (id INT PRIMARY KEY, v STRING)`) + require.NoError(t, err) + + _, err = db.Exec(`INSERT INTO foo VALUES($1, $2)`, 1, "bar") + require.NoError(t, err) + + var id int + var v string + require.NoError(t, db.QueryRow(`SELECT * FROM foo LIMIT 1`).Scan(&id, &v)) + require.Equal(t, 1, id) + require.Equal(t, "bar", v) +} diff --git a/pkg/server/config.go b/pkg/server/config.go index c2e0b84f4eb8..f04008f962ea 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -327,6 +327,11 @@ type SQLConfig struct { // QueryCacheSize is the memory size (in bytes) of the query plan cache. QueryCacheSize int64 + + // TenantKVAddrs are the entry points to the KV layer. + // + // Only applies when the SQL server is deployed individually. + TenantKVAddrs []string } // MakeSQLConfig returns a SQLConfig with default values. diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index 0d7380a4fa2c..9466e7882a00 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -616,21 +616,21 @@ func (ts *TestServer) StartTenant(params base.TestTenantArgs) (pgAddr string, _ ClusterSettingsUpdater: st.MakeUpdater(), } } - return startTenant( + sqlCfg.TenantKVAddrs = []string{ts.RPCAddr()} + return StartTenant( ctx, ts.Stopper(), ts.Cfg.ClusterName, - ts.RPCAddr(), baseCfg, sqlCfg, ) } -func startTenant( +// StartTenant starts a stand-alone SQL server against a KV backend. +func StartTenant( ctx context.Context, stopper *stop.Stopper, kvClusterName string, // NB: gone after https://github.com/cockroachdb/cockroach/issues/42519 - tsRPCAddr string, baseCfg BaseConfig, sqlCfg SQLConfig, ) (pgAddr string, _ error) { @@ -673,13 +673,17 @@ func startTenant( orphanedLeasesTimeThresholdNanos := args.clock.Now().WallTime { - rsvlr, err := resolver.NewResolver(tsRPCAddr) - if err != nil { - return "", err + rs := make([]resolver.Resolver, len(sqlCfg.TenantKVAddrs)) + for i := range rs { + var err error + rs[i], err = resolver.NewResolver(sqlCfg.TenantKVAddrs[i]) + if err != nil { + return "", err + } } // NB: gossip server is not bound to any address, so the advertise addr does // not matter. - args.gossip.Start(pgL.Addr(), []resolver.Resolver{rsvlr}) + args.gossip.Start(pgL.Addr(), rs) } if err := s.start(ctx,