Merge pull request #47691 from knz/20200420-backport-45149-46396

docs/generated/settings/settings.html

@@ -90,6 +90,7 @@
 <tr><td><code>server.rangelog.ttl</code></td><td>duration</td><td><code>720h0m0s</code></td><td>if nonzero, range log entries older than this duration are deleted every 10m0s. Should not be lowered below 24 hours.</td></tr>
 <tr><td><code>server.remote_debugging.mode</code></td><td>string</td><td><code>local</code></td><td>set to enable remote debugging, localhost-only or disable (any, local, off)</td></tr>
 <tr><td><code>server.shutdown.drain_wait</code></td><td>duration</td><td><code>0s</code></td><td>the amount of time a server waits in an unready state before proceeding with the rest of the shutdown process</td></tr>
+<tr><td><code>server.shutdown.lease_transfer_wait</code></td><td>duration</td><td><code>5s</code></td><td>the amount of time a server waits to transfer range leases before proceeding with the rest of the shutdown process</td></tr>
 <tr><td><code>server.shutdown.query_wait</code></td><td>duration</td><td><code>10s</code></td><td>the server will wait for at least this amount of time for active queries to finish</td></tr>
 <tr><td><code>server.time_until_store_dead</code></td><td>duration</td><td><code>5m0s</code></td><td>the time after which if there is no new gossiped information about a store, it is considered dead</td></tr>
 <tr><td><code>server.web_session_timeout</code></td><td>duration</td><td><code>168h0m0s</code></td><td>the duration that a newly created web session will be valid</td></tr>

pkg/cli/cli_test.go

@@ -1592,12 +1592,12 @@ Available Commands:
   start-single-node start a single-node cluster
   init              initialize a cluster
   cert              create ca, node, and client certs
-  quit              drain and shutdown node
+  quit              drain and shut down a node
 
   sql               open a sql shell
   user              get, set, list and remove users (deprecated)
   auth-session      log in and out of HTTP sessions
-  node              list, inspect or remove nodes
+  node              list, inspect, drain or remove nodes
   dump              dump sql tables
 
   demo              open a demo sql shell

pkg/cli/cliflags/flags.go

@@ -843,6 +843,13 @@ If specified, decommissions the node and waits for it to rebalance before
 shutting down the node.`,
 	}
 
+	DrainWait = FlagInfo{
+		Name: "drain-wait",
+		Description: `
+When non-zero, wait for the specified amount of time for the node to
+drain all active client connections and migrate away range leases.`,
+	}
+
 	Wait = FlagInfo{
 		Name: "wait",
 		Description: `

pkg/cli/context.go

@@ -121,6 +121,7 @@ func initCLIDefaults() {
 	startCtx.inBackground = false
 
 	quitCtx.serverDecommission = false
+	quitCtx.drainWait = 10 * time.Minute
 
 	nodeCtx.nodeDecommissionWait = nodeDecommissionWaitAll
 	nodeCtx.statusShowRanges = false
@@ -310,10 +311,16 @@ var startCtx struct {
 	logDir log.DirName
 }
 
-// quitCtx captures the command-line parameters of the `quit` command.
+// quitCtx captures the command-line parameters of the `quit` and
+// `node drain` commands.
 // Defaults set by InitCLIDefaults() above.
 var quitCtx struct {
+	// serverDecommission indicates the server should be decommissioned
+	// before it is drained.
 	serverDecommission bool
+	// drainWait is the amount of time to wait for the server
+	// to drain. Set to 0 to disable a timeout (let the server decide).
+	drainWait time.Duration
 }
 
 // nodeCtx captures the command-line parameters of the `node` command.

pkg/cli/flags.go

@@ -542,6 +542,13 @@ func init() {
 	// Quit command.
 	BoolFlag(quitCmd.Flags(), &quitCtx.serverDecommission, cliflags.Decommission, quitCtx.serverDecommission)
 
+	// Quit and node drain.
+	for _, cmd := range []*cobra.Command{quitCmd, drainNodeCmd} {
+		f := cmd.Flags()
+		DurationFlag(f, &quitCtx.drainWait, cliflags.DrainWait, quitCtx.drainWait)
+	}
+
+	// SQL and demo commands.
 	for _, cmd := range append([]*cobra.Command{sqlShellCmd, demoCmd}, demoCmd.Commands()...) {
 		f := cmd.Flags()
 		VarFlag(f, &sqlCtx.setStmts, cliflags.Set)

pkg/cli/interactive_tests/test_missing_log_output.tcl

@@ -99,7 +99,7 @@ start_test "Test that quit does not show INFO by default with --logtostderr"
 # that the default logging level is WARNING, so that no INFO messages
 # are printed between the marker and the (first line) error message
 # from quit. Quit will error out because the server is already stopped.
-send "echo marker; $argv quit --logtostderr 2>&1 | grep -vE '^\[WEF\]\[0-9\]+'\r"
+send "echo marker; $argv quit --logtostderr 2>&1 | grep -vE '^\[WEF\]\[0-9\]+|^node is draining'\r"
 eexpect "marker\r\nok"
 eexpect ":/# "
 end_test

pkg/cli/node.go

@@ -434,18 +434,55 @@ func runRecommissionNode(cmd *cobra.Command, args []string) error {
 	return printDecommissionStatus(*resp)
 }
 
+var drainNodeCmd = &cobra.Command{
+	Use:   "drain",
+	Short: "drain a node without shutting it down",
+	Long: `
+Prepare a server for shutting down. This stops accepting client
+connections, stops extant connections, and finally pushes range
+leases onto other nodes, subject to various timeout parameters
+configurable via cluster settings.`,
+	Args: cobra.NoArgs,
+	RunE: MaybeDecorateGRPCError(runDrain),
+}
+
+// runNodeDrain calls the Drain RPC without the flag to stop the
+// server process.
+func runDrain(cmd *cobra.Command, args []string) (err error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// At the end, we'll report "ok" if there was no error.
+	defer func() {
+		if err == nil {
+			fmt.Println("ok")
+		}
+	}()
+
+	// Establish a RPC connection.
+	c, finish, err := getAdminClient(ctx)
+	if err != nil {
+		return err
+	}
+	defer finish()
+
+	_, _, err = doDrain(ctx, c)
+	return err
+}
+
 // Sub-commands for node command.
 var nodeCmds = []*cobra.Command{
 	lsNodesCmd,
 	statusNodeCmd,
 	decommissionNodeCmd,
 	recommissionNodeCmd,
+	drainNodeCmd,
 }
 
 var nodeCmd = &cobra.Command{
 	Use:   "node [command]",
-	Short: "list, inspect or remove nodes",
-	Long:  "List, inspect or remove nodes.",
+	Short: "list, inspect, drain or remove nodes",
+	Long:  "List, inspect, drain or remove nodes.",
 	RunE:  usageAndErr,
 }