diff --git a/pkg/cli/rpc_node_shutdown.go b/pkg/cli/rpc_node_shutdown.go index c008b767b058..4cb765bc7132 100644 --- a/pkg/cli/rpc_node_shutdown.go +++ b/pkg/cli/rpc_node_shutdown.go @@ -64,6 +64,39 @@ func doDrain( return doDrainNoTimeout(ctx, c, targetNode) } + shutdownSettings, err := c.Settings(ctx, &serverpb.SettingsRequest{ + Keys: []string{ + "server.shutdown.drain_wait", + "server.shutdown.connection_wait", + "server.shutdown.query_wait", + "server.shutdown.lease_transfer_wait", + }, + UnredactedValues: true, + }) + if err != nil { + return false, true, err + } + + // Add an extra buffer of 10 seconds for the timeout. + minWait := 10 * time.Second + for k, v := range shutdownSettings.KeyValues { + wait, err := time.ParseDuration(v.Value) + if err != nil { + return false, true, err + } + minWait += wait + // query_wait is used twice during draining, so count it twice here. + if k == "server.shutdown.query_wait" { + minWait += wait + } + } + if minWait > drainCtx.drainWait { + log.Infof(ctx, "--drain-wait is %s, but the server.shutdown.{drain,query,connection,lease_transfer}_wait "+ + "cluster settings require a value of at least %s; using the larger value", + drainCtx.drainWait, minWait) + drainCtx.drainWait = minWait + } + err = contextutil.RunWithTimeout(ctx, "drain", drainCtx.drainWait, func(ctx context.Context) (err error) { hardError, remainingWork, err = doDrainNoTimeout(ctx, c, targetNode) return err