Skip to content

Commit

Permalink
cli: compute --drain-wait based on cluster setting values
Browse files Browse the repository at this point in the history
Release note (cli change): The --drain-wait argument for the `drain`
command will be automatically increased if the command detects that it
is smaller than the sum of server.shutdown.drain_wait,
server.shutdown.connection_wait, server.shutdown.query_wait times two,
and server.shutdown.lease_transfer_wait. If the --drain-wait argument is
0, then no timeout is used.

This recommendation was already documented, but now the advice will be
applied automatically.
  • Loading branch information
rafiss committed Mar 14, 2023
1 parent 7098a15 commit e976106
Showing 1 changed file with 33 additions and 0 deletions.
33 changes: 33 additions & 0 deletions pkg/cli/rpc_node_shutdown.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,39 @@ func doDrain(
return doDrainNoTimeout(ctx, c, targetNode)
}

shutdownSettings, err := c.Settings(ctx, &serverpb.SettingsRequest{
Keys: []string{
"server.shutdown.drain_wait",
"server.shutdown.connection_wait",
"server.shutdown.query_wait",
"server.shutdown.lease_transfer_wait",
},
UnredactedValues: true,
})
if err != nil {
return false, true, err
}

// Add an extra buffer of 10 seconds for the timeout.
minWait := 10 * time.Second
for k, v := range shutdownSettings.KeyValues {
wait, err := time.ParseDuration(v.Value)
if err != nil {
return false, true, err
}
minWait += wait
// query_wait is used twice during draining, so count it twice here.
if k == "server.shutdown.query_wait" {
minWait += wait
}
}
if minWait > drainCtx.drainWait {
log.Infof(ctx, "--drain-wait is %s, but the server.shutdown.{drain,query,connection,lease_transfer}_wait "+
"cluster settings require a value of at least %s; using the larger value",
drainCtx.drainWait, minWait)
drainCtx.drainWait = minWait
}

err = contextutil.RunWithTimeout(ctx, "drain", drainCtx.drainWait, func(ctx context.Context) (err error) {
hardError, remainingWork, err = doDrainNoTimeout(ctx, c, targetNode)
return err
Expand Down

0 comments on commit e976106

Please sign in to comment.