-
Notifications
You must be signed in to change notification settings - Fork 386
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Configurable job timeout #906
Changes from 3 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,8 @@ package main | |
|
||
import ( | ||
"encoding/base64" | ||
"errors" | ||
"fmt" | ||
"log" | ||
"os" | ||
"os/exec" | ||
|
@@ -74,12 +76,6 @@ func (s *Shell) ExecuteImpl(args *dktypes.ExecuteRequest, cb dkplugin.StatusHelp | |
cmd.Stderr = reportingWriter{buffer: output, cb: cb, isError: true} | ||
cmd.Stdout = reportingWriter{buffer: output, cb: cb} | ||
|
||
// Start a timer to warn about slow handlers | ||
slowTimer := time.AfterFunc(2*time.Hour, func() { | ||
log.Printf("shell: Script '%s' slow, execution exceeding %v", command, 2*time.Hour) | ||
}) | ||
defer slowTimer.Stop() | ||
|
||
stdin, err := cmd.StdinPipe() | ||
if err != nil { | ||
return nil, err | ||
|
@@ -96,18 +92,53 @@ func (s *Shell) ExecuteImpl(args *dktypes.ExecuteRequest, cb dkplugin.StatusHelp | |
stdin.Close() | ||
|
||
log.Printf("shell: going to run %s", command) | ||
|
||
jobTimeout := args.Config["timeout"] | ||
var jt time.Duration | ||
|
||
if jobTimeout != "" { | ||
jt, err = time.ParseDuration(jobTimeout) | ||
if err != nil { | ||
return nil, errors.New("shell: Error parsing job timeout") | ||
} | ||
} | ||
|
||
err = cmd.Start() | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
// Warn if buffer is overritten | ||
var jobTimeoutMessage string | ||
var jobWasKilled bool | ||
|
||
slowTimer := time.AfterFunc(jt, func() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I just realized that this will be 0s in case there is no timeout set. There should not be timeout when not specifying a timeout. Could you please open a new PR with the change @andreygolev? And yes! sorry for my fast typing, thanks for the work here @andreygolev of course! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, sure! :) |
||
err = cmd.Process.Kill() | ||
if err != nil { | ||
jobTimeoutMessage = fmt.Sprintf("shell: Job '%s' execution time exceeding defined timeout %v. SIGKILL returned error. Job probably was not killed", command, jt) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. "Job probably was not killed" - the probability of this is debatable. |
||
} else { | ||
jobTimeoutMessage = fmt.Sprintf("shell: Job '%s' execution time exceeding defined timeout %v. Job was killed", command, jt) | ||
} | ||
|
||
jobWasKilled = true | ||
return | ||
}) | ||
|
||
defer slowTimer.Stop() | ||
|
||
// Warn if buffer is ovewritten | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You made a typo correcting the typo ;D There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. lol :) |
||
if output.TotalWritten() > output.Size() { | ||
log.Printf("shell: Script '%s' generated %d bytes of output, truncated to %d", command, output.TotalWritten(), output.Size()) | ||
} | ||
|
||
err = cmd.Wait() | ||
|
||
if jobWasKilled { | ||
_, err := output.Write([]byte(jobTimeoutMessage)) | ||
if err != nil { | ||
log.Printf("Error writing output on timeout event: %v", err) | ||
} | ||
} | ||
|
||
// Always log output | ||
log.Printf("shell: Command output %s", output) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -357,6 +357,13 @@ func (j *Job) Validate() error { | |
return err | ||
} | ||
|
||
if j.Executor == "shell" && j.ExecutorConfig["timeout"] != "" { | ||
_, err := time.ParseDuration(j.ExecutorConfig["timeout"]) | ||
if err != nil { | ||
return fmt.Errorf("Error parsing job timeout value") | ||
} | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Great! Would you mind creating a test in api_test.go for this validation, analoguous to the other validation tests? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sure |
||
return nil | ||
} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
jobWasKilled
value actually signifies something different. I suggest renaming it tojobTimedOut
, so it is in line withjobTimeoutMessage
.