-
Notifications
You must be signed in to change notification settings - Fork 639
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1193 from stanislav-zaprudskiy/add_termination_gr…
…ace_period_seconds AWX: Add `termination_grace_period_seconds`
- Loading branch information
Showing
13 changed files
with
376 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| # file, which when exists, indicates that `master` script has successfully | ||
| # completed pre-stop script execution | ||
| marker_file="${PRE_STOP_MARKER_FILE:-/var/lib/pre-stop/.termination_marker}" | ||
|
|
||
| # file which the running `master` script continuously updates (mtime) to | ||
| # indicate it's still running. this file is then read by `watcher`s to | ||
| # understand if they still have to wait for `termination_marker` | ||
| heartbeat_file="${PRE_STOP_HEARTBEAT_FILE:-/var/lib/pre-stop/.heartbeat}" | ||
|
|
||
| # file which: | ||
| # * `watcher`s create when they bail out because they didn't see the | ||
| # `heartbeat_file` to be updated within `$heartbeat_failed_threshold`; | ||
| # * `master` creates when its handler command fails; | ||
| # when scripts see such file, they also give up | ||
| bailout_file="${PRE_STOP_BAILOUT_FILE:-/var/lib/pre-stop/.bailout}" | ||
| heartbeat_threshold="${PRE_STOP_HEARTBEAT_THRESHOLD:-60}" | ||
|
|
||
| # where the scripts' stdout/stderr are streamed | ||
| stdout="${PRE_STOP_STDOUT:-/proc/1/fd/1}" | ||
| stderr="${PRE_STOP_STDERR:-/proc/1/fd/2}" | ||
|
|
||
| # command the `master` script executes, which when successfully finishes, | ||
| # causes the script to create the `marker_file` | ||
| handler="${PRE_STOP_HANDLER:-bash -c \"PYTHONUNBUFFERED=x awx-manage disable_instance --wait --retry=inf\"}" | ||
|
|
||
| log_prefix="${PRE_STOP_LOG_PREFIX:-preStop.exec}" | ||
| [[ -n ${PRE_STOP_LOG_ROLE} ]] && log_prefix="${log_prefix}] [$PRE_STOP_LOG_ROLE" | ||
|
|
||
| # interval at which `watcher`s check for `marker_file` presence | ||
| recheck_sleep="${PRE_STOP_RECHECK_SLEEP:-1}" | ||
| # interval at which `watcher`s report into $stdout that they are still watching | ||
| report_every="${PRE_STOP_REPORT_EVERY:-30}" | ||
|
|
||
| function log { | ||
| printf "[%s] $1\n" "$log_prefix" "${@:2}" | ||
| } | ||
|
|
||
| function parameters_string { | ||
| for param in "$@"; do | ||
| printf "%s=\"%s\"\n" "$param" "${!param}" | ||
| done | paste -s -d ' ' | ||
| } | ||
|
|
||
| function check_bailout { | ||
| if [[ -f $bailout_file ]]; then | ||
| log "\"%s\" file has been detected, accepting bail out signal and failing the hook script" \ | ||
| "$bailout_file" | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| function check_heartbeat { | ||
| if [[ -f $heartbeat_file ]]; then | ||
| delta=$(( $(date +%s) - $(stat -c %Y "$heartbeat_file") )) | ||
| else | ||
| delta=$(( $(date +%s) - $1 )) | ||
| fi | ||
|
|
||
| if [[ $delta -gt $heartbeat_threshold ]]; then | ||
| log "The heartbeat file hasn't been updated since %ss, which is above the threshold of %ds, assuming the master is not operating and failing the hook script" \ | ||
| $delta | ||
| $heartbeat_threshold | ||
| touch "$bailout_file" | ||
| exit 1 | ||
| fi | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| #/usr/bin/env bash | ||
|
|
||
| PRE_STOP_LOG_ROLE="${PRE_STOP_LOG_ROLE:-master}" | ||
| source $(dirname "$0")/termination-env | ||
|
|
||
| { | ||
|
|
||
| log "The hook has started: %s" \ | ||
| "$(parameters_string \ | ||
| "marker_file" \ | ||
| "heartbeat_file" \ | ||
| "bailout_file" \ | ||
| "handler" \ | ||
| )" | ||
|
|
||
| touch "$heartbeat_file" | ||
|
|
||
| set -o pipefail | ||
| eval "$handler" 2>&1 | while IFS= read -r line; do | ||
| # we check the files here and break early, but overall script termination | ||
| # happens later - as we need to distinguish between files detection and | ||
| # command failure, while bash doesn't offer a simple way to do this here | ||
| # inside the loop (`exit` does not terminate the script) | ||
| [[ -f $bailout_file ]] && break | ||
| [[ -f $marker_file ]] && break | ||
|
|
||
| log "[handler] %s" "$line" | ||
| touch "$heartbeat_file" | ||
| done | ||
| ec=$? | ||
| set +o pipefail | ||
|
|
||
| # process various cases in specific order | ||
| check_bailout | ||
|
|
||
| if [[ -f $marker_file ]]; then | ||
| log "Done! The marker file has been detected, assuming some other instance of the script has run to completion" | ||
| exit 0 | ||
| elif [[ $ec -ne 0 ]]; then | ||
| log "The handler has failed with \"%d\" exit code, failing the hook script too" \ | ||
| $ec | ||
| # signal others to bail out | ||
| touch "$bailout_file" | ||
| exit $ec | ||
| else | ||
| log "Done! Generating the marker file allowing to proceed to termination" | ||
| touch "$marker_file" | ||
| fi | ||
|
|
||
| } > "$stdout" 2> "$stderr" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| #/usr/bin/env bash | ||
|
|
||
| PRE_STOP_LOG_ROLE="${PRE_STOP_LOG_ROLE:-waiter}" | ||
| source $(dirname "$0")/termination-env | ||
|
|
||
| { | ||
|
|
||
| log "The hook has started: %s" \ | ||
| "$(parameters_string \ | ||
| "marker_file" \ | ||
| "heartbeat_file" \ | ||
| "bailout_file" \ | ||
| "recheck_sleep" \ | ||
| "report_every" \ | ||
| )" | ||
|
|
||
| n=0 | ||
| checks_started=$(date +%s) | ||
|
|
||
| while ! [[ -f $marker_file ]]; do | ||
| check_bailout | ||
| check_heartbeat $checks_started | ||
|
|
||
| if [[ $(($n % $report_every)) -eq 0 ]]; then | ||
| log "Waiting for the marker file to be accessible..." | ||
| fi | ||
| n=$(($n + 1)) | ||
| sleep $recheck_sleep | ||
| done | ||
|
|
||
| log "The marker file found, exiting to proceed to termination" | ||
|
|
||
| } > "$stdout" 2> "$stderr" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.