-
Notifications
You must be signed in to change notification settings - Fork 271
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
In POSIX shells, `set -e` is not respected in certain contexts. One of these is if it is in a function that is called on the left of a `||`. See https://stackoverflow.com/questions/19789102/why-is-bash-errexit-not-behaving-as-expected-in-function-calls We previously attempted to run this script in a function and exit status 2 if there was an error, but this was not working due to the above. Since it was not working, I took the liberty to change the exit code. Exit code 2 is too overloaded in my opinion. I chose 53. This number looks like "S3", failure to network with which could be the source of some transient failure of this hook.
- Loading branch information
Showing
1 changed file
with
126 additions
and
127 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,144 +1,143 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -euo pipefail | ||
| set -Eeuo pipefail | ||
|
|
||
| enviroment() { | ||
| echo "~~~ :earth_asia: Setting up environment variables" | ||
| # shellcheck source=/dev/null | ||
| source ~/cfn-env | ||
|
|
||
| # a clean docker config for each job, for improved isolation | ||
| BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY=$(mktemp -d) | ||
| export BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY | ||
| export DOCKER_CONFIG="$BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY" | ||
| handle_err() { | ||
| echo "^^^ +++" | ||
| echo ":alert: Elastic CI Stack environment hook failed" >&2 | ||
| exit 53 | ||
| } | ||
|
|
||
| if [ "${BUILDKITE_DOCKER_EXPERIMENTAL:-false}" = "true" ]; then | ||
| if [ ! -f "${DOCKER_CONFIG}/config.json" ]; then | ||
| echo "{}" > "${DOCKER_CONFIG}/config.json" | ||
| fi | ||
| trap handle_err ERR | ||
|
|
||
| #shellcheck disable=SC2094 # Redirections to the same command are processed in order | ||
| cat <<< "$(jq '.experimental="enabled"' "${DOCKER_CONFIG}/config.json")" > "${DOCKER_CONFIG}/config.json" | ||
| fi | ||
| echo "~~~ :earth_asia: Setting up environment variables" | ||
| # shellcheck source=/dev/null | ||
| source ~/cfn-env | ||
|
|
||
| echo "~~~ :llama: Setting up elastic stack environment ($BUILDKITE_STACK_VERSION)" | ||
| # a clean docker config for each job, for improved isolation | ||
| BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY=$(mktemp -d) | ||
| export BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY | ||
| export DOCKER_CONFIG="$BUILDKITE_DOCKER_CONFIG_TEMP_DIRECTORY" | ||
|
|
||
| echo "Checking docker" | ||
| if ! docker ps; then | ||
| echo "^^^ +++" | ||
| echo ":alert: Docker isn't running!" | ||
| set -x | ||
| pgrep -lf docker || tail -n 50 /var/log/docker | ||
| exit 1 | ||
| if [ "${BUILDKITE_DOCKER_EXPERIMENTAL:-false}" = "true" ]; then | ||
| if [ ! -f "${DOCKER_CONFIG}/config.json" ]; then | ||
| echo "{}" > "${DOCKER_CONFIG}/config.json" | ||
| fi | ||
|
|
||
| echo "Checking disk space" | ||
| if ! /usr/local/bin/bk-check-disk-space.sh; then | ||
| #shellcheck disable=SC2094 # Redirections to the same command are processed in order | ||
| cat <<< "$(jq '.experimental="enabled"' "${DOCKER_CONFIG}/config.json")" > "${DOCKER_CONFIG}/config.json" | ||
| fi | ||
|
|
||
| echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL:-4h}" | ||
| docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}" | ||
| echo "~~~ :llama: Setting up elastic stack environment ($BUILDKITE_STACK_VERSION)" | ||
|
|
||
| echo "Checking disk space again" | ||
| if ! /usr/local/bin/bk-check-disk-space.sh; then | ||
| echo "Disk health checks failed" >&2 | ||
| exit 1 | ||
| fi | ||
| fi | ||
|
|
||
| echo "Configuring built-in plugins" | ||
|
|
||
| [[ ! ${SECRETS_PLUGIN_ENABLED:-true} =~ (on|1|true) ]] && PLUGINS_ENABLED=${PLUGINS_ENABLED/secrets/} | ||
| [[ ! ${DOCKER_LOGIN_PLUGIN_ENABLED:-true} =~ (on|1|true) ]] && PLUGINS_ENABLED=${PLUGINS_ENABLED/docker-login/} | ||
| [[ ! ${ECR_PLUGIN_ENABLED:-true} =~ (on|1|true) ]] && PLUGINS_ENABLED=${PLUGINS_ENABLED/ecr/} | ||
|
|
||
| SECRETS_PLUGIN_ENABLED=0 | ||
| DOCKER_LOGIN_PLUGIN_ENABLED=0 | ||
| ECR_PLUGIN_ENABLED=0 | ||
|
|
||
| for plugin in $PLUGINS_ENABLED; do | ||
| case "$plugin" in | ||
| secrets) | ||
| export SECRETS_PLUGIN_ENABLED=1 | ||
| echo "Secrets plugin enabled" | ||
| ;; | ||
| docker-login) | ||
| export DOCKER_LOGIN_PLUGIN_ENABLED=1 | ||
| echo "Docker-login plugin enabled" | ||
| ;; | ||
| ecr) | ||
| export ECR_PLUGIN_ENABLED=1 | ||
| echo "ECR plugin enabled" | ||
| ;; | ||
| esac | ||
| done | ||
|
|
||
| if [[ -n "${BUILDKITE_SECRETS_BUCKET:-}" && "${SECRETS_PLUGIN_ENABLED:-}" == "1" ]]; then | ||
| export BUILDKITE_PLUGIN_S3_SECRETS_BUCKET="$BUILDKITE_SECRETS_BUCKET" | ||
| export BUILDKITE_PLUGIN_S3_SECRETS_REGION="$BUILDKITE_SECRETS_BUCKET_REGION" | ||
|
|
||
| # shellcheck source=/dev/null | ||
| source /usr/local/buildkite-aws-stack/plugins/secrets/hooks/environment | ||
| echo "Checking docker" | ||
| if ! docker ps; then | ||
| echo "^^^ +++" | ||
| echo ":alert: Docker isn't running!" | ||
| set -x | ||
| pgrep -lf docker || tail -n 50 /var/log/docker | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "Checking disk space" | ||
| if ! /usr/local/bin/bk-check-disk-space.sh; then | ||
| echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL:-4h}" | ||
| docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}" | ||
|
|
||
| echo "Checking disk space again" | ||
| if ! /usr/local/bin/bk-check-disk-space.sh; then | ||
| echo "Disk health checks failed" >&2 | ||
| exit 1 | ||
| fi | ||
| fi | ||
|
|
||
| echo "Configuring built-in plugins" | ||
|
|
||
| [[ ! ${SECRETS_PLUGIN_ENABLED:-true} =~ (on|1|true) ]] && PLUGINS_ENABLED=${PLUGINS_ENABLED/secrets/} | ||
| [[ ! ${DOCKER_LOGIN_PLUGIN_ENABLED:-true} =~ (on|1|true) ]] && PLUGINS_ENABLED=${PLUGINS_ENABLED/docker-login/} | ||
| [[ ! ${ECR_PLUGIN_ENABLED:-true} =~ (on|1|true) ]] && PLUGINS_ENABLED=${PLUGINS_ENABLED/ecr/} | ||
|
|
||
| SECRETS_PLUGIN_ENABLED=0 | ||
| DOCKER_LOGIN_PLUGIN_ENABLED=0 | ||
| ECR_PLUGIN_ENABLED=0 | ||
|
|
||
| for plugin in $PLUGINS_ENABLED; do | ||
| case "$plugin" in | ||
| secrets) | ||
| export SECRETS_PLUGIN_ENABLED=1 | ||
| echo "Secrets plugin enabled" | ||
| ;; | ||
| docker-login) | ||
| export DOCKER_LOGIN_PLUGIN_ENABLED=1 | ||
| echo "Docker-login plugin enabled" | ||
| ;; | ||
| ecr) | ||
| export ECR_PLUGIN_ENABLED=1 | ||
| echo "ECR plugin enabled" | ||
| ;; | ||
| esac | ||
| done | ||
|
|
||
| if [[ -n "${BUILDKITE_SECRETS_BUCKET:-}" && "${SECRETS_PLUGIN_ENABLED:-}" == "1" ]]; then | ||
| export BUILDKITE_PLUGIN_S3_SECRETS_BUCKET="$BUILDKITE_SECRETS_BUCKET" | ||
| export BUILDKITE_PLUGIN_S3_SECRETS_REGION="$BUILDKITE_SECRETS_BUCKET_REGION" | ||
|
|
||
| if [[ "${BUILDKITE_ECR_POLICY:-}" != "none" && "${ECR_PLUGIN_ENABLED:-}" == "1" ]]; then | ||
| export BUILDKITE_PLUGIN_ECR_LOGIN=1 | ||
| export BUILDKITE_PLUGIN_ECR_RETRIES=3 | ||
| # shellcheck source=/dev/null | ||
| source /usr/local/buildkite-aws-stack/plugins/secrets/hooks/environment | ||
| fi | ||
|
|
||
| # map AWS_ECR_LOGIN_REGISTRY_IDS into the plugin list format | ||
| if [[ -n "${AWS_ECR_LOGIN_REGISTRY_IDS:-}" ]]; then | ||
| export BUILDKITE_PLUGIN_ECR_ACCOUNT_IDS_0="${AWS_ECR_LOGIN_REGISTRY_IDS}" | ||
| fi | ||
| if [[ "${BUILDKITE_ECR_POLICY:-}" != "none" && "${ECR_PLUGIN_ENABLED:-}" == "1" ]]; then | ||
| export BUILDKITE_PLUGIN_ECR_LOGIN=1 | ||
| export BUILDKITE_PLUGIN_ECR_RETRIES=3 | ||
|
|
||
| # shellcheck source=/dev/null | ||
| source /usr/local/buildkite-aws-stack/plugins/ecr/hooks/environment | ||
| # map AWS_ECR_LOGIN_REGISTRY_IDS into the plugin list format | ||
| if [[ -n "${AWS_ECR_LOGIN_REGISTRY_IDS:-}" ]]; then | ||
| export BUILDKITE_PLUGIN_ECR_ACCOUNT_IDS_0="${AWS_ECR_LOGIN_REGISTRY_IDS}" | ||
| fi | ||
|
|
||
| if [[ "${DOCKER_USERNS_REMAP:-false}" == "false" ]]; then | ||
| # We need to scope the next bit to only the currently running agent dir and | ||
| # pipeline, but we also need to control security and make sure arbitrary folders | ||
| # can't be chmoded. | ||
| # | ||
| # The agent builds path isn't exposed nicely by itself. The agent name also | ||
| # doesn't quite map to its builds path. We do have a complete checkout path, | ||
| # but we need to chop it up, safely. The path looks like: | ||
| # | ||
| # BUILDKITE_BUILD_CHECKOUT_PATH="/var/lib/buildkite-agent/builds/my-agent-1/my-org/my-pipeline" | ||
| # | ||
| # We know the beginning of this path, it's in BUILDKITE_BUILD_PATH: | ||
| # | ||
| # BUILDKITE_BUILD_PATH="/var/lib/buildkite-agent/builds" | ||
|
|
||
| # So we can calculate the suffix as a substring: | ||
| AGENT_ORG_PIPELINE_DIR="${BUILDKITE_BUILD_CHECKOUT_PATH#"${BUILDKITE_BUILD_PATH}/"}" | ||
| # => "my-agent-1/my-org/my-pipeline" | ||
|
|
||
| # Then we can grab just the first path component, the agent name, by removing | ||
| # the longest suffix starting with a slash: | ||
| AGENT_DIR="${AGENT_ORG_PIPELINE_DIR%%/*}" | ||
| # => "my-agent-1" | ||
|
|
||
| # Then we can figure out the org/pipeline path component | ||
| ORG_PIPELINE_DIR="${AGENT_ORG_PIPELINE_DIR#"${AGENT_DIR}/"}" | ||
| # => "my-org/my-pipeline" | ||
|
|
||
| # Then we grab just the first path component, the org, by removing the longest | ||
| # suffix starting with a slash: | ||
| ORG_DIR="${ORG_PIPELINE_DIR%%/*}" | ||
| # => "my-org" | ||
|
|
||
| # Then we can figure out the pipeline path component using the org dir | ||
| PIPELINE_DIR="${ORG_PIPELINE_DIR#"${ORG_DIR}/"}" | ||
| # => "my-pipeline" | ||
|
|
||
| # Now we can pass this to the sudo script which will validate it before safely chmodding: | ||
| echo "~~~ Fixing permissions for '${AGENT_DIR}/${ORG_DIR}/${PIPELINE_DIR}'..." | ||
| sudo /usr/bin/fix-buildkite-agent-builds-permissions "${AGENT_DIR}" "${ORG_DIR}" "${PIPELINE_DIR}" | ||
| echo | ||
| fi | ||
| } | ||
|
|
||
| enviroment || { | ||
| echo "^^^ +++" | ||
| echo ":alert: Running elastic stack environment hook failed" >&2 | ||
| exit 2 | ||
| } | ||
| # shellcheck source=/dev/null | ||
| source /usr/local/buildkite-aws-stack/plugins/ecr/hooks/environment | ||
| fi | ||
|
|
||
| if [[ "${DOCKER_USERNS_REMAP:-false}" == "false" ]]; then | ||
| # We need to scope the next bit to only the currently running agent dir and | ||
| # pipeline, but we also need to control security and make sure arbitrary folders | ||
| # can't be chmoded. | ||
| # | ||
| # The agent builds path isn't exposed nicely by itself. The agent name also | ||
| # doesn't quite map to its builds path. We do have a complete checkout path, | ||
| # but we need to chop it up, safely. The path looks like: | ||
| # | ||
| # BUILDKITE_BUILD_CHECKOUT_PATH="/var/lib/buildkite-agent/builds/my-agent-1/my-org/my-pipeline" | ||
| # | ||
| # We know the beginning of this path, it's in BUILDKITE_BUILD_PATH: | ||
| # | ||
| # BUILDKITE_BUILD_PATH="/var/lib/buildkite-agent/builds" | ||
|
|
||
| # So we can calculate the suffix as a substring: | ||
| AGENT_ORG_PIPELINE_DIR="${BUILDKITE_BUILD_CHECKOUT_PATH#"${BUILDKITE_BUILD_PATH}/"}" | ||
| # => "my-agent-1/my-org/my-pipeline" | ||
|
|
||
| # Then we can grab just the first path component, the agent name, by removing | ||
| # the longest suffix starting with a slash: | ||
| AGENT_DIR="${AGENT_ORG_PIPELINE_DIR%%/*}" | ||
| # => "my-agent-1" | ||
|
|
||
| # Then we can figure out the org/pipeline path component | ||
| ORG_PIPELINE_DIR="${AGENT_ORG_PIPELINE_DIR#"${AGENT_DIR}/"}" | ||
| # => "my-org/my-pipeline" | ||
|
|
||
| # Then we grab just the first path component, the org, by removing the longest | ||
| # suffix starting with a slash: | ||
| ORG_DIR="${ORG_PIPELINE_DIR%%/*}" | ||
| # => "my-org" | ||
|
|
||
| # Then we can figure out the pipeline path component using the org dir | ||
| PIPELINE_DIR="${ORG_PIPELINE_DIR#"${ORG_DIR}/"}" | ||
| # => "my-pipeline" | ||
|
|
||
| # Now we can pass this to the sudo script which will validate it before safely chmodding: | ||
| echo "~~~ Fixing permissions for '${AGENT_DIR}/${ORG_DIR}/${PIPELINE_DIR}'..." | ||
| sudo /usr/bin/fix-buildkite-agent-builds-permissions "${AGENT_DIR}" "${ORG_DIR}" "${PIPELINE_DIR}" | ||
| echo | ||
| fi |