From 70ce9182c02b9d08c3164a5ee5c52689fa0c32ad Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Fri, 12 May 2023 16:27:30 +1000 Subject: [PATCH 1/3] Add `upload_to_s3` command This is a copy of `upload_artifact` minus the logic to use a unique key based on the Buildkite job. --- bin/upload_to_s3 | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 bin/upload_to_s3 diff --git a/bin/upload_to_s3 b/bin/upload_to_s3 new file mode 100644 index 00000000..046beee5 --- /dev/null +++ b/bin/upload_to_s3 @@ -0,0 +1,72 @@ +#!/bin/bash -eu + +while getopts ":f:n:b:-" opt; do + case "${opt}" in + f) + ARTIFACT_PATH="${OPTARG}" + ;; + n) + NAME="${OPTARG}" + ;; + b) + BUCKET="${OPTARG}" + ;; + -) + case "${OPTARG}" in + file) + ARTIFACT_PATH="${OPTARG}" + ;; + name) + NAME="${OPTARG}" + ;; + bucket) + BUCKET="${OPTARG}" + ;; + *) + echo "Invalid option: --${OPTARG}" >&2 + exit 1 + ;; + esac + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$ARTIFACT_PATH" ]; then + echo "You must pass the file you want to be stored with the -f or --file option" + exit 1 +fi + +if [ ! -f "$ARTIFACT_PATH" ]; then + echo "No file found at $ARTIFACT_PATH" + exit 2 +fi + +BUCKET=${ARTIFACTS_S3_BUCKET-} + +if [ -z "$BUCKET" ]; then + echo "You must either pass an S3 bucket via the -b or --bucket option, or set the \`ARTIFACTS_S3_BUCKET\` environment variable" + exit 3 +fi + +if [[ -z "$NAME" ]]; then + echo "You must set a name for the artifact you want to upload via the -n or --name option" + exit 4 +fi + +# If the bucket has transfer acceleration enabled, use it! +ACCELERATION_STATUS=$(aws s3api get-bucket-accelerate-configuration --bucket "$BUCKET" | jq '.Status' -r || true) + +if [ "$ACCELERATION_STATUS" = "Enabled" ]; then + echo "Uploading with transfer acceleration" + aws s3 cp "$ARTIFACT_PATH" "s3://$BUCKET/$NAME" --endpoint-url https://s3-accelerate.amazonaws.com +else + aws s3 cp "$ARTIFACT_PATH" "s3://$BUCKET/$NAME" +fi From 61af53d500df5c92abcd1bae92e16861f76b9db1 Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Fri, 12 May 2023 16:29:41 +1000 Subject: [PATCH 2/3] Use `upload_to_s3` in `upload_artifact` --- bin/upload_artifact | 13 +++---- bin/upload_to_s3 | 83 +++++++++++++++++++++------------------------ 2 files changed, 43 insertions(+), 53 deletions(-) diff --git a/bin/upload_artifact b/bin/upload_artifact index 052289c3..8e5c21e4 100755 --- a/bin/upload_artifact +++ b/bin/upload_artifact @@ -28,12 +28,7 @@ fi BASENAME=$(basename "$ARTIFACT_PATH") KEY="$BUILDKITE_BUILD_ID/$BASENAME" -# If the bucket has transfer acceleration enabled, use it! -ACCELERATION_STATUS=$(aws s3api get-bucket-accelerate-configuration --bucket "$BUCKET" | jq '.Status' -r || true) - -if [ "$ACCELERATION_STATUS" = "Enabled" ]; then - echo "Uploading with transfer acceleration" - aws s3 cp "$ARTIFACT_PATH" "s3://$BUCKET/$KEY" --endpoint-url https://s3-accelerate.amazonaws.com -else - aws s3 cp "$ARTIFACT_PATH" "s3://$BUCKET/$KEY" -fi +upload_to_s3 \ + --file="$ARTIFACT_PATH" \ + --bucket="$BUCKET" \ + --name="$KEY" diff --git a/bin/upload_to_s3 b/bin/upload_to_s3 index 046beee5..d3cd1921 100644 --- a/bin/upload_to_s3 +++ b/bin/upload_to_s3 @@ -1,47 +1,47 @@ #!/bin/bash -eu -while getopts ":f:n:b:-" opt; do - case "${opt}" in - f) - ARTIFACT_PATH="${OPTARG}" - ;; - n) - NAME="${OPTARG}" - ;; - b) - BUCKET="${OPTARG}" - ;; - -) - case "${OPTARG}" in - file) - ARTIFACT_PATH="${OPTARG}" - ;; - name) - NAME="${OPTARG}" - ;; - bucket) - BUCKET="${OPTARG}" - ;; - *) - echo "Invalid option: --${OPTARG}" >&2 - exit 1 - ;; - esac - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 +function usage { + echo "Usage $0 --file=/path/to/file --name=file_name_on_S3 [--bucket=S3_bucket_name]" +} + +# Option parsing logic based upon https://stackoverflow.com/a/28466267/809944 +function exit_with_message() { + echo "$*" >&2; exit 1; +} + +function needs_arg() { + if [ -z "$OPTARG" ]; then + exit_with_message "No argument for --$OPT option. Use format --$OPT=value."; + fi +} + +while getopts "f:n:b:-:" OPT; do + if [ "$OPT" = "-" ]; then # long option: reformulate OPT and OPTARG + OPT="${OPTARG%%=*}" # extract long option name + OPTARG="${OPTARG#"$OPT"}" # extract long option argument (may be empty) + OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` + fi + + case "$OPT" in + f | file) + needs_arg; ARTIFACT_PATH="$OPTARG";; + b | bucket) + needs_arg; BUCKET="$OPTARG";; + n | name) + needs_arg; NAME="$OPTARG";; + ??*) # bad long option + usage + exit_with_message "Illegal option --$OPT" ;; + ?) # bad short option + usage + exit 1;; esac done -if [ -z "$ARTIFACT_PATH" ]; then - echo "You must pass the file you want to be stored with the -f or --file option" - exit 1 +if [[ -z ${ARTIFACT_PATH:-} ]] || [[ -z ${NAME:-} ]]; then + usage + exit 1 fi if [ ! -f "$ARTIFACT_PATH" ]; then @@ -52,15 +52,10 @@ fi BUCKET=${ARTIFACTS_S3_BUCKET-} if [ -z "$BUCKET" ]; then - echo "You must either pass an S3 bucket via the -b or --bucket option, or set the \`ARTIFACTS_S3_BUCKET\` environment variable" + echo "You must either pass an S3 bucket via the -b or --bucket option, or set the \`ARTIFACTS_S3_BUCKET\` environment variable." exit 3 fi -if [[ -z "$NAME" ]]; then - echo "You must set a name for the artifact you want to upload via the -n or --name option" - exit 4 -fi - # If the bucket has transfer acceleration enabled, use it! ACCELERATION_STATUS=$(aws s3api get-bucket-accelerate-configuration --bucket "$BUCKET" | jq '.Status' -r || true) From 9dd76b253c213a4efd520a35eeaaedad7f392479 Mon Sep 17 00:00:00 2001 From: Gio Lodi Date: Sat, 13 May 2023 10:28:07 +1000 Subject: [PATCH 3/3] Make `upload_to_s3` executable `chmod u+x` --- bin/upload_to_s3 | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/upload_to_s3 diff --git a/bin/upload_to_s3 b/bin/upload_to_s3 old mode 100644 new mode 100755