diff --git a/README.md b/README.md index bc512f5..73f0ad8 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Defines how the cache is stored and restored. Can be any string (see [Customizab Very basic local filesystem backend. -The `BUILDKITE_PLUGIN_FS_CACHE_FOLDER` environment variable defines where the copies are (default: `/var/cache/buildkite`). If you don't change it, you will need to make sure that the folder exists and `buildkite-agent` has the propper permissions, otherwise the plugin will fail. +The `BUILDKITE_PLUGIN_FS_CACHE_FOLDER` environment variable defines where the copies are (default: `/var/cache/buildkite`). If you don't change it, you will need to make sure that the folder exists and `buildkite-agent` has the propper permissions, otherwise the plugin will fail. **IMPORTANT**: the `fs` backend just copies files to a different location in the current agent, as it is not a shared or external resource, its caching possibilities are quite limited. @@ -59,6 +59,7 @@ Store things in an S3 bucket. You need to make sure that the `aws` command is av You also need the agent to have access to the following defined environment variables: * `BUILDKITE_PLUGIN_S3_CACHE_BUCKET`: the bucket to use (backend will fail if not defined) * `BUILDKITE_PLUGIN_S3_CACHE_PREFIX`: optional prefix to use for the cache within the bucket +* `BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT`: optional S3 custom endpoint to use Setting the `BUILDKITE_PLUGIN_S3_CACHE_ONLY_SHOW_ERRORS` environment variable will reduce logging of file operations towards S3. @@ -91,7 +92,7 @@ When restoring from cache, **all levels, in the described order, up to the one s One of the greatest flexibilities of this plugin is its flexible backend architecture. You can provide whatever value you want for the `backend` option of this plugin (`X` for example) as long as there is an executable script accessible to the agent named `cache_X` that respects the following execution protocol: -* `cache_X exists $KEY` +* `cache_X exists $KEY` Should exit successfully (0 return code) if any previous call to this very same plugin was made with `cache_x save $KEY`. Any other exit code will mean that there is no valid cache and will be ignored. @@ -122,7 +123,7 @@ You can always have more complicated logic by using the plugin multiple times wi ```yaml steps: - label: ':nodejs: Install dependencies' - command: npm ci + command: npm ci plugins: - cache#v0.5.0: manifest: package-lock.json diff --git a/backends/cache_s3 b/backends/cache_s3 index 6642af6..3bde451 100755 --- a/backends/cache_s3 +++ b/backends/cache_s3 @@ -18,13 +18,30 @@ s3_sync() { local to="$2" aws_cmd=(aws s3 sync) + if [ -n "${BUILDKITE_PLUGIN_S3_CACHE_ONLY_SHOW_ERRORS}" ]; then aws_cmd+=(--only-show-errors) fi + if [ -n "${BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT}" ]; then + aws_cmd+=(--endpoint-url "${BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT}") + fi + "${aws_cmd[@]}" "${from}" "${to}" } +s3_listobjects() { + local prefix="$1" + + aws_cmd=(aws s3api list-objects-v2 --bucket "${BUILDKITE_PLUGIN_S3_CACHE_BUCKET}" --prefix "$(build_key "${prefix}")" --max-items 1) + + if [ -n "${BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT}" ]; then + aws_cmd+=(--endpoint-url "${BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT}") + fi + + "${aws_cmd[@]}" +} + restore_cache() { local from="$1" local to="$2" @@ -39,7 +56,7 @@ save_cache() { exists_cache() { if [ -z "$1" ]; then exit 1; fi - [ -n "$(aws s3api list-objects-v2 --bucket "${BUILDKITE_PLUGIN_S3_CACHE_BUCKET}" --prefix "$(build_key "$1")" --max-items 1)" ] + [ -n "$(s3_listobjects "$1")" ] } OPCODE="$1" diff --git a/tests/cache_s3.bats b/tests/cache_s3.bats index 5866e7f..67268e6 100644 --- a/tests/cache_s3.bats +++ b/tests/cache_s3.bats @@ -91,6 +91,52 @@ setup() { unstub aws } +@test 'Endpoint URL flag passed when environment is set' { + export BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT=https://s3.somewhere.com + + stub aws \ + 's3 sync --endpoint-url https://s3.somewhere.com \* \* : echo ' \ + 's3 sync --endpoint-url https://s3.somewhere.com \* \* : echo ' \ + 's3api list-objects-v2 --bucket \* --prefix \* --max-items 1 --endpoint-url https://s3.somewhere.com : echo exists' \ + 's3 sync \* \* : echo ' \ + 's3 sync \* \* : echo ' \ + 's3api list-objects-v2 --bucket \* --prefix \* --max-items 1 : echo exists' + + run "${PWD}/backends/cache_s3" save from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_s3" get from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_s3" exists to + + assert_success + assert_output '' + + unset BUILDKITE_PLUGIN_S3_CACHE_ENDPOINT + + run "${PWD}/backends/cache_s3" save from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_s3" get from to + + assert_success + assert_output '' + + run "${PWD}/backends/cache_s3" exists to + + assert_success + assert_output '' + + unstub aws +} + @test 'File exists and can be restored after save' { touch "${BATS_TEST_TMPDIR}/new-file" mkdir "${BATS_TEST_TMPDIR}/s3-cache"