Merge pull request #1197 from buildkite/pdp-1478-fix-tags-builds-crea…

…te-a-latest-template-even-when-they-are

.buildkite/pipeline.yml → .buildkite/pipeline.yaml

@@ -178,6 +178,8 @@ steps:
     command: .buildkite/steps/publish.sh
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    env:
+      BUILDKITE_GIT_FETCH_FLAGS: -v --prune --tags
     concurrency_group: "aws-stack-publish"
     concurrency: 1
     concurrency_method: eager

.buildkite/steps/copy.sh

@@ -105,9 +105,9 @@ if [ $# -eq 0 ] ; then
     windows_amd64_source_image_id=$(buildkite-agent meta-data get "windows_amd64_image_id")
 fi
 
-# If we're not on the master branch or a tag build skip the copy
-if [[ $BUILDKITE_BRANCH != "master" ]] && [[ "$BUILDKITE_TAG" != "$BUILDKITE_BRANCH" ]] && [[ ${COPY_TO_ALL_REGIONS:-"false"} != "true" ]]; then
-  echo "--- Skipping AMI copy on non-master/tag branch " >&2
+# If we're not on the main branch or a tag build skip the copy
+if [[ $BUILDKITE_BRANCH != main ]] && [[ $BUILDKITE_TAG != "$BUILDKITE_BRANCH" ]] && [[ ${COPY_TO_ALL_REGIONS:-"false"} != "true" ]]; then
+  echo "--- Skipping AMI copy on non-main/tag branch " >&2
   mkdir -p "$(dirname "$mapping_file")"
   cat << EOF > "$mapping_file"
 Mappings:

.buildkite/steps/publish.sh

@@ -1,14 +1,6 @@
 #!/bin/bash
 set -euo pipefail
 
-is_latest_tag() {
-  [[ "$BUILDKITE_TAG" = $(git describe --abbrev=0 --tags --match 'v*') ]]
-}
-
-is_prerelease_tag() {
-  [[ "$BUILDKITE_TAG" =~ - ]]
-}
-
 s3_upload_templates() {
   local bucket_prefix="${1:-}"
 
@@ -32,15 +24,19 @@ echo "--- Building :cloudformation: CloudFormation templates"
 make build/aws-stack.yml
 
 echo "--- Uploading :cloudformation: CloudFormation templates"
-
-# Publish the top-level mappings only on when we see the most recent tag on master
-if is_latest_tag ; then
-  if ! is_prerelease_tag ; then
-    s3_upload_templates "latest/"
-  fi
+trunk="origin/${BUILDKITE_PIPELINE_DEFAULT_BRANCH}"
+latest_tag="$(git describe --tags --abbrev=0 --match='v*' "$trunk")"
+# Pre-release tags are those that have a hyphen in them, e.g. v1.0.0-rc1
+latest_stable_tag="$(git describe --tags --abbrev=0 --match='v*' --exclude='*-*' "$trunk")"
+
+# Only publish to 'latest' (and the empty prefix) if this tag is the latest stable tag.
+if [[ "${BUILDKITE_TAG}" == "${latest_stable_tag}" ]]; then
+  s3_upload_templates "latest/"
   s3_upload_templates
+elif [[ "${BUILDKITE_TAG}" == "${latest_tag}" ]]; then
+  echo "Skipping publishing latest, although ${BUILDKITE_TAG} matches ${latest_tag} it does not doesn't match ${latest_stable_tag}"
 else
-  echo "Skipping publishing latest, '$BUILDKITE_TAG' doesn't match '$(git describe origin/master --tags --match='v*')'"
+  echo "Skipping publishing latest, $BUILDKITE_TAG doesn't match $latest_tag"
 fi
 
 publish_for_branch() {
@@ -57,6 +53,7 @@ Published template <a href="https://s3.amazonaws.com/${BUILDKITE_AWS_STACK_TEMPL
 EOF
 }
 
+# NOTE: in tag builds, $BUILDKITE_BRANCH == $BUILDKITE_TAG, so this will publish to the tag and exit
 if [[ "$BUILDKITE_BRANCH" != "$BUILDKITE_PIPELINE_DEFAULT_BRANCH" ]]; then
   publish_for_branch "$BUILDKITE_BRANCH"
 

docs/updating-buildkite-agent-scaler.md

@@ -4,7 +4,7 @@ The [buildkite-agent-scaler](https://github.com/buildkite/buildkite-agent-scaler
 is brought in to the Elastic CI Stack for AWS template using the AWS
 Serverless Application Repository.
 
-Once you have [released](https://github.com/buildkite/buildkite-agent-scaler/blob/master/RELEASE.md)
+Once you have [released](https://github.com/buildkite/buildkite-agent-scaler/blob/-/RELEASE.md)
 an updated version, you can incorporate it into the Elastic CI Stack for AWS
 template.
 
@@ -20,7 +20,7 @@ example of updating the buildkite-agent-scaler.
 	1. In the AWS Console, launch CloudFormation
 	1. Create a new stack, select new resources
 	1. Supply the template URL from the build annotation for the Amazon S3 URL field
-	1. Supply a Buildkite Agent token in the `BuildkiteAgentToken` 
+	1. Supply a Buildkite Agent token in the `BuildkiteAgentToken`
 	or `BuildkiteAgentTokenParameterStorePath` parameters
 	1. Supply a queue in the `BuildkiteQueue` parameter
 	1. Create the stack and wait for it to complete

docs/updating-secrets.md

@@ -4,7 +4,7 @@ The elastic-ci-stack-s3secrets-hooks are included in the AMIs by the Packer
 build. The hooks are copied in directly from the submodule, the binaries are
 downloaded from the GitHub release.
 
-Once you have [released](https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/blob/master/RELEASE.md)
+Once you have [released](https://github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/blob/-/RELEASE.md)
 an updated version of the s3secrets-hooks, you can incorporate it into the
 Elastic CI Stack for AWS template.
 

templates/aws-stack.yml

@@ -171,7 +171,7 @@ Parameters:
     Default: "false"
 
   BuildkiteAgentExperiments:
-    Description: Agent experiments to enable, comma delimited. See https://github.com/buildkite/agent/blob/master/EXPERIMENTS.md.
+    Description: Agent experiments to enable, comma delimited. See https://github.com/buildkite/agent/blob/-/EXPERIMENTS.md.
     Type: String
     Default: ""
 
@@ -304,7 +304,7 @@ Parameters:
     Description: How often the event schedule for buildkite-agent-scaler is triggered (in minutes)
     Type: String
     Default: "1"
-  
+
   ScalerMinPollInterval:
     Description: Minimum interval at which the auto scaler should poll the AWS API
     Type: String