buildkite · triarius · Jul 26, 2023 · Mar 20, 2023 · Mar 21, 2023 · Mar 21, 2023
diff --git a/.buildkite/steps/launch.sh b/.buildkite/steps/launch.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eu
+set -euo pipefail
 
 os="${1:-linux}"
 arch="${2:-amd64}"
@@ -22,16 +22,16 @@ echo "Using AMI $image_id for $os/$arch"
 service_role="$(buildkite-agent meta-data get service-role-arn)"
 echo "Using service role ${service_role}"
 
-instance_type="t3.nano"
+instance_type="t3.small"
 instance_disk="10"
 
-if [[ "$os" == "windows" ]] ; then
+if [[ "$os" == "windows" ]]; then
   instance_type="m5.large"
   instance_disk="100"
 fi
 
-if [[ "$arch" == "arm64" ]] ; then
-  instance_type="m6g.large"
+if [[ "$arch" == "arm64" ]]; then
+  instance_type="t4g.small"
 fi
 
 cat << EOF > config.json
@@ -49,7 +49,7 @@ cat << EOF > config.json
     "ParameterValue": "${AWS_KEYPAIR:-aws-stack-test}"
   },
   {
-    "ParameterKey": "InstanceType",
+    "ParameterKey": "InstanceTypes",
     "ParameterValue": "${instance_type}"
   },
   {
@@ -89,7 +89,7 @@ cat << EOF > config.json
     "ParameterValue": "true"
   },
   {
-    "ParameterKey": "EnableAgentGitMirrorsExperiment",
+    "ParameterKey": "BuildkiteAgentEnableGitMirrors",
     "ParameterValue": "true"
   },
   {

diff --git a/.buildkite/steps/publish.sh b/.buildkite/steps/publish.sh
@@ -43,12 +43,28 @@ else
   echo "Skipping publishing latest, '$BUILDKITE_TAG' doesn't match '$(git describe origin/master --tags --match='v*')'"
 fi
 
-# Publish the most recent commit from each branch
-s3_upload_templates "${BUILDKITE_BRANCH}/"
+publish_for_branch() {
+  local branch="$1"
 
-# Publish each build to a unique URL, to let people roll back to old versions
-s3_upload_templates "${BUILDKITE_BRANCH}/${BUILDKITE_COMMIT}."
+  # Publish the most recent commit from each branch
+  s3_upload_templates "${branch}/"
 
-cat << EOF | buildkite-agent annotate --style "info"
-Published template <a href="https://s3.amazonaws.com/${BUILDKITE_AWS_STACK_TEMPLATE_BUCKET}/${BUILDKITE_BRANCH}/aws-stack.yml">${BUILDKITE_BRANCH}/aws-stack.yml</a>
+  # Publish each build to a unique URL, to let people roll back to old versions
+  s3_upload_templates "${branch}/${BUILDKITE_COMMIT}."
+
+  cat << EOF | buildkite-agent annotate --style "info"
+Published template <a href="https://s3.amazonaws.com/${BUILDKITE_AWS_STACK_TEMPLATE_BUCKET}/${branch}/aws-stack.yml">${branch}/aws-stack.yml</a>
 EOF
+}
+
+if [[ "$BUILDKITE_BRANCH" != "$BUILDKITE_PIPELINE_DEFAULT_BRANCH" ]]; then
+  publish_for_branch "$BUILDKITE_BRANCH"
+
+  exit 0
+fi
+
+# TODO: remove "master" from this list
+default_branch_aliases=(master main)
+for branch in "${default_branch_aliases[@]}"; do
+  publish_for_branch "$branch"
+done
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,42 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [v6.0.0](https://github.com/buildkite/elastic-ci-stack-for-aws/tree/v6.0.0) (2023-07-25)
+[Full Changelog](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.22.2...v6.0.0)
+
+### Changed
+- Upgrade base image to Amazon Linux 2023 [#1122](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1122) (@triarius)
+    - Many packages have been added, upgraded, or removed since Amazon Linux 2. We've explicitly called out what's been intentionally left out by us below. Refer to [docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html](https://docs.aws.amazon.com/linux/al2023/ug/compare-with-al2.html) for the changes Amazon have made.
+- Publish template to both `main` and `master` [#1129](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1129) (@triarius)
+- Increase job cancel grace period to 60s [#1144](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1144) (@triarius)
+- Allow the `MaxSize` to be 0 [#1140](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1140) (@triarius)
+- Default EC2 instance names to stack name [#1137](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1137) (@triarius)
+- Rename the parameter `InstanceType` to `InstanceTypes` [#1138](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1138) (@triarius)
+- Rename the parameter `ManagedPolicyARN` to `ManagedPolicyARNs` [#1138](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1138) (@triarius)
+- Rename the parameter `SecurityGroupId` to `SecurityGroupIds` [#1128](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1128) (@triarius)
+- Rename the parameter `EnableAgentGitMirrorsExperiment` to `BuildkiteAgentEnableGitMirrors` [#1123](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1123) (@triarius)
+- Enable the `ansi-timestamps` setting if and only if `BuildkiteAgentTimestampLines` parameter is `"false"` [#1132](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1132) (@triarius)
+- Bump docker compose to v2.20.2 [#1150](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1150) (@triarius)
+- Bump buildx to v0.11.2 [#1150](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1150) (@triarius)
+
+### Added
+- Support running and building multi-platform docker images [#1139](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1139) [#1122](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1122) [#1149](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1149) (@triarius)
+- Support i4g instance types [#1138](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1138) (@triarius)
+- Added the parameter `SpotAllocationStrategy` [#1130](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1130) (@triarius)
+
+### Fixed
+- Guard against `BUILDKITE_AGENT_ENABLE_GIT_MIRRORS` not being set in startup script [#1135](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1135) (@triarius)
+
+### Removed
+- Remove deprecated `SpotPrice` parameter [#1130](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1130) (@triarius)
+- Removed packages. These packages are either not available on Amazon Linux 2023, or not installed by default on the base image we use. We have decided to not install them as suitable replacements may be found.
+  - Python 2
+  - OpenSSL v1.0
+  - AWS CLI v1
+  - Docker-Compose v1
+    - The `docker-compose` executable will prepend the `--compatibility` flag to docker-compose v2 [#1148](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1148) (@triarius)
+  - Cronie
+
 ## [v5.22.2](https://github.com/buildkite/elastic-ci-stack-for-aws/tree/v5.22.2) (2023-07-24)
 [Full Changelog](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.22.1...v5.22.2)
 
@@ -12,6 +48,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Internal
 - Set `allow_dependency_failure: true` on stack cleanup jobs [#1159](https://github.com/buildkite/elastic-ci-stack-for-aws/pull/1159) (@triarius)
+
 ## [v5.22.1](https://github.com/buildkite/elastic-ci-stack-for-aws/tree/v5.22.1) (2023-07-21)
 [Full Changelog](https://github.com/buildkite/elastic-ci-stack-for-aws/compare/v5.22.0...v5.22.1)
 

diff --git a/Makefile b/Makefile
@@ -3,7 +3,7 @@
 VERSION = $(shell git describe --tags --candidates=1)
 SHELL = /bin/bash -o pipefail
 
-PACKER_VERSION ?= 1.6.2
+PACKER_VERSION ?= 1.8.6
 PACKER_LINUX_FILES = $(exec find packer/linux)
 PACKER_WINDOWS_FILES = $(exec find packer/windows)
 

diff --git a/README.md b/README.md
@@ -41,23 +41,22 @@ of per-operating system support:
 
 Feature | Linux | Windows
 --- | --- | ---
-Docker | ✅ | ✅ 
-Docker Compose | ✅ | ✅ 
-AWS CLI | ✅ | ✅ 
-S3 Secrets Bucket | ✅ | ✅ 
-ECR Login | ✅ | ✅ 
-Docker Login | ✅ | ✅ 
-CloudWatch Logs Agent | ✅ | ✅ 
-Per-Instance Bootstrap Script | ✅ | ✅ 
-🧑‍🔬 git-mirrors experiment | ✅ | ✅ 
-SSM Access | ✅ | ✅ 
-Instance Storage (NVMe) | ✅ | 
-SSH Access | ✅ | 
-Periodic authorized_keys Refresh | ✅ | 
-Periodic Instance Health Check | ✅ | 
-git lfs | ✅ | 
-Additional sudo Permissions | ✅ | 
-RDP Access | | ✅ 
+Docker | ✅ | ✅
+Docker Compose | ✅ | ✅
+AWS CLI | ✅ | ✅
+S3 Secrets Bucket | ✅ | ✅
+ECR Login | ✅ | ✅
+Docker Login | ✅ | ✅
+CloudWatch Logs Agent | ✅ | ✅
+Per-Instance Bootstrap Script | ✅ | ✅
+SSM Access | ✅ | ✅
+Instance Storage (NVMe) | ✅ |
+SSH Access | ✅ |
+Periodic authorized_keys Refresh | ✅ |
+Periodic Instance Health Check | ✅ |
+git lfs | ✅ |
+Additional sudo Permissions | ✅ |
+RDP Access | | ✅
 
 ## Security
 
@@ -154,7 +153,7 @@ You may wish to preview any updates to your stack from this template
 [using a CloudFormation Stack Change Set](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-changesets.html)
 to decide whether to apply it.
 
-## Recommended reading  
+## Recommended reading
 
 To gain a better understanding of how Elastic CI Stack works and how to use it most effectively and securely, see the following resources:
 

diff --git a/goss.yaml b/goss.yaml
@@ -40,6 +40,10 @@ service:
     enabled: true
     running: true
 
+  amazon-ssm-agent:
+    enabled: true
+    running: true
+
   docker:
     enabled: true
     running: true
@@ -73,7 +77,7 @@ user:
     gid: 74
     groups:
     - sshd
-    home: /var/empty/sshd
+    home: /usr/share/empty.sshd
     shell: /sbin/nologin
 
 group:
@@ -83,7 +87,7 @@ group:
 
   docker:
     exists: true
-    gid: 1001
+    gid: 993
 
   sshd:
     exists: true
@@ -100,34 +104,85 @@ process:
     running: true
 
 command:
-  "aws --version":
+  aws --version:
+    exit-status: 0
+
+  curl --version:
+    exit-status: 0
+
+  dig -h:
+    exit-status: 0
+
+  git --version:
+    exit-status: 0
+
+  git-lfs --version:
+    exit-status: 0
+
+  command -v lsof:
+    exit-status: 0
+
+  make --version:
     exit-status: 0
 
-  "git --version":
+  wget --version:
     exit-status: 0
 
-  "git-lfs --version":
+  systemctl is-enabled docker-gc.timer:
     exit-status: 0
 
-  "/etc/cron.hourly/docker-low-disk-gc":
+  /usr/local/bin/docker-gc:
     exit-status: 0
 
-  "/etc/cron.hourly/docker-gc":
+  systemctl is-enabled docker-low-disk-gc.timer:
     exit-status: 0
 
-  # Checks that docker is running
-  "docker info":
+  /usr/local/bin/docker-low-disk-gc:
     exit-status: 0
-    timeout: 30000 # it can take some time for the daemon to start
 
-  # Checks that docker containers can run
-  "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker:latest version":
+  docker buildx build -f tests/Dockerfile --progress=plain -t buildkite-postgres:latest tests:
     exit-status: 0
-    timeout: 30000 # it can take some time to download the image
+    timeout: 30000
 
-  # Checks that permissions
-  'sh -c "docker run --rm -v \"$PWD:/pwd\" alpine:latest touch /pwd/test && stat -c %U:%G test"':
+  # Check docker userns is enabled
+  # Note that goss will evaluate the outer layer of templating, and docker will evaluate the second
+  # Running `goss validate --format documentation` will print this with the first layer of templating evaluated
+  '{{ `docker info --format=",{{range .SecurityOptions}}{{.}},{{end}}"` }}':
     exit-status: 0
-    timeout: 30000 # it can take some time to download the image
+    timeout: 30000
+    stdout:
+    - /,name=userns,/
+
+  # Check docker plugins are installed
+  # Note that goss will evaluate the first layer of templating, and docker will evaluate the second
+  # Running `goss validate --format documentation` will print this with the first layer of templating evaluated
+  '{{ `docker info --format=",{{range .ClientInfo.Plugins}}{{.Name}},{{end}}"` }}':
+    exit-status: 0
+    timeout: 30000
+    stdout:
+    - /,buildx,/
+    - /,compose,/
+
+  # Check that docker containers can run
+  docker run --rm -v /var/run/docker.sock:/var/run/docker.sock docker:latest version:
+    exit-status: 0
+    timeout: 30000
+
+  # Check that userns allows writing as buildkite-agent
+  sh -c 'docker run --rm -v "$PWD:/pwd" alpine:latest touch /pwd/test && stat -c %U:%G test' && rm test:
+    exit-status: 0
+    timeout: 30000
     stdout:
     - buildkite-agent:docker
+
+  docker run --rm -t arm64v8/ubuntu uname -m:
+    exit-status: 0
+    timeout: 30000
+    stdout:
+    - aarch64
+
+  docker run --rm -t amd64/ubuntu uname -m:
+    exit-status: 0
+    timeout: 30000
+    stdout:
+    - x86_64
diff --git a/packer/linux/buildkite-ami.json b/packer/linux/buildkite-ami.json
@@ -11,7 +11,7 @@
       "region": "{{user `region`}}",
       "source_ami_filter": {
         "filters": {
-          "name": "amzn2-ami-kernel-5.10-hvm-2.0.*-gp2",
+          "name": "al2023-ami-minimal-2023.0.*.*-kernel-*",
           "architecture": "{{user `arch`}}",
           "virtualization-type": "hvm"
         },
@@ -21,7 +21,7 @@
       "instance_type": "{{user `instance_type`}}",
       "ssh_username": "ec2-user",
       "ami_name": "buildkite-stack-linux-{{user `arch`}}-{{isotime | clean_resource_name}}",
-      "ami_description": "Buildkite Elastic Stack (Amazon Linux 2 LTS w/ docker)",
+      "ami_description": "Buildkite Elastic Stack (Amazon Linux 2023 w/ docker)",
       "ami_groups": ["all"]
     }
   ],
@@ -44,10 +44,6 @@
       "type": "shell",
       "script": "scripts/install-cloudwatch-agent.sh"
     },
-    {
-      "type": "shell",
-      "script": "scripts/install-lifecycled.sh"
-    },
     {
       "type": "shell",
       "script": "scripts/install-docker.sh"
@@ -58,29 +54,11 @@
     },
     {
       "type": "shell",
-      "script": "scripts/install-s3secrets-helper.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/install-git-lfs.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/install-session-manager-plugin.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/install-nvme-cli.sh"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/upgrade-kernel.sh"
+      "script": "scripts/install-buildkite-utils.sh"
     },
     {
       "type": "shell",
-      "inline": [
-        "rm /home/ec2-user/.ssh/authorized_keys"
-      ]
+      "inline": ["rm /home/ec2-user/.ssh/authorized_keys"]
     }
   ]
 }