`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
})) | [
{
"file_path": "/var/log/messages",
"log_group_name": "messages",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/var/log/user-data.log",
"log_group_name": "user_data",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",
"log_group_name": "runner",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
}
]
| no |
+| [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
})) | [
{
"file_path": "/var/log/messages",
"log_group_name": "messages",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/var/log/user-data.log",
"log_group_name": "user_data",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/var/log/runner-startup.log",
"log_group_name": "runner-startup",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",
"log_group_name": "runner",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
}
]
| no |
| [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
| [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
| [runners\_maximum\_count](#input\_runners\_maximum\_count) | The maximum number of runners that will be created. | `number` | `3` | no |
@@ -160,6 +164,7 @@ No modules.
| [sqs\_build\_queue](#input\_sqs\_build\_queue) | SQS queue to consume accepted build events. | object({
arn = string
}) | n/a | yes |
| [subnet\_ids](#input\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes |
| [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
+| [enabled\_userdata](#input\_enabled_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no |
| [userdata\_post\_install](#input\_userdata\_post\_install) | User-data script snippet to insert after GitHub action runner install | `string` | `""` | no |
| [userdata\_pre\_install](#input\_userdata\_pre\_install) | User-data script snippet to insert before GitHub action runner install | `string` | `""` | no |
| [userdata\_template](#input\_userdata\_template) | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no |
diff --git a/modules/runners/logging.tf b/modules/runners/logging.tf
index b66fe29c33..e0b19d59a8 100644
--- a/modules/runners/logging.tf
+++ b/modules/runners/logging.tf
@@ -28,7 +28,7 @@ resource "aws_cloudwatch_log_group" "gh_runners" {
}
resource "aws_iam_role_policy" "cloudwatch" {
- count = var.enable_ssm_on_runners ? 1 : 0
+ count = var.enable_cloudwatch_agent ? 1 : 0
name = "CloudWatchLogginAndMetrics"
role = aws_iam_role.runner.name
policy = templatefile("${path.module}/policies/instance-cloudwatch-policy.json",
diff --git a/modules/runners/main.tf b/modules/runners/main.tf
index de788f1b2f..8a48fece0e 100644
--- a/modules/runners/main.tf
+++ b/modules/runners/main.tf
@@ -3,20 +3,18 @@ locals {
{
"Name" = format("%s-action-runner", var.environment)
},
- {
- "Environment" = format("%s", var.environment)
- },
var.tags,
)
- name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
- name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
- role_path = var.role_path == null ? "/${var.environment}/" : var.role_path
- instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
- lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip
- userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template
- userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl"
- userdata_install_config_runner = "${path.module}/templates/install-config-runner.sh"
+ name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
+ name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
+ role_path = var.role_path == null ? "/${var.environment}/" : var.role_path
+ instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
+ lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip
+ userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template
+ userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl"
+ userdata_install_runner = "${path.module}/templates/install-runner.sh"
+ userdata_start_runner = "${path.module}/templates/start-runner.sh"
instance_types = distinct(var.instance_types == null ? [var.instance_type] : var.instance_types)
@@ -112,32 +110,27 @@ resource "aws_launch_template" "runner" {
}
- user_data = base64encode(templatefile(local.userdata_template, {
+ user_data = var.enabled_userdata ? base64encode(templatefile(local.userdata_template, {
+ pre_install = var.userdata_pre_install
+ install_runner = templatefile(local.userdata_install_runner, {
+ S3_LOCATION_RUNNER_DISTRIBUTION = var.s3_location_runner_binaries
+ ARM_PATCH = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : ""
+ })
+ post_install = var.userdata_post_install
+ start_runner = templatefile(local.userdata_start_runner, {})
+ ghes_url = var.ghes_url
+ ghes_ssl_verify = var.ghes_ssl_verify
+ ## retain these for backwards compatibility
environment = var.environment
- pre_install = var.userdata_pre_install
- post_install = var.userdata_post_install
enable_cloudwatch_agent = var.enable_cloudwatch_agent
ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : ""
- ghes_url = var.ghes_url
- ghes_ssl_verify = var.ghes_ssl_verify
- install_config_runner = local.install_config_runner
- }))
+ })) : ""
tags = local.tags
update_default_version = true
}
-locals {
- arm_patch = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : ""
- install_config_runner = templatefile(local.userdata_install_config_runner, {
- environment = var.environment
- s3_location_runner_distribution = var.s3_location_runner_binaries
- run_as_root_user = var.runner_as_root ? "root" : ""
- arm_patch = local.arm_patch
- })
-}
-
resource "aws_security_group" "runner_sg" {
name_prefix = "${var.environment}-github-actions-runner-sg"
description = "Github Actions Runner security group"
diff --git a/modules/runners/policies-runner.tf b/modules/runners/policies-runner.tf
index 2aa62da8dd..212f9a4b24 100644
--- a/modules/runners/policies-runner.tf
+++ b/modules/runners/policies-runner.tf
@@ -26,7 +26,8 @@ resource "aws_iam_role_policy" "ssm_parameters" {
role = aws_iam_role.runner.name
policy = templatefile("${path.module}/policies/instance-ssm-parameters-policy.json",
{
- arn_ssm_parameters = "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*"
+ arn_ssm_parameters_prefix= "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*"
+ arn_ssm_parameters_path ="arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}/*"
}
)
}
@@ -41,6 +42,12 @@ resource "aws_iam_role_policy" "dist_bucket" {
)
}
+resource "aws_iam_role_policy" "describe_tags" {
+ name = "runner-describe-tags"
+ role = aws_iam_role.runner.name
+ policy = file("${path.module}/policies/instance-describe-tags-policy.json")
+}
+
resource "aws_iam_role_policy_attachment" "managed_policies" {
count = length(var.runner_iam_role_managed_policy_arns)
role = aws_iam_role.runner.name
diff --git a/modules/runners/policies/instance-describe-tags-policy.json b/modules/runners/policies/instance-describe-tags-policy.json
new file mode 100644
index 0000000000..c66074fdaf
--- /dev/null
+++ b/modules/runners/policies/instance-describe-tags-policy.json
@@ -0,0 +1,10 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "ec2:DescribeTags",
+ "Resource": "*"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/modules/runners/policies/instance-ssm-parameters-policy.json b/modules/runners/policies/instance-ssm-parameters-policy.json
index 96e8b02c95..5a7aa9e356 100644
--- a/modules/runners/policies/instance-ssm-parameters-policy.json
+++ b/modules/runners/policies/instance-ssm-parameters-policy.json
@@ -6,15 +6,19 @@
"Action": [
"ssm:DeleteParameter"
],
- "Resource": "${arn_ssm_parameters}"
+ "Resource": "${arn_ssm_parameters_prefix}"
},
{
"Effect": "Allow",
"Action": [
+ "ssm:GetParameter",
"ssm:GetParameters",
- "ssm:GetParameter"
+ "ssm:GetParametersByPath"
],
- "Resource": "${arn_ssm_parameters}"
+ "Resource": [
+ "${arn_ssm_parameters_prefix}",
+ "${arn_ssm_parameters_path}"
+ ]
}
]
}
diff --git a/modules/runners/runner-config.tf b/modules/runners/runner-config.tf
new file mode 100644
index 0000000000..eb6370e58f
--- /dev/null
+++ b/modules/runners/runner-config.tf
@@ -0,0 +1,21 @@
+resource "aws_ssm_parameter" "runner_config_run_as" {
+ name = "/${var.environment}/runner/run-as"
+ type = "String"
+ value = var.runner_as_root ? "root" : "ec2-user"
+ tags = local.tags
+}
+
+resource "aws_ssm_parameter" "runner_agent_mode" {
+ name = "/${var.environment}/runner/agent-mode"
+ type = "String"
+ # TODO: Update this to allow for ephemeral runners
+ value = "persistent"
+ tags = local.tags
+}
+
+resource "aws_ssm_parameter" "runner_enable_cloudwatch" {
+ name = "/${var.environment}/runner/enable-cloudwatch"
+ type = "String"
+ value = var.enable_cloudwatch_agent
+ tags = local.tags
+}
diff --git a/modules/runners/templates/install-config-runner.sh b/modules/runners/templates/install-config-runner.sh
deleted file mode 100644
index a1147303b1..0000000000
--- a/modules/runners/templates/install-config-runner.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-cd /home/$USER_NAME
-mkdir actions-runner && cd actions-runner
-
-TOKEN=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180")
-REGION=$(curl -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
-
-aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz --region $REGION
-tar xzf ./actions-runner.tar.gz
-rm -rf actions-runner.tar.gz
-
-${arm_patch}
-
-INSTANCE_ID=$(curl -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)
-
-echo wait for configuration
-while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do
- echo Waiting for configuration ...
- sleep 1
-done
-CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value")
-aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION
-
-export RUNNER_ALLOW_RUNASROOT=1
-os_id=$(awk -F= '/^ID/{print $2}' /etc/os-release)
-if [[ "$os_id" =~ ^ubuntu.* ]]; then
- ./bin/installdependencies.sh
-fi
-
-./config.sh --unattended --name $INSTANCE_ID --work "_work" $CONFIG
-
-chown -R $USER_NAME:$USER_NAME .
-OVERWRITE_SERVICE_USER=${run_as_root_user}
-SERVICE_USER=$${OVERWRITE_SERVICE_USER:-$USER_NAME}
-
-./svc.sh install $SERVICE_USER
diff --git a/modules/runners/templates/install-runner.sh b/modules/runners/templates/install-runner.sh
new file mode 100644
index 0000000000..94e440e265
--- /dev/null
+++ b/modules/runners/templates/install-runner.sh
@@ -0,0 +1,50 @@
+# shellcheck shell=bash
+
+## install the runner
+
+s3_location=${S3_LOCATION_RUNNER_DISTRIBUTION}
+
+if [ -z "$RUNNER_TARBALL_URL" ] && [ -z "$s3_location" ]; then
+ echo "Neither RUNNER_TARBALL_URL or s3_location are set"
+ exit 1
+fi
+
+file_name="actions-runner.tar.gz"
+
+echo "Creating actions-runner directory for the GH Action installtion"
+cd /home/"$user_name"
+mkdir actions-runner && cd actions-runner
+
+
+if [[ -n "$RUNNER_TARBALL_URL" ]]; then
+ echo "Downloading the GH Action runner from $RUNNER_TARBALL_URL to $file_name"
+ curl -o $file_name -L "$RUNNER_TARBALL_URL"
+else
+ echo "Retrieving TOKEN from AWS API"
+ token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180")
+
+ region=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
+ echo "Reteieved REGION from AWS API ($region)"
+
+ echo "Downloading the GH Action runner from s3 bucket $s3_location"
+ aws s3 cp "$s3_location" "$file_name" --region "$region"
+fi
+
+echo "Un-tar action runner"
+tar xzf ./$file_name
+echo "Delete tar file"
+rm -rf $file_name
+
+${ARM_PATCH}
+
+echo "export RUNNER_ALLOW_RUNASROOT=1"
+export RUNNER_ALLOW_RUNASROOT=1
+
+os_id=$(awk -F= '/^ID/{print $2}' /etc/os-release)
+if [[ "$os_id" =~ ^ubuntu.* ]]; then
+ echo "Installing dependencies"
+ ./bin/installdependencies.sh
+fi
+
+echo "Set file ownership of action runner"
+chown -R "$user_name":"$user_name" .
\ No newline at end of file
diff --git a/modules/runners/templates/start-runner.sh b/modules/runners/templates/start-runner.sh
new file mode 100644
index 0000000000..92e1e34526
--- /dev/null
+++ b/modules/runners/templates/start-runner.sh
@@ -0,0 +1,77 @@
+# shellcheck shell=bash
+
+## Retrieve instance metadata
+
+echo "Retrieving TOKEN from AWS API"
+token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180")
+
+region=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
+echo "Reteieved REGION from AWS API ($region)"
+
+instance_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/instance-id)
+echo "Reteieved INSTANCE_ID from AWS API ($instance_id)"
+
+tags=$(aws ec2 describe-tags --region "$region" --filters "Name=resource-id,Values=$instance_id")
+echo "Retrieved tags from AWS API ($tags)"
+
+environment=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:environment") | .Value')
+echo "Reteieved ghr:environment tag - ($environment)"
+
+parameters=$(aws ssm get-parameters-by-path --path "/$environment/runner" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}")
+echo "Retrieved parameters from AWS SSM ($parameters)"
+
+run_as=$(echo "$parameters" | jq --arg environment "$environment" -r '.[] | select(.Name == "/\($environment)/runner/run-as") | .Value')
+echo "Retrieved /$environment/runner/run-as parameter - ($run_as)"
+
+enable_cloudwatch_agent=$(echo "$parameters" | jq --arg environment "$environment" -r '.[] | select(.Name == "/\($environment)/runner/enable-cloudwatch") | .Value')
+echo "Retrieved /$environment/runner/enable-cloudwatch parameter - ($enable_cloudwatch_agent)"
+
+agent_mode=$(echo "$parameters" | jq --arg environment "$environment" -r '.[] | select(.Name == "/\($environment)/runner/agent-mode") | .Value')
+echo "Retrieved /$environment/runner/agent-mode parameter - ($agent_mode)"
+
+if [[ -n "$enable_cloudwatch_agent" ]]; then
+ echo "Cloudwatch is enabled"
+ amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:$environment-cloudwatch_agent_config_runner"
+fi
+
+
+## Configure the runner
+
+echo "Get GH Runner token from AWS SSM"
+config=$(aws ssm get-parameters --names "$environment"-"$instance_id" --with-decryption --region "$region" | jq -r ".Parameters | .[0] | .Value")
+
+while [[ -z "$config" ]]; do
+ echo "Waiting for GH Runner token to become available in AWS SSM"
+ sleep 1
+ config=$(aws ssm get-parameters --names "$environment"-"$instance_id" --with-decryption --region "$region" | jq -r ".Parameters | .[0] | .Value")
+done
+
+echo "Delete GH Runner token from AWS SSM"
+aws ssm delete-parameter --name "$environment"-"$instance_id" --region "$region"
+
+if [ -z "$run_as" ]; then
+ run_as="ec2-user"
+fi
+
+echo "Configure GH Runner as user $run_as"
+sudo -u "$run_as" -- ./config.sh --unattended --name "$instance_id" --work "_work" $${config}
+
+## Start the runner
+echo "Starting runner after $(awk '{print int($1/3600)":"int(($1%3600)/60)":"int($1%60)}' /proc/uptime)"
+echo "Starting the runner as user $run_as"
+
+if [[ $agent_mode = "ephemeral" ]]; then
+ echo "Starting the runner in ephemeral mode"
+ sudo -u "$run_as" -- ./run.sh
+ echo "Runner has finished"
+
+ echo "Stopping cloudwatch service"
+ service awslogsd stop
+ echo "Terminating instance"
+ aws ec2 terminate-instances --instance-ids "$instance_id" --region "$region"
+else
+ echo "Installing the runner as a service"
+ ./svc.sh install "$run_as"
+ echo "Starting the runner in persistent mode"
+ ./svc.sh start
+fi
\ No newline at end of file
diff --git a/modules/runners/templates/user-data.sh b/modules/runners/templates/user-data.sh
index 0fdf4faa7b..568e48c8c6 100644
--- a/modules/runners/templates/user-data.sh
+++ b/modules/runners/templates/user-data.sh
@@ -5,21 +5,17 @@ ${pre_install}
yum update -y
-%{ if enable_cloudwatch_agent ~}
-yum install amazon-cloudwatch-agent -y
-amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${ssm_key_cloudwatch_agent_config}
-%{ endif ~}
-
# Install docker
amazon-linux-extras install docker
service docker start
usermod -a -G docker ec2-user
-yum install -y curl jq git
+yum install -y amazon-cloudwatch-agent curl jq git
+
+user_name=ec2-user
-USER_NAME=ec2-user
-${install_config_runner}
+${install_runner}
${post_install}
-./svc.sh start
+${start_runner}
diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf
index fd72a55710..c349557fde 100644
--- a/modules/runners/variables.tf
+++ b/modules/runners/variables.tf
@@ -84,6 +84,12 @@ variable "ami_owners" {
default = ["amazon"]
}
+variable "enabled_userdata" {
+ description = "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI"
+ type = bool
+ default = true
+}
+
variable "userdata_template" {
description = "Alternative user-data template, replacing the default template. By providing your own user_data you have to take care of installing all required software, including the action runner. Variables userdata_pre/post_install are ignored."
type = string
@@ -297,7 +303,13 @@ variable "runner_log_files" {
"prefix_log_group" : true,
"file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log",
"log_stream_name" : "{instance_id}"
- }
+ },
+ {
+ "log_group_name" : "runner-startup",
+ "prefix_log_group" : true,
+ "file_path" : "/var/log/runner-startup.log",
+ "log_stream_name" : "{instance_id}"
+ },
]
}
diff --git a/variables.tf b/variables.tf
index 21b8ed3dea..a627a067f2 100644
--- a/variables.tf
+++ b/variables.tf
@@ -165,6 +165,12 @@ variable "kms_key_arn" {
default = null
}
+variable "enabled_userdata" {
+ description = "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI"
+ type = bool
+ default = true
+}
+
variable "userdata_template" {
description = "Alternative user-data template, replacing the default template. By providing your own user_data you have to take care of installing all required software, including the action runner. Variables userdata_pre/post_install are ignored."
type = string
@@ -313,7 +319,13 @@ variable "runner_log_files" {
"prefix_log_group" : true,
"file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log",
"log_stream_name" : "{instance_id}"
- }
+ },
+ {
+ "log_group_name" : "runner-startup",
+ "prefix_log_group" : true,
+ "file_path" : "/var/log/runner-startup.log",
+ "log_stream_name" : "{instance_id}"
+ },
]
}