From d060e81e9fa1912976f1bf174ddfd51f58116f7c Mon Sep 17 00:00:00 2001 From: Scott Guymer Date: Fri, 3 Dec 2021 15:56:41 +0100 Subject: [PATCH] feat: Add hooks for prebuilt images (AMI), including amazon linux packer example (#1444) * Initial creation of runner image * Refactored startup script and added it to the per-boot folder * Make the runner location a variable So we can pass the runner version in at packer build time if we want to update the runner version. * Retrieve external config setting via tags Retrieve the required config via the instance tags so we dont have to pass in and set environment on the instance in an awkward way. * Enable tag based config Give the instance the permission to query its own tags and set the correct tags on the instance. * Add a CI job * Fix the CI build * Fix the formatting * Retain user_data provisioning and remove duplication refactored to make sure user_data continues to work with minimal breaking changes. Use a single set of scripts shared between image and user_data provisioning. * Fix interpolation issues in template file * fix build * Fix formatting * minor tweaks and fixes * Fixes from testing * Enable docker on boot * Add in output of start time for the runner * Scoop up the runner log * Add a powershell build script for windows users * Fix formatting * Use SSM parameters for configuration Its best practice to use SSM parameters for configuration of the runners. In adding this i have also added parameter path based config so its easy to extend in the future. * Make the SSM policy more specific * Update .github/workflows/packer-build.yml Co-authored-by: Niek Palm * Added condition to the describe tags policy * Dont use templatefile on the tags policy Because of the use of ${} in the policy terraform is trying to replace it. * Added an option to turn off userdata scripting * Added/updated documentation * Revert policy as it has no effect on the permissions * Add reference to prebuilt images in the main readme * Add an example of deploying with prebuilt images * Update readme * Use current user as ami_owner * Update example to 5 secs * Updated ami name to include the arch * Fixed log file variable * Added explicit info about required settings to the readme * Change userdata_enabled to enabled_userdata Keep within existing naming convention Co-authored-by: Niek Palm --- .ci/build.ps1 | 8 ++ .editorconfig | 2 + .github/workflows/packer-build.yml | 35 ++++++++ .github/workflows/terraform.yml | 2 +- README.md | 15 +++- examples/prebuilt/.terraform.lock.hcl | 60 +++++++++++++ examples/prebuilt/README.md | 83 ++++++++++++++++++ examples/prebuilt/lambdas-download/main.tf | 25 ++++++ examples/prebuilt/main.tf | 46 ++++++++++ examples/prebuilt/outputs.tf | 15 ++++ examples/prebuilt/providers.tf | 3 + examples/prebuilt/variables.tf | 4 + examples/prebuilt/versions.tf | 15 ++++ examples/prebuilt/vpc.tf | 7 ++ images/README.md | 37 ++++++++ images/install-runner.sh | 8 ++ images/linux-amzn2/github_agent.linux.pkr.hcl | 87 +++++++++++++++++++ images/start-runner.sh | 9 ++ main.tf | 4 +- modules/runners/README.md | 7 +- modules/runners/logging.tf | 2 +- modules/runners/main.tf | 49 +++++------ modules/runners/policies-runner.tf | 9 +- .../instance-describe-tags-policy.json | 10 +++ .../instance-ssm-parameters-policy.json | 10 ++- modules/runners/runner-config.tf | 21 +++++ .../templates/install-config-runner.sh | 35 -------- modules/runners/templates/install-runner.sh | 50 +++++++++++ modules/runners/templates/start-runner.sh | 77 ++++++++++++++++ modules/runners/templates/user-data.sh | 14 ++- modules/runners/variables.tf | 14 ++- variables.tf | 14 ++- 32 files changed, 691 insertions(+), 86 deletions(-) create mode 100644 .ci/build.ps1 create mode 100644 .editorconfig create mode 100644 .github/workflows/packer-build.yml create mode 100644 examples/prebuilt/.terraform.lock.hcl create mode 100644 examples/prebuilt/README.md create mode 100644 examples/prebuilt/lambdas-download/main.tf create mode 100644 examples/prebuilt/main.tf create mode 100644 examples/prebuilt/outputs.tf create mode 100644 examples/prebuilt/providers.tf create mode 100644 examples/prebuilt/variables.tf create mode 100644 examples/prebuilt/versions.tf create mode 100644 examples/prebuilt/vpc.tf create mode 100644 images/README.md create mode 100644 images/install-runner.sh create mode 100644 images/linux-amzn2/github_agent.linux.pkr.hcl create mode 100644 images/start-runner.sh create mode 100644 modules/runners/policies/instance-describe-tags-policy.json create mode 100644 modules/runners/runner-config.tf delete mode 100644 modules/runners/templates/install-config-runner.sh create mode 100644 modules/runners/templates/install-runner.sh create mode 100644 modules/runners/templates/start-runner.sh diff --git a/.ci/build.ps1 b/.ci/build.ps1 new file mode 100644 index 0000000000..d92a1b62e6 --- /dev/null +++ b/.ci/build.ps1 @@ -0,0 +1,8 @@ +$TOP_DIR=$(git rev-parse --show-toplevel) +$OUTPUT_DIR="$TOP_DIR/lambda_output" + +New-Item "$OUTPUT_DIR" -ItemType Directory -ErrorAction SilentlyContinue + +$env:DOCKER_BUILDKIT=1 +docker build --no-cache --target=final --output=type=local,dest="$OUTPUT_DIR" -f "$TOP_DIR/.ci/Dockerfile" "$TOP_DIR" + diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..270106b1fa --- /dev/null +++ b/.editorconfig @@ -0,0 +1,2 @@ +[*] +end_of_line = lf diff --git a/.github/workflows/packer-build.yml b/.github/workflows/packer-build.yml new file mode 100644 index 0000000000..96f3de47a3 --- /dev/null +++ b/.github/workflows/packer-build.yml @@ -0,0 +1,35 @@ +name: "Packer checks" +on: + push: + branches: + - master + - develop + pull_request: + paths: + - "images/**" + - ".github/workflows/packer-build.yml" + +env: + AWS_REGION: eu-west-1 + +jobs: + verify_packer: + name: Verify packer + runs-on: ubuntu-latest + container: + image: hashicorp/packer:1.7.8 + defaults: + run: + working-directory: images/linux-amzn2 + steps: + - name: "Checkout" + uses: actions/checkout@v2 + + - name: packer init + run: packer init . + + - name: check terraform formatting + run: packer fmt -recursive -check=true . + + - name: packer validate + run: packer validate . diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 450976e2e1..273280b545 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -43,7 +43,7 @@ jobs: fail-fast: false matrix: terraform: [0.14.3, 0.15.5, 1.0.8] - example: ["default", "ubuntu"] + example: ["default", "ubuntu", "prebuilt"] defaults: run: working-directory: examples/${{ matrix.example }} diff --git a/README.md b/README.md index 58aef2a7cb..02a7f79fb5 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This [Terraform](https://www.terraform.io/) module creates the required infrastr - [Install app](#install-app) - [Encryption](#encryption) - [Idle runners](#idle-runners) + - [Prebuilt Images](#prebuilt-images) - [Examples](#examples) - [Sub modules](#sub-modules) - [ARM64 configuration for submodules](#arm64-configuration-for-submodules) @@ -265,6 +266,10 @@ idle_config = [{ }] ``` +### Prebuilt Images + +This module also allows you to run agents from a prebuilt AMI to gain faster startup times. You can find more information in [the image README.md](/images/README.md) + #### Supported config Cron expressions are parsed by [cron-parser](https://github.com/harrisiirak/cron-parser#readme). The supported syntax. @@ -289,6 +294,7 @@ Examples are located in the [examples](./examples) directory. The following exam - _[Default](examples/default/README.md)_: The default example of the module - _[Permissions boundary](examples/permissions-boundary/README.md)_: Example usages of permissions boundaries. +- _[Prebuilt Images](examples/prebuilt/README.md)_: Example usages of deploying runners with a custom prebuilt image. ## Sub modules @@ -346,10 +352,10 @@ In case the setup does not work as intended follow the trace of events: | Name | Source | Version | |------|--------|---------| -| [runner\_binaries](#module\_runner\_binaries) | ./modules/runner-binaries-syncer | n/a | -| [runners](#module\_runners) | ./modules/runners | n/a | -| [ssm](#module\_ssm) | ./modules/ssm | n/a | -| [webhook](#module\_webhook) | ./modules/webhook | n/a | +| [runner\_binaries](#module\_runner\_binaries) | ./modules/runner-binaries-syncer | | +| [runners](#module\_runners) | ./modules/runners | | +| [ssm](#module\_ssm) | ./modules/ssm | | +| [webhook](#module\_webhook) | ./modules/webhook | | ## Resources @@ -422,6 +428,7 @@ In case the setup does not work as intended follow the trace of events: | [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | +| [enabled_userdata](#input\_enabled_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no | | [userdata\_post\_install](#input\_userdata\_post\_install) | Script to be ran after the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no | | [userdata\_pre\_install](#input\_userdata\_pre\_install) | Script to be ran before the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no | | [userdata\_template](#input\_userdata\_template) | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no | diff --git a/examples/prebuilt/.terraform.lock.hcl b/examples/prebuilt/.terraform.lock.hcl new file mode 100644 index 0000000000..ba82aa4f3a --- /dev/null +++ b/examples/prebuilt/.terraform.lock.hcl @@ -0,0 +1,60 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.61.0" + constraints = ">= 3.27.0, >= 3.38.0" + hashes = [ + "h1:YZaozdn4J1Ax01NkubRAdv809vlpJOXRBC4KsqBzpvA=", + "h1:fpZ14qQnn+uEOO2ZOlBFHgty48Ol8IOwd+ewxZ4z3zc=", + "zh:0483ca802ddb0ae4f73144b4357ba72242c6e2641aeb460b1aa9a6f6965464b0", + "zh:274712214ebeb0c1269cbc468e5705bb5741dc45b05c05e9793ca97f22a1baa1", + "zh:3c6bd97a2ca809469ae38f6893348386c476cb3065b120b785353c1507401adf", + "zh:53dd41a9aed9860adbbeeb71a23e4f8195c656fd15a02c90fa2d302a5f577d8c", + "zh:65c639c547b97bc880fd83e65511c0f4bbfc91b63cada3b8c0d5776444221700", + "zh:a2769e19137ff480c1dd3e4f248e832df90fb6930a22c66264d9793895161714", + "zh:a5897a99332cc0071e46a71359b86a8e53ab09c1453e94cd7cf45a0b577ff590", + "zh:bdc2353642d16d8e2437a9015cd4216a1772be9736645cc17d1a197480e2b5b7", + "zh:cbeace1deae938f6c0aca3734e6088f3633ca09611aff701c15cb6d42f2b918a", + "zh:d33ca19012aabd98cc03fdeccd0bd5ce56e28f61a1dfbb2eea88e89487de7fb3", + "zh:d548b29a864b0687e85e8a993f208e25e3ecc40fcc5b671e1985754b32fdd658", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.1.0" + hashes = [ + "h1:/OpJKWupvFd8WJX1mTt8vi01pP7dkA6e//4l4C3TExE=", + "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", + "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", + "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", + "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", + "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", + "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", + "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", + "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", + "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", + "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", + "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", + "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.1.0" + hashes = [ + "h1:EPIax4Ftp2SNdB9pUfoSjxoueDoLc/Ck3EUoeX0Dvsg=", + "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + ] +} diff --git a/examples/prebuilt/README.md b/examples/prebuilt/README.md new file mode 100644 index 0000000000..3a560fb80d --- /dev/null +++ b/examples/prebuilt/README.md @@ -0,0 +1,83 @@ +# Action runners deployment with prebuilt image + +This module shows how to create GitHub action runners using a prebuilt AMI for the runners + +## Usages + +Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](../../README.md). + +### Lambdas + +You can either download the released lambda code or build them locally yourself. + +First download the Lambda releases from GitHub. Ensure you have set the version in `lambdas-download/main.tf` for running the example. The version needs to be set to a GitHub release version, see https://github.com/philips-labs/terraform-aws-github-runner/releases + +```bash +cd lambdas-download +terraform init +terraform apply +cd .. +``` + +Alternatively you can build the lambdas locally with Node or Docker, there is a simple build script in `/.ci/build.sh`. In the `main.tf` you need to specify the build location for all of the zip files. + +```hcl + webhook_lambda_zip = "../../lambda_output/webhook.zip" + runner_binaries_syncer_lambda_zip = "../../lambda_output/runner-binaries-syncer.zip" + runners_lambda_zip = "../../lambda_output/runners.zip" +``` + +### GitHub App Configuration + +Before running Terraform, ensure the GitHub app is configured. See the [configuration details](../../README.md#usages) for more details. + +### Packer Image + +You will need to build your image. This example deployment uses the image example in `/images/linux-amz2`. You must build this image with packer in your AWS account first. Once you have built this you need to provider your owner ID as a variable + +## Deploy + +To use your image in the terraform modules you will need to set some values on the module. + +Assuming you have built the `linux-amzn2` image which has a pre-defined AMI name in the following format `github-runner-amzn2-x86_64-YYYYMMDDhhmm` you can use the following values. + +```hcl + +module "runners" { + ... + # set the name of the ami to use + ami_filter = { name = ["github-runner-amzn2-x86_64-2021*"] } + # provide the owner id of + ami_owners = [""] + + enabled_userdata = false + ... +} +``` + +If your owner is the same as the account you are logging into then you can use `aws_caller_identity` to retrieve it dynamically. + +```hcl +data "aws_caller_identity" "current" {} + +module "runners" { + ... + ami_owners = [data.aws_caller_identity.current.account_id] + ... +} +``` + +You can then deploy the terraform + +```bash +terraform init +terraform apply +``` + +You can receive the webhook details by running: + +```bash +terraform output -raw webhook_secret +``` + +Be-aware some shells will print some end of line character `%`. diff --git a/examples/prebuilt/lambdas-download/main.tf b/examples/prebuilt/lambdas-download/main.tf new file mode 100644 index 0000000000..87f31bd8a9 --- /dev/null +++ b/examples/prebuilt/lambdas-download/main.tf @@ -0,0 +1,25 @@ +locals { + version = "" +} + +module "lambdas" { + source = "../../../modules/download-lambda" + lambdas = [ + { + name = "webhook" + tag = local.version + }, + { + name = "runners" + tag = local.version + }, + { + name = "runner-binaries-syncer" + tag = local.version + } + ] +} + +output "files" { + value = module.lambdas.files +} diff --git a/examples/prebuilt/main.tf b/examples/prebuilt/main.tf new file mode 100644 index 0000000000..f67c5d984d --- /dev/null +++ b/examples/prebuilt/main.tf @@ -0,0 +1,46 @@ +locals { + environment = "prebuilt" + aws_region = "eu-west-1" +} + +resource "random_password" "random" { + length = 28 +} + +data "aws_caller_identity" "current" {} + +module "runners" { + source = "../../" + create_service_linked_role_spot = true + aws_region = local.aws_region + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + environment = local.environment + + github_app = { + key_base64 = var.github_app_key_base64 + id = var.github_app_id + webhook_secret = random_password.random.result + } + + webhook_lambda_zip = "../../lambda_output/webhook.zip" + runner_binaries_syncer_lambda_zip = "../../lambda_output/runner-binaries-syncer.zip" + runners_lambda_zip = "../../lambda_output/runners.zip" + + runner_extra_labels = "default,example" + + # configure your pre-built AMI + enabled_userdata = false + ami_filter = { name = ["github-runner-amzn2-x86_64-2021*"] } + ami_owners = [data.aws_caller_identity.current.account_id] + + # enable access to the runners via SSM + enable_ssm_on_runners = true + + # override delay of events in seconds + delay_webhook_event = 5 + + # override scaling down + scale_down_schedule_expression = "cron(* * * * ? *)" +} diff --git a/examples/prebuilt/outputs.tf b/examples/prebuilt/outputs.tf new file mode 100644 index 0000000000..d6886efe36 --- /dev/null +++ b/examples/prebuilt/outputs.tf @@ -0,0 +1,15 @@ +output "runners" { + value = { + lambda_syncer_name = module.runners.binaries_syncer.lambda.function_name + } +} + +output "webhook_endpoint" { + value = module.runners.webhook.endpoint +} + +output "webhook_secret" { + sensitive = true + value = random_password.random.result +} + diff --git a/examples/prebuilt/providers.tf b/examples/prebuilt/providers.tf new file mode 100644 index 0000000000..b6c81d5415 --- /dev/null +++ b/examples/prebuilt/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = local.aws_region +} diff --git a/examples/prebuilt/variables.tf b/examples/prebuilt/variables.tf new file mode 100644 index 0000000000..69dcd0c61c --- /dev/null +++ b/examples/prebuilt/variables.tf @@ -0,0 +1,4 @@ + +variable "github_app_key_base64" {} + +variable "github_app_id" {} diff --git a/examples/prebuilt/versions.tf b/examples/prebuilt/versions.tf new file mode 100644 index 0000000000..c96d0eee84 --- /dev/null +++ b/examples/prebuilt/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.27" + } + local = { + source = "hashicorp/local" + } + random = { + source = "hashicorp/random" + } + } + required_version = ">= 0.14" +} diff --git a/examples/prebuilt/vpc.tf b/examples/prebuilt/vpc.tf new file mode 100644 index 0000000000..a7d21422f1 --- /dev/null +++ b/examples/prebuilt/vpc.tf @@ -0,0 +1,7 @@ +module "vpc" { + source = "git::https://github.com/philips-software/terraform-aws-vpc.git?ref=2.2.0" + + environment = local.environment + aws_region = local.aws_region + create_private_hosted_zone = false +} diff --git a/images/README.md b/images/README.md new file mode 100644 index 0000000000..172f97d0a1 --- /dev/null +++ b/images/README.md @@ -0,0 +1,37 @@ +# Prebuilt Images + +The images inside this folder are pre-built images designed to shorten the boot time of your runners and make using ephemeral runners a faster experience. + +These images share the same scripting as used in the user-data mechanism in `/modules/runners/templates/`. We use a `tempaltefile` mechanism to insert the relevant script fragments into the scripts used for provisioning the images. + +The example in `linux-amzn2` also uploads a `start-runner.sh` script that uses the exact same startup process as used in the user-data mechanism. This means that the image created here does not need any extra scripts injected or changes to boot up and connect to GH. + +## Building your own + +To build these images you first need to install packer. +You will also need an amazon account and to have provisioned your credentials for packer to consume. + +Assuming you are building the `linux-amzn2` image. Then run the following from within the `linux-amzn2` folder + +```bash +packer init . +packer validate . +packer build github_agent.linux.pkr.hcl +``` + +Your image will then begin to build inside AWS and when finished you will be provided with complete AMI. + +## Using your image + +To use your image in the terraform modules you will need to set some values on the module. + +Assuming you have built the `linux-amzn2` image which has a pre-defined AMI name in the following format `github-runner-amzn2-x86_64-YYYYMMDDhhmm` you can use the following values. + +```hcl +# set the name of the ami to use +ami_filter = { name = ["github-runner-amzn2-x86_64-2021*"] } +# provide the owner id of +ami_owners = [""] + +enabled_userdata = false +``` diff --git a/images/install-runner.sh b/images/install-runner.sh new file mode 100644 index 0000000000..e042333f00 --- /dev/null +++ b/images/install-runner.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +user_name=ec2-user + +## This wrapper file re-uses scripts in the /modules/runners/templates directory +## of this repo. These are the same that are used by the user_data functionality +## to bootstrap the instance if it is started from an existing AMI. +${install_runner} \ No newline at end of file diff --git a/images/linux-amzn2/github_agent.linux.pkr.hcl b/images/linux-amzn2/github_agent.linux.pkr.hcl new file mode 100644 index 0000000000..566423d03f --- /dev/null +++ b/images/linux-amzn2/github_agent.linux.pkr.hcl @@ -0,0 +1,87 @@ +packer { + required_plugins { + amazon = { + version = ">= 0.0.2" + source = "github.com/hashicorp/amazon" + } + } +} + +variable "action_runner_url" { + description = "The URL to the tarball of the action runner" + type = string + default = "https://github.com/actions/runner/releases/download/v2.284.0/actions-runner-linux-x64-2.284.0.tar.gz" +} + +variable "region" { + description = "The region to build the image in" + type = string + default = "eu-west-1" +} + +source "amazon-ebs" "githubrunner" { + ami_name = "github-runner-amzn2-x86_64-${formatdate("YYYYMMDDhhmm", timestamp())}" + instance_type = "m3.medium" + region = var.region + source_ami_filter { + filters = { + name = "amzn2-ami-hvm-2.*-x86_64-ebs" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["137112412989"] + } + ssh_username = "ec2-user" + tags = { + OS_Version = "amzn2" + Release = "Latest" + Base_AMI_Name = "{{ .SourceAMIName }}" + } +} + +build { + name = "githubactions-runner" + sources = [ + "source.amazon-ebs.githubrunner" + ] + provisioner "shell" { + environment_vars = [] + inline = [ + "sudo yum update -y", + "sudo yum install -y amazon-cloudwatch-agent curl jq git", + "sudo amazon-linux-extras install docker", + "sudo systemctl enable docker.service", + "sudo systemctl enable containerd.service", + "sudo service docker start", + "sudo usermod -a -G docker ec2-user", + ] + } + + provisioner "shell" { + environment_vars = [ + "RUNNER_TARBALL_URL=${var.action_runner_url}" + ] + inline = [templatefile("../install-runner.sh", { + install_runner = templatefile("../../modules/runners/templates/install-runner.sh", { + ARM_PATCH = "" + S3_LOCATION_RUNNER_DISTRIBUTION = "" + }) + })] + } + + provisioner "file" { + content = templatefile("../start-runner.sh", { + start_runner = templatefile("../../modules/runners/templates/start-runner.sh", {}) + }) + destination = "/tmp/start-runner.sh" + } + + provisioner "shell" { + inline = [ + "sudo mv /tmp/start-runner.sh /var/lib/cloud/scripts/per-boot/start-runner.sh", + "sudo chmod +x /var/lib/cloud/scripts/per-boot/start-runner.sh", + ] + } + +} \ No newline at end of file diff --git a/images/start-runner.sh b/images/start-runner.sh new file mode 100644 index 0000000000..7555e44225 --- /dev/null +++ b/images/start-runner.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e +exec > >(tee /var/log/runner-startup.log | logger -t user-data -s 2>/dev/console) 2>&1 + +cd /home/ec2-user/actions-runner + +## This wrapper file re-uses scripts in the /modules/runners/templates directory +## of this repo. These are the same that are used by the user_data functionality +## to bootstrap the instance if it is started from an existing AMI. +${start_runner} \ No newline at end of file diff --git a/main.tf b/main.tf index 4c9b3858ec..6708b4cd06 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,7 @@ locals { tags = merge(var.tags, { - Environment = var.environment + Environment = var.environment, + "ghr:environment" = format("%s", var.environment) }) s3_action_runner_url = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key}" @@ -125,6 +126,7 @@ module "runners" { role_path = var.role_path role_permissions_boundary = var.role_permissions_boundary + enabled_userdata = var.enabled_userdata userdata_template = var.userdata_template userdata_pre_install = var.userdata_pre_install userdata_post_install = var.userdata_post_install diff --git a/modules/runners/README.md b/modules/runners/README.md index fcc9d6b85b..05cb8b2c42 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -79,6 +79,7 @@ No modules. | [aws_iam_role.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy.cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.describe_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.dist_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.runner_session_manager_aws_managed](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | @@ -98,6 +99,9 @@ No modules. | [aws_launch_template.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_security_group.runner_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_ssm_parameter.cloudwatch_agent_config_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_agent_mode](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_config_run_as](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_enable_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | | [aws_ami.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.lambda_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -149,7 +153,7 @@ No modules. | [runner\_extra\_labels](#input\_runner\_extra\_labels) | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no | | [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no | | [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no | -| [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | 
list(object({
    log_group_name   = string
    prefix_log_group = bool
    file_path        = string
    log_stream_name  = string
  }))
| 
[
  {
    "file_path": "/var/log/messages",
    "log_group_name": "messages",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  },
  {
    "file_path": "/var/log/user-data.log",
    "log_group_name": "user_data",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  },
  {
    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",
    "log_group_name": "runner",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  }
]
| no | +| [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | 
list(object({
    log_group_name   = string
    prefix_log_group = bool
    file_path        = string
    log_stream_name  = string
  }))
| 
[
  {
    "file_path": "/var/log/messages",
    "log_group_name": "messages",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  },
  {
    "file_path": "/var/log/user-data.log",
    "log_group_name": "user_data",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  },
  {
    "file_path": "/var/log/runner-startup.log",
    "log_group_name": "runner-startup",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  },
  {
    "file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",
    "log_group_name": "runner",
    "log_stream_name": "{instance_id}",
    "prefix_log_group": true
  }
]
| no | | [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | | [runners\_maximum\_count](#input\_runners\_maximum\_count) | The maximum number of runners that will be created. | `number` | `3` | no | @@ -160,6 +164,7 @@ No modules. | [sqs\_build\_queue](#input\_sqs\_build\_queue) | SQS queue to consume accepted build events. | 
object({
    arn = string
  })
| n/a | yes | | [subnet\_ids](#input\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | +| [enabled\_userdata](#input\_enabled_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no | | [userdata\_post\_install](#input\_userdata\_post\_install) | User-data script snippet to insert after GitHub action runner install | `string` | `""` | no | | [userdata\_pre\_install](#input\_userdata\_pre\_install) | User-data script snippet to insert before GitHub action runner install | `string` | `""` | no | | [userdata\_template](#input\_userdata\_template) | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no | diff --git a/modules/runners/logging.tf b/modules/runners/logging.tf index b66fe29c33..e0b19d59a8 100644 --- a/modules/runners/logging.tf +++ b/modules/runners/logging.tf @@ -28,7 +28,7 @@ resource "aws_cloudwatch_log_group" "gh_runners" { } resource "aws_iam_role_policy" "cloudwatch" { - count = var.enable_ssm_on_runners ? 1 : 0 + count = var.enable_cloudwatch_agent ? 1 : 0 name = "CloudWatchLogginAndMetrics" role = aws_iam_role.runner.name policy = templatefile("${path.module}/policies/instance-cloudwatch-policy.json", diff --git a/modules/runners/main.tf b/modules/runners/main.tf index de788f1b2f..8a48fece0e 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -3,20 +3,18 @@ locals { { "Name" = format("%s-action-runner", var.environment) }, - { - "Environment" = format("%s", var.environment) - }, var.tags, ) - name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] - name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"] - role_path = var.role_path == null ? "/${var.environment}/" : var.role_path - instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path - lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip - userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template - userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" - userdata_install_config_runner = "${path.module}/templates/install-config-runner.sh" + name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] + name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"] + role_path = var.role_path == null ? "/${var.environment}/" : var.role_path + instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path + lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip + userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template + userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" + userdata_install_runner = "${path.module}/templates/install-runner.sh" + userdata_start_runner = "${path.module}/templates/start-runner.sh" instance_types = distinct(var.instance_types == null ? [var.instance_type] : var.instance_types) @@ -112,32 +110,27 @@ resource "aws_launch_template" "runner" { } - user_data = base64encode(templatefile(local.userdata_template, { + user_data = var.enabled_userdata ? base64encode(templatefile(local.userdata_template, { + pre_install = var.userdata_pre_install + install_runner = templatefile(local.userdata_install_runner, { + S3_LOCATION_RUNNER_DISTRIBUTION = var.s3_location_runner_binaries + ARM_PATCH = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : "" + }) + post_install = var.userdata_post_install + start_runner = templatefile(local.userdata_start_runner, {}) + ghes_url = var.ghes_url + ghes_ssl_verify = var.ghes_ssl_verify + ## retain these for backwards compatibility environment = var.environment - pre_install = var.userdata_pre_install - post_install = var.userdata_post_install enable_cloudwatch_agent = var.enable_cloudwatch_agent ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : "" - ghes_url = var.ghes_url - ghes_ssl_verify = var.ghes_ssl_verify - install_config_runner = local.install_config_runner - })) + })) : "" tags = local.tags update_default_version = true } -locals { - arm_patch = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : "" - install_config_runner = templatefile(local.userdata_install_config_runner, { - environment = var.environment - s3_location_runner_distribution = var.s3_location_runner_binaries - run_as_root_user = var.runner_as_root ? "root" : "" - arm_patch = local.arm_patch - }) -} - resource "aws_security_group" "runner_sg" { name_prefix = "${var.environment}-github-actions-runner-sg" description = "Github Actions Runner security group" diff --git a/modules/runners/policies-runner.tf b/modules/runners/policies-runner.tf index 2aa62da8dd..212f9a4b24 100644 --- a/modules/runners/policies-runner.tf +++ b/modules/runners/policies-runner.tf @@ -26,7 +26,8 @@ resource "aws_iam_role_policy" "ssm_parameters" { role = aws_iam_role.runner.name policy = templatefile("${path.module}/policies/instance-ssm-parameters-policy.json", { - arn_ssm_parameters = "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*" + arn_ssm_parameters_prefix= "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*" + arn_ssm_parameters_path ="arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}/*" } ) } @@ -41,6 +42,12 @@ resource "aws_iam_role_policy" "dist_bucket" { ) } +resource "aws_iam_role_policy" "describe_tags" { + name = "runner-describe-tags" + role = aws_iam_role.runner.name + policy = file("${path.module}/policies/instance-describe-tags-policy.json") +} + resource "aws_iam_role_policy_attachment" "managed_policies" { count = length(var.runner_iam_role_managed_policy_arns) role = aws_iam_role.runner.name diff --git a/modules/runners/policies/instance-describe-tags-policy.json b/modules/runners/policies/instance-describe-tags-policy.json new file mode 100644 index 0000000000..c66074fdaf --- /dev/null +++ b/modules/runners/policies/instance-describe-tags-policy.json @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:DescribeTags", + "Resource": "*" + } + ] +} \ No newline at end of file diff --git a/modules/runners/policies/instance-ssm-parameters-policy.json b/modules/runners/policies/instance-ssm-parameters-policy.json index 96e8b02c95..5a7aa9e356 100644 --- a/modules/runners/policies/instance-ssm-parameters-policy.json +++ b/modules/runners/policies/instance-ssm-parameters-policy.json @@ -6,15 +6,19 @@ "Action": [ "ssm:DeleteParameter" ], - "Resource": "${arn_ssm_parameters}" + "Resource": "${arn_ssm_parameters_prefix}" }, { "Effect": "Allow", "Action": [ + "ssm:GetParameter", "ssm:GetParameters", - "ssm:GetParameter" + "ssm:GetParametersByPath" ], - "Resource": "${arn_ssm_parameters}" + "Resource": [ + "${arn_ssm_parameters_prefix}", + "${arn_ssm_parameters_path}" + ] } ] } diff --git a/modules/runners/runner-config.tf b/modules/runners/runner-config.tf new file mode 100644 index 0000000000..eb6370e58f --- /dev/null +++ b/modules/runners/runner-config.tf @@ -0,0 +1,21 @@ +resource "aws_ssm_parameter" "runner_config_run_as" { + name = "/${var.environment}/runner/run-as" + type = "String" + value = var.runner_as_root ? "root" : "ec2-user" + tags = local.tags +} + +resource "aws_ssm_parameter" "runner_agent_mode" { + name = "/${var.environment}/runner/agent-mode" + type = "String" + # TODO: Update this to allow for ephemeral runners + value = "persistent" + tags = local.tags +} + +resource "aws_ssm_parameter" "runner_enable_cloudwatch" { + name = "/${var.environment}/runner/enable-cloudwatch" + type = "String" + value = var.enable_cloudwatch_agent + tags = local.tags +} diff --git a/modules/runners/templates/install-config-runner.sh b/modules/runners/templates/install-config-runner.sh deleted file mode 100644 index a1147303b1..0000000000 --- a/modules/runners/templates/install-config-runner.sh +++ /dev/null @@ -1,35 +0,0 @@ -cd /home/$USER_NAME -mkdir actions-runner && cd actions-runner - -TOKEN=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180") -REGION=$(curl -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) - -aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz --region $REGION -tar xzf ./actions-runner.tar.gz -rm -rf actions-runner.tar.gz - -${arm_patch} - -INSTANCE_ID=$(curl -f -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/instance-id) - -echo wait for configuration -while [[ $(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") == null ]]; do - echo Waiting for configuration ... - sleep 1 -done -CONFIG=$(aws ssm get-parameters --names ${environment}-$INSTANCE_ID --with-decryption --region $REGION | jq -r ".Parameters | .[0] | .Value") -aws ssm delete-parameter --name ${environment}-$INSTANCE_ID --region $REGION - -export RUNNER_ALLOW_RUNASROOT=1 -os_id=$(awk -F= '/^ID/{print $2}' /etc/os-release) -if [[ "$os_id" =~ ^ubuntu.* ]]; then - ./bin/installdependencies.sh -fi - -./config.sh --unattended --name $INSTANCE_ID --work "_work" $CONFIG - -chown -R $USER_NAME:$USER_NAME . -OVERWRITE_SERVICE_USER=${run_as_root_user} -SERVICE_USER=$${OVERWRITE_SERVICE_USER:-$USER_NAME} - -./svc.sh install $SERVICE_USER diff --git a/modules/runners/templates/install-runner.sh b/modules/runners/templates/install-runner.sh new file mode 100644 index 0000000000..94e440e265 --- /dev/null +++ b/modules/runners/templates/install-runner.sh @@ -0,0 +1,50 @@ +# shellcheck shell=bash + +## install the runner + +s3_location=${S3_LOCATION_RUNNER_DISTRIBUTION} + +if [ -z "$RUNNER_TARBALL_URL" ] && [ -z "$s3_location" ]; then + echo "Neither RUNNER_TARBALL_URL or s3_location are set" + exit 1 +fi + +file_name="actions-runner.tar.gz" + +echo "Creating actions-runner directory for the GH Action installtion" +cd /home/"$user_name" +mkdir actions-runner && cd actions-runner + + +if [[ -n "$RUNNER_TARBALL_URL" ]]; then + echo "Downloading the GH Action runner from $RUNNER_TARBALL_URL to $file_name" + curl -o $file_name -L "$RUNNER_TARBALL_URL" +else + echo "Retrieving TOKEN from AWS API" + token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180") + + region=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) + echo "Reteieved REGION from AWS API ($region)" + + echo "Downloading the GH Action runner from s3 bucket $s3_location" + aws s3 cp "$s3_location" "$file_name" --region "$region" +fi + +echo "Un-tar action runner" +tar xzf ./$file_name +echo "Delete tar file" +rm -rf $file_name + +${ARM_PATCH} + +echo "export RUNNER_ALLOW_RUNASROOT=1" +export RUNNER_ALLOW_RUNASROOT=1 + +os_id=$(awk -F= '/^ID/{print $2}' /etc/os-release) +if [[ "$os_id" =~ ^ubuntu.* ]]; then + echo "Installing dependencies" + ./bin/installdependencies.sh +fi + +echo "Set file ownership of action runner" +chown -R "$user_name":"$user_name" . \ No newline at end of file diff --git a/modules/runners/templates/start-runner.sh b/modules/runners/templates/start-runner.sh new file mode 100644 index 0000000000..92e1e34526 --- /dev/null +++ b/modules/runners/templates/start-runner.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash + +## Retrieve instance metadata + +echo "Retrieving TOKEN from AWS API" +token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180") + +region=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) +echo "Reteieved REGION from AWS API ($region)" + +instance_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/instance-id) +echo "Reteieved INSTANCE_ID from AWS API ($instance_id)" + +tags=$(aws ec2 describe-tags --region "$region" --filters "Name=resource-id,Values=$instance_id") +echo "Retrieved tags from AWS API ($tags)" + +environment=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:environment") | .Value') +echo "Reteieved ghr:environment tag - ($environment)" + +parameters=$(aws ssm get-parameters-by-path --path "/$environment/runner" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}") +echo "Retrieved parameters from AWS SSM ($parameters)" + +run_as=$(echo "$parameters" | jq --arg environment "$environment" -r '.[] | select(.Name == "/\($environment)/runner/run-as") | .Value') +echo "Retrieved /$environment/runner/run-as parameter - ($run_as)" + +enable_cloudwatch_agent=$(echo "$parameters" | jq --arg environment "$environment" -r '.[] | select(.Name == "/\($environment)/runner/enable-cloudwatch") | .Value') +echo "Retrieved /$environment/runner/enable-cloudwatch parameter - ($enable_cloudwatch_agent)" + +agent_mode=$(echo "$parameters" | jq --arg environment "$environment" -r '.[] | select(.Name == "/\($environment)/runner/agent-mode") | .Value') +echo "Retrieved /$environment/runner/agent-mode parameter - ($agent_mode)" + +if [[ -n "$enable_cloudwatch_agent" ]]; then + echo "Cloudwatch is enabled" + amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:$environment-cloudwatch_agent_config_runner" +fi + + +## Configure the runner + +echo "Get GH Runner token from AWS SSM" +config=$(aws ssm get-parameters --names "$environment"-"$instance_id" --with-decryption --region "$region" | jq -r ".Parameters | .[0] | .Value") + +while [[ -z "$config" ]]; do + echo "Waiting for GH Runner token to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameters --names "$environment"-"$instance_id" --with-decryption --region "$region" | jq -r ".Parameters | .[0] | .Value") +done + +echo "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$environment"-"$instance_id" --region "$region" + +if [ -z "$run_as" ]; then + run_as="ec2-user" +fi + +echo "Configure GH Runner as user $run_as" +sudo -u "$run_as" -- ./config.sh --unattended --name "$instance_id" --work "_work" $${config} + +## Start the runner +echo "Starting runner after $(awk '{print int($1/3600)":"int(($1%3600)/60)":"int($1%60)}' /proc/uptime)" +echo "Starting the runner as user $run_as" + +if [[ $agent_mode = "ephemeral" ]]; then + echo "Starting the runner in ephemeral mode" + sudo -u "$run_as" -- ./run.sh + echo "Runner has finished" + + echo "Stopping cloudwatch service" + service awslogsd stop + echo "Terminating instance" + aws ec2 terminate-instances --instance-ids "$instance_id" --region "$region" +else + echo "Installing the runner as a service" + ./svc.sh install "$run_as" + echo "Starting the runner in persistent mode" + ./svc.sh start +fi \ No newline at end of file diff --git a/modules/runners/templates/user-data.sh b/modules/runners/templates/user-data.sh index 0fdf4faa7b..568e48c8c6 100644 --- a/modules/runners/templates/user-data.sh +++ b/modules/runners/templates/user-data.sh @@ -5,21 +5,17 @@ ${pre_install} yum update -y -%{ if enable_cloudwatch_agent ~} -yum install amazon-cloudwatch-agent -y -amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${ssm_key_cloudwatch_agent_config} -%{ endif ~} - # Install docker amazon-linux-extras install docker service docker start usermod -a -G docker ec2-user -yum install -y curl jq git +yum install -y amazon-cloudwatch-agent curl jq git + +user_name=ec2-user -USER_NAME=ec2-user -${install_config_runner} +${install_runner} ${post_install} -./svc.sh start +${start_runner} diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index fd72a55710..c349557fde 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -84,6 +84,12 @@ variable "ami_owners" { default = ["amazon"] } +variable "enabled_userdata" { + description = "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI" + type = bool + default = true +} + variable "userdata_template" { description = "Alternative user-data template, replacing the default template. By providing your own user_data you have to take care of installing all required software, including the action runner. Variables userdata_pre/post_install are ignored." type = string @@ -297,7 +303,13 @@ variable "runner_log_files" { "prefix_log_group" : true, "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log", "log_stream_name" : "{instance_id}" - } + }, + { + "log_group_name" : "runner-startup", + "prefix_log_group" : true, + "file_path" : "/var/log/runner-startup.log", + "log_stream_name" : "{instance_id}" + }, ] } diff --git a/variables.tf b/variables.tf index 21b8ed3dea..a627a067f2 100644 --- a/variables.tf +++ b/variables.tf @@ -165,6 +165,12 @@ variable "kms_key_arn" { default = null } +variable "enabled_userdata" { + description = "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI" + type = bool + default = true +} + variable "userdata_template" { description = "Alternative user-data template, replacing the default template. By providing your own user_data you have to take care of installing all required software, including the action runner. Variables userdata_pre/post_install are ignored." type = string @@ -313,7 +319,13 @@ variable "runner_log_files" { "prefix_log_group" : true, "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log", "log_stream_name" : "{instance_id}" - } + }, + { + "log_group_name" : "runner-startup", + "prefix_log_group" : true, + "file_path" : "/var/log/runner-startup.log", + "log_stream_name" : "{instance_id}" + }, ] }