From 74cfa511291f6175f3418cf3595b08ac2894ae04 Mon Sep 17 00:00:00 2001 From: Nathan Dench Date: Wed, 19 Jan 2022 08:27:46 +1000 Subject: [PATCH] feat(runner): Replace patch by install ICU package for ARM runners (#1624) * Update arm-runner-patch.tpl The runtimeconfig.json files seem to have changed on the latest Amazon Linux AMI. When running the user data scripts, the patch of these files fails which causes initialization of the runner instance to fail. * fix patch again * install libicu60 instead of patching * remove arm-patch * Revert "remove arm-patch" This reverts commit 639c46e15b37935264842a360d55a1d1d8ad0051. * Add ARM64 documentation * Remove arm-runner-patch.tpl and include in install-runner.sh * add arm64, ephemeral and windows examples to github workflow --- .github/workflows/terraform.yml | 2 +- README.md | 11 ++- examples/arm64/.terraform.lock.hcl | 57 ++++++++++++++ examples/arm64/README.md | 31 ++++++++ examples/arm64/lambdas-download/main.tf | 25 ++++++ examples/arm64/main.tf | 77 +++++++++++++++++++ examples/arm64/outputs.tf | 15 ++++ examples/arm64/providers.tf | 3 + examples/arm64/variables.tf | 4 + examples/arm64/versions.tf | 15 ++++ examples/arm64/vpc.tf | 7 ++ examples/default/main.tf | 2 +- modules/runners/main.tf | 3 +- .../runners/templates/arm-runner-patch.tpl | 49 ------------ modules/runners/templates/install-runner.sh | 9 ++- 15 files changed, 248 insertions(+), 62 deletions(-) create mode 100644 examples/arm64/.terraform.lock.hcl create mode 100644 examples/arm64/README.md create mode 100644 examples/arm64/lambdas-download/main.tf create mode 100644 examples/arm64/main.tf create mode 100644 examples/arm64/outputs.tf create mode 100644 examples/arm64/providers.tf create mode 100644 examples/arm64/variables.tf create mode 100644 examples/arm64/versions.tf create mode 100644 examples/arm64/vpc.tf delete mode 100644 modules/runners/templates/arm-runner-patch.tpl diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 273280b545..7df799ba61 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -43,7 +43,7 @@ jobs: fail-fast: false matrix: terraform: [0.14.3, 0.15.5, 1.0.8] - example: ["default", "ubuntu", "prebuilt"] + example: ["default", "ubuntu", "prebuilt", "arm64", "ephemeral", "windows"] defaults: run: working-directory: examples/${{ matrix.example }} diff --git a/README.md b/README.md index 521b5c9da5..aa15948a2b 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ To be able to support a number of use-cases the module has quite a lot configura #### ARM64 support via Graviton/Graviton2 instance-types -When using the default example or top-level module, specifying an `instance_type` that matches a Graviton/Graviton 2 (ARM64) architecture (e.g. a1, t4g or any 6th-gen `g` or `gd` type), the sub-modules will be automatically configured to provision with ARM64 AMIs and leverage GitHub's ARM64 action runner. See below for more details. +When using the default example or top-level module, specifying `instance_types` that match a Graviton/Graviton 2 (ARM64) architecture (e.g. a1, t4g or any 6th-gen `g` or `gd` type), you must also specify `runner_architecture = "arm64"` and the sub-modules will be automatically configured to provision with ARM64 AMIs and leverage GitHub's ARM64 action runner. See below for more details. ## Usages @@ -184,8 +184,6 @@ module "github-runner" { } ``` -**ARM64** support: Specify an `a1`, `t4g` or `*6g*` (6th-gen Graviton2) instance type to stand up an ARM64 runner, otherwise the default is x86_64. - Run terraform by using the following commands ```bash @@ -322,9 +320,10 @@ This module also allows you to run agents from a prebuilt AMI to gain faster sta Examples are located in the [examples](./examples) directory. The following examples are provided: - _[Default](examples/default/README.md)_: The default example of the module +- _[ARM64](examples/arm64/README.md)_: Example usage with ARM64 architecture - _[Ubuntu](examples/ubuntu/README.md)_: Example usage of creating a runner using Ubuntu AMIs. - _[Windows](examples/windows/README.md)_: Example usage of creating a runner using Windows as the OS. -- _[Ephemeral](examples/ephemeral/README.md) : Example usages of ephemeral runners based on the default example. +- _[Ephemeral](examples/ephemeral/README.md)_: Example usages of ephemeral runners based on the default example. - _[Prebuilt Images](examples/prebuilt/README.md)_: Example usages of deploying runners with a custom prebuilt image. - _[Permissions boundary](examples/permissions-boundary/README.md)_: Example usages of permissions boundaries. @@ -345,7 +344,7 @@ The following sub modules are optional and are provided as example or utility: ### ARM64 configuration for submodules -When using the top level module configure `runner_architecture = arm64` and insure the list of `instance_types` matches. When not using the top-level ensure the bot properties are set on the submodules. +When using the top level module configure `runner_architecture = "arm64"` and ensure the list of `instance_types` matches. When not using the top-level, ensure these properties are set on the submodules. ## Debugging @@ -354,7 +353,7 @@ In case the setup does not work as intended follow the trace of events: - In the GitHub App configuration, the Advanced page displays all webhook events that were sent. - In AWS CloudWatch, every lambda has a log group. Look at the logs of the `webhook` and `scale-up` lambdas. - In AWS SQS you can see messages available or in flight. -- Once an EC2 instance is running, you can connect to it in the EC2 user interface using Session Manager. Check the user data script using `cat /var/log/user-data.log`. By default several log files of the instances are streamed to AWS CloudWatch, look for a log group named `/runners`. In the log group you should see at least the log streams for the user data installation and runner agent. +- Once an EC2 instance is running, you can connect to it in the EC2 user interface using Session Manager (use `enable_ssm_on_runners = true`). Check the user data script using `cat /var/log/user-data.log`. By default several log files of the instances are streamed to AWS CloudWatch, look for a log group named `/runners`. In the log group you should see at least the log streams for the user data installation and runner agent. - Registered instances should show up in the Settings - Actions page of the repository or organization (depending on the installation mode). diff --git a/examples/arm64/.terraform.lock.hcl b/examples/arm64/.terraform.lock.hcl new file mode 100644 index 0000000000..d940521fcb --- /dev/null +++ b/examples/arm64/.terraform.lock.hcl @@ -0,0 +1,57 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.61.0" + constraints = ">= 3.27.0" + hashes = [ + "h1:fpZ14qQnn+uEOO2ZOlBFHgty48Ol8IOwd+ewxZ4z3zc=", + "zh:0483ca802ddb0ae4f73144b4357ba72242c6e2641aeb460b1aa9a6f6965464b0", + "zh:274712214ebeb0c1269cbc468e5705bb5741dc45b05c05e9793ca97f22a1baa1", + "zh:3c6bd97a2ca809469ae38f6893348386c476cb3065b120b785353c1507401adf", + "zh:53dd41a9aed9860adbbeeb71a23e4f8195c656fd15a02c90fa2d302a5f577d8c", + "zh:65c639c547b97bc880fd83e65511c0f4bbfc91b63cada3b8c0d5776444221700", + "zh:a2769e19137ff480c1dd3e4f248e832df90fb6930a22c66264d9793895161714", + "zh:a5897a99332cc0071e46a71359b86a8e53ab09c1453e94cd7cf45a0b577ff590", + "zh:bdc2353642d16d8e2437a9015cd4216a1772be9736645cc17d1a197480e2b5b7", + "zh:cbeace1deae938f6c0aca3734e6088f3633ca09611aff701c15cb6d42f2b918a", + "zh:d33ca19012aabd98cc03fdeccd0bd5ce56e28f61a1dfbb2eea88e89487de7fb3", + "zh:d548b29a864b0687e85e8a993f208e25e3ecc40fcc5b671e1985754b32fdd658", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.1.0" + hashes = [ + "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", + "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", + "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", + "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", + "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", + "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", + "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", + "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", + "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", + "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", + "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", + "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.1.0" + hashes = [ + "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + ] +} diff --git a/examples/arm64/README.md b/examples/arm64/README.md new file mode 100644 index 0000000000..75c0089ba4 --- /dev/null +++ b/examples/arm64/README.md @@ -0,0 +1,31 @@ +# Action runners deployment with ARM64 architecture + +This module shows how to create GitHub action runners using AWS Graviton instances which have ARM64 architecture. Lambda release will be downloaded from GitHub. + +## Usages + +Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](../../README.md). First download the Lambda releases from GitHub. Alternatively you can build the lambdas locally with Node or Docker, there is a simple build script in `/.ci/build.sh`. In the `main.tf` you can simply remove the location of the lambda zip files, the default location will work in this case. + +> Ensure you have set the version in `lambdas-download/main.tf` for running the example. The version needs to be set to a GitHub release version, see https://github.com/philips-labs/terraform-aws-github-runner/releases + +```bash +cd lambdas-download +terraform init +terraform apply +cd .. +``` + +Before running Terraform, ensure the GitHub app is configured. See the [configuration details](../../README.md#usages) for more details. + +```bash +terraform init +terraform apply +``` + +You can receive the webhook details by running: + +```bash +terraform output -raw webhook_secret +``` + +Be-aware some shells will print some end of line character `%`. diff --git a/examples/arm64/lambdas-download/main.tf b/examples/arm64/lambdas-download/main.tf new file mode 100644 index 0000000000..87f31bd8a9 --- /dev/null +++ b/examples/arm64/lambdas-download/main.tf @@ -0,0 +1,25 @@ +locals { + version = "" +} + +module "lambdas" { + source = "../../../modules/download-lambda" + lambdas = [ + { + name = "webhook" + tag = local.version + }, + { + name = "runners" + tag = local.version + }, + { + name = "runner-binaries-syncer" + tag = local.version + } + ] +} + +output "files" { + value = module.lambdas.files +} diff --git a/examples/arm64/main.tf b/examples/arm64/main.tf new file mode 100644 index 0000000000..8a728ebe3d --- /dev/null +++ b/examples/arm64/main.tf @@ -0,0 +1,77 @@ +locals { + environment = "default" + aws_region = "eu-west-1" +} + +resource "random_id" "random" { + byte_length = 20 +} + + +################################################################################ +### Hybrid account +################################################################################ + +module "runners" { + source = "../../" + create_service_linked_role_spot = true + aws_region = local.aws_region + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + environment = local.environment + tags = { + Project = "ProjectX" + } + + github_app = { + key_base64 = var.github_app_key_base64 + id = var.github_app_id + webhook_secret = random_id.random.hex + } + + # Grab zip files via lambda_download, will automatically get the ARM64 build + webhook_lambda_zip = "lambdas-download/webhook.zip" + runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip" + runners_lambda_zip = "lambdas-download/runners.zip" + + enable_organization_runners = false + # Runners will automatically get the "arm64" label + runner_extra_labels = "default,example" + + # enable access to the runners via SSM + enable_ssm_on_runners = true + + # use S3 or KMS SSE to runners S3 bucket + # runner_binaries_s3_sse_configuration = { + # rule = { + # apply_server_side_encryption_by_default = { + # sse_algorithm = "AES256" + # } + # } + # } + + # Uncommet idle config to have idle runners from 9 to 5 in time zone Amsterdam + # idle_config = [{ + # cron = "* * 9-17 * * *" + # timeZone = "Europe/Amsterdam" + # idleCount = 1 + # }] + + # Let the module manage the service linked role + # create_service_linked_role_spot = true + + runner_architecture = "arm64" + # Ensure all instance types have ARM64 architecture (ie. AWS Graviton processors) + instance_types = ["t4g.large", "c6g.large"] + + # override delay of events in seconds + delay_webhook_event = 5 + runners_maximum_count = 1 + + # set up a fifo queue to remain order + fifo_build_queue = true + + # override scaling down + scale_down_schedule_expression = "cron(* * * * ? *)" +} diff --git a/examples/arm64/outputs.tf b/examples/arm64/outputs.tf new file mode 100644 index 0000000000..c50214f566 --- /dev/null +++ b/examples/arm64/outputs.tf @@ -0,0 +1,15 @@ +output "runners" { + value = { + lambda_syncer_name = module.runners.binaries_syncer.lambda.function_name + } +} + +output "webhook_endpoint" { + value = module.runners.webhook.endpoint +} + +output "webhook_secret" { + sensitive = true + value = random_id.random.hex +} + diff --git a/examples/arm64/providers.tf b/examples/arm64/providers.tf new file mode 100644 index 0000000000..b6c81d5415 --- /dev/null +++ b/examples/arm64/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = local.aws_region +} diff --git a/examples/arm64/variables.tf b/examples/arm64/variables.tf new file mode 100644 index 0000000000..69dcd0c61c --- /dev/null +++ b/examples/arm64/variables.tf @@ -0,0 +1,4 @@ + +variable "github_app_key_base64" {} + +variable "github_app_id" {} diff --git a/examples/arm64/versions.tf b/examples/arm64/versions.tf new file mode 100644 index 0000000000..c96d0eee84 --- /dev/null +++ b/examples/arm64/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.27" + } + local = { + source = "hashicorp/local" + } + random = { + source = "hashicorp/random" + } + } + required_version = ">= 0.14" +} diff --git a/examples/arm64/vpc.tf b/examples/arm64/vpc.tf new file mode 100644 index 0000000000..a7d21422f1 --- /dev/null +++ b/examples/arm64/vpc.tf @@ -0,0 +1,7 @@ +module "vpc" { + source = "git::https://github.com/philips-software/terraform-aws-vpc.git?ref=2.2.0" + + environment = local.environment + aws_region = local.aws_region + create_private_hosted_zone = false +} diff --git a/examples/default/main.tf b/examples/default/main.tf index 46e57fc183..68d8903049 100644 --- a/examples/default/main.tf +++ b/examples/default/main.tf @@ -9,7 +9,7 @@ resource "random_id" "random" { ################################################################################ -### Hybrid acccount +### Hybrid account ################################################################################ module "runners" { diff --git a/modules/runners/main.tf b/modules/runners/main.tf index 6c539bcfe4..a5a61863b5 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -12,7 +12,6 @@ locals { instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip userdata_template = var.userdata_template == null ? local.default_userdata_template[var.runner_os] : var.userdata_template - userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" kms_key_arn = var.kms_key_arn != null ? var.kms_key_arn : "" default_ami = { @@ -119,7 +118,7 @@ resource "aws_launch_template" "runner" { pre_install = var.userdata_pre_install install_runner = templatefile(local.userdata_install_runner[var.runner_os], { S3_LOCATION_RUNNER_DISTRIBUTION = var.s3_location_runner_binaries - ARM_PATCH = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : "" + RUNNER_ARCHITECTURE = var.runner_architecture }) post_install = var.userdata_post_install start_runner = templatefile(local.userdata_start_runner[var.runner_os], {}) diff --git a/modules/runners/templates/arm-runner-patch.tpl b/modules/runners/templates/arm-runner-patch.tpl deleted file mode 100644 index 306fe559c8..0000000000 --- a/modules/runners/templates/arm-runner-patch.tpl +++ /dev/null @@ -1,49 +0,0 @@ -# Patch for ARM64 (no ICU install by default) -yum install -y patch -patch -p1 <