diff --git a/README.md b/README.md index 02a7f79fb5..6890e055a0 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,8 @@ idle_config = [{ }] ``` +_**Note**_: When using Windows runners it's recommended to keep a few runners warmed up due to the minutes-long cold start time. + ### Prebuilt Images This module also allows you to run agents from a prebuilt AMI to gain faster startup times. You can find more information in [the image README.md](/images/README.md) @@ -294,7 +296,9 @@ Examples are located in the [examples](./examples) directory. The following exam - _[Default](examples/default/README.md)_: The default example of the module - _[Permissions boundary](examples/permissions-boundary/README.md)_: Example usages of permissions boundaries. +- _[Ubuntu](examples/ubuntu/README.md)_: Example usage of creating a runner using Ubuntu AMIs. - _[Prebuilt Images](examples/prebuilt/README.md)_: Example usages of deploying runners with a custom prebuilt image. +- _[Windows](examples/windows/README.md)_: Example usage of creating a runner using Windows as the OS. ## Sub modules diff --git a/examples/default/main.tf b/examples/default/main.tf index 5e433d385f..2a5ec1edee 100644 --- a/examples/default/main.tf +++ b/examples/default/main.tf @@ -3,8 +3,8 @@ locals { aws_region = "eu-west-1" } -resource "random_password" "random" { - length = 28 +resource "random_id" "random" { + byte_length = 20 } @@ -27,7 +27,7 @@ module "runners" { github_app = { key_base64 = var.github_app_key_base64 id = var.github_app_id - webhook_secret = random_password.random.result + webhook_secret = random_id.random.hex } webhook_lambda_zip = "lambdas-download/webhook.zip" diff --git a/examples/default/outputs.tf b/examples/default/outputs.tf index d6886efe36..c50214f566 100644 --- a/examples/default/outputs.tf +++ b/examples/default/outputs.tf @@ -10,6 +10,6 @@ output "webhook_endpoint" { output "webhook_secret" { sensitive = true - value = random_password.random.result + value = random_id.random.hex } diff --git a/examples/permissions-boundary/main.tf b/examples/permissions-boundary/main.tf index eb578f1382..7929c96032 100644 --- a/examples/permissions-boundary/main.tf +++ b/examples/permissions-boundary/main.tf @@ -3,8 +3,8 @@ locals { aws_region = "eu-west-1" } -resource "random_password" "random" { - length = 32 +resource "random_id" "random" { + byte_length = 20 } data "terraform_remote_state" "iam" { @@ -46,7 +46,7 @@ module "runners" { id = var.github_app_id client_id = var.github_app_client_id client_secret = var.github_app_client_secret - webhook_secret = random_password.random.result + webhook_secret = random_id.random.hex } webhook_lambda_zip = "lambdas-download/webhook.zip" diff --git a/examples/permissions-boundary/outputs.tf b/examples/permissions-boundary/outputs.tf index 6af3be0192..fe4a965473 100644 --- a/examples/permissions-boundary/outputs.tf +++ b/examples/permissions-boundary/outputs.tf @@ -6,7 +6,7 @@ output "runners" { output "webhook" { value = { - secret = random_password.random.result + secret = random_id.random.hex endpoint = module.runners.webhook.endpoint } } diff --git a/examples/prebuilt/main.tf b/examples/prebuilt/main.tf index f67c5d984d..7e417f460f 100644 --- a/examples/prebuilt/main.tf +++ b/examples/prebuilt/main.tf @@ -3,8 +3,8 @@ locals { aws_region = "eu-west-1" } -resource "random_password" "random" { - length = 28 +resource "random_id" "random" { + byte_length = 20 } data "aws_caller_identity" "current" {} @@ -21,7 +21,7 @@ module "runners" { github_app = { key_base64 = var.github_app_key_base64 id = var.github_app_id - webhook_secret = random_password.random.result + webhook_secret = random_id.random.hex } webhook_lambda_zip = "../../lambda_output/webhook.zip" diff --git a/examples/prebuilt/outputs.tf b/examples/prebuilt/outputs.tf index d6886efe36..c50214f566 100644 --- a/examples/prebuilt/outputs.tf +++ b/examples/prebuilt/outputs.tf @@ -10,6 +10,6 @@ output "webhook_endpoint" { output "webhook_secret" { sensitive = true - value = random_password.random.result + value = random_id.random.hex } diff --git a/examples/ubuntu/main.tf b/examples/ubuntu/main.tf index 6cadee69d4..346d6f4edd 100644 --- a/examples/ubuntu/main.tf +++ b/examples/ubuntu/main.tf @@ -3,8 +3,8 @@ locals { aws_region = "eu-west-1" } -resource "random_password" "random" { - length = 28 +resource "random_id" "random" { + byte_length = 20 } module "runners" { @@ -22,7 +22,7 @@ module "runners" { github_app = { key_base64 = var.github_app_key_base64 id = var.github_app_id - webhook_secret = random_password.random.result + webhook_secret = random_id.random.hex } # webhook_lambda_zip = "lambdas-download/webhook.zip" diff --git a/examples/ubuntu/outputs.tf b/examples/ubuntu/outputs.tf index 6af3be0192..fe4a965473 100644 --- a/examples/ubuntu/outputs.tf +++ b/examples/ubuntu/outputs.tf @@ -6,7 +6,7 @@ output "runners" { output "webhook" { value = { - secret = random_password.random.result + secret = random_id.random.hex endpoint = module.runners.webhook.endpoint } } diff --git a/examples/windows/.terraform.lock.hcl b/examples/windows/.terraform.lock.hcl new file mode 100644 index 0000000000..9a3138e033 --- /dev/null +++ b/examples/windows/.terraform.lock.hcl @@ -0,0 +1,39 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.68.0" + constraints = ">= 3.38.0" + hashes = [ + "h1:w546dMDYshe7eeOsxSZt7ihMJOKCbl/7ifZ9lI1PUAY=", + "zh:05a43a7dbd409451c08a958610234619d7e0d102e601220b60aad025bf2b6e2c", + "zh:0d195fa738a348e511550de39caec3f10cfb9afe8d69ed2104b39e9129438739", + "zh:3d88a19b2a810559bc6953fe92b7a7c6e3251c5501866c94ef34648df3fdf461", + "zh:3e42fdaf9df636a3741871c4209c9665549d67f07a69dd8700dcdcd43cd367fb", + "zh:690418e0969eb36807832b48099f09e686e3d0fda42f483efc835bdef6363888", + "zh:7158d5ef79dc90f2da61b6bc28d450e8d61a58b314d9abed8a03a09b80a41316", + "zh:7ed4fac5d8de0141559fc4dbf97dd754d5af8c245a946d955b11530293f6f4d6", + "zh:d0961612800f75321014347b69148e2f326d8b9ff2a9ac99074d35ee3f289d17", + "zh:e8d35599fc8f7ca796ada775828f1dbf10668e0c7eb1f052330360eb8a2f83e3", + "zh:e989ac0324fd9d443da317b3d97ec9fb8c8122fa2951ac2356302891a20bb595", + "zh:ff135b9cac355ecd8f69a64206751503fa9aa41147241c9f99ad766f27a6dcd3", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.1.0" + hashes = [ + "h1:EPIax4Ftp2SNdB9pUfoSjxoueDoLc/Ck3EUoeX0Dvsg=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", + ] +} diff --git a/examples/windows/README.md b/examples/windows/README.md new file mode 100644 index 0000000000..8f3e572586 --- /dev/null +++ b/examples/windows/README.md @@ -0,0 +1,26 @@ +# Action runners deployment windows example + +This module shows how to create GitHub action runners using an Windows Runners. Lambda release will be downloaded from GitHub. + +## Usages + +Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](../../README.md). First, download the Lambda releases from GitHub. Alternatively you can build the lambdas locally with Node or Docker, for which there is a build script available at `/.ci/build.sh`. In the `main.tf` you can remove the location of the lambda zip files, the default location will work in this case. + +> Ensure you have set the version in `lambdas-download/main.tf` for running the example. The version needs to be set to a GitHub release version, see + + +```pwsh +cd lambdas-download +terraform init +terraform apply +cd .. +``` + +Before running Terraform, ensure the GitHub app is configured. + +```bash +terraform init +terraform apply +``` + +_**Note**_: It can take upwards of ten minutes for a runner to start processing jobs, and about as long for logs to start showing up. It's recommend that scale the runners via a warm-up job and then keep them idled. diff --git a/examples/windows/lambdas-download/main.tf b/examples/windows/lambdas-download/main.tf new file mode 100644 index 0000000000..87f31bd8a9 --- /dev/null +++ b/examples/windows/lambdas-download/main.tf @@ -0,0 +1,25 @@ +locals { + version = "" +} + +module "lambdas" { + source = "../../../modules/download-lambda" + lambdas = [ + { + name = "webhook" + tag = local.version + }, + { + name = "runners" + tag = local.version + }, + { + name = "runner-binaries-syncer" + tag = local.version + } + ] +} + +output "files" { + value = module.lambdas.files +} diff --git a/examples/windows/main.tf b/examples/windows/main.tf new file mode 100644 index 0000000000..d79edcb59e --- /dev/null +++ b/examples/windows/main.tf @@ -0,0 +1,48 @@ +locals { + environment = "windows" + aws_region = "eu-west-1" +} + +resource "random_id" "random" { + byte_length = 20 +} + +module "runners" { + source = "../../" + + aws_region = local.aws_region + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + environment = local.environment + + github_app = { + key_base64 = var.github_app_key_base64 + id = var.github_app_id + webhook_secret = random_id.random.hex + } + + # Grab the lambda packages from local directory. Must run /.ci/build.sh first + webhook_lambda_zip = "../../lambda_output/webhook.zip" + runner_binaries_syncer_lambda_zip = "../../lambda_output/runner-binaries-syncer.zip" + runners_lambda_zip = "../../lambda_output/runners.zip" + + enable_organization_runners = false + # no need to add extra windows tag here as it is automatically added by GitHub + runner_extra_labels = "default,example" + + # Set the OS to Windows + runner_os = "win" + # we need to give the runner time to start because this is windows. + runner_boot_time_in_minutes = 20 + + # enable access to the runners via SSM + enable_ssm_on_runners = true + + instance_types = ["m5.large", "c5.large"] + + # override delay of events in seconds for testing + delay_webhook_event = 5 + + # override scaling down for testing + scale_down_schedule_expression = "cron(* * * * ? *)" +} diff --git a/examples/windows/outputs.tf b/examples/windows/outputs.tf new file mode 100644 index 0000000000..c50214f566 --- /dev/null +++ b/examples/windows/outputs.tf @@ -0,0 +1,15 @@ +output "runners" { + value = { + lambda_syncer_name = module.runners.binaries_syncer.lambda.function_name + } +} + +output "webhook_endpoint" { + value = module.runners.webhook.endpoint +} + +output "webhook_secret" { + sensitive = true + value = random_id.random.hex +} + diff --git a/examples/windows/providers.tf b/examples/windows/providers.tf new file mode 100644 index 0000000000..b6c81d5415 --- /dev/null +++ b/examples/windows/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = local.aws_region +} diff --git a/examples/windows/variables.tf b/examples/windows/variables.tf new file mode 100644 index 0000000000..69dcd0c61c --- /dev/null +++ b/examples/windows/variables.tf @@ -0,0 +1,4 @@ + +variable "github_app_key_base64" {} + +variable "github_app_id" {} diff --git a/examples/windows/vpc.tf b/examples/windows/vpc.tf new file mode 100644 index 0000000000..a7d21422f1 --- /dev/null +++ b/examples/windows/vpc.tf @@ -0,0 +1,7 @@ +module "vpc" { + source = "git::https://github.com/philips-software/terraform-aws-vpc.git?ref=2.2.0" + + environment = local.environment + aws_region = local.aws_region + create_private_hosted_zone = false +} diff --git a/main.tf b/main.tf index 6708b4cd06..9f3ace823b 100644 --- a/main.tf +++ b/main.tf @@ -6,9 +6,6 @@ locals { s3_action_runner_url = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key}" runner_architecture = substr(var.instance_type, 0, 2) == "a1" || substr(var.instance_type, 1, 2) == "6g" ? "arm64" : "x64" - - ami_filter = length(var.ami_filter) > 0 ? var.ami_filter : local.runner_architecture == "arm64" ? { name = ["amzn2-ami-hvm-2*-arm64-gp2"] } : { name = ["amzn2-ami-hvm-2.*-x86_64-ebs"] } - github_app_parameters = { id = module.ssm.parameters.github_app_id key_base64 = module.ssm.parameters.github_app_key_base64 @@ -82,13 +79,14 @@ module "runners" { s3_bucket_runner_binaries = module.runner_binaries.bucket s3_location_runner_binaries = local.s3_action_runner_url + runner_os = var.runner_os instance_type = var.instance_type instance_types = var.instance_types market_options = var.market_options block_device_mappings = var.block_device_mappings runner_architecture = local.runner_architecture - ami_filter = local.ami_filter + ami_filter = var.ami_filter ami_owners = var.ami_owners sqs_build_queue = aws_sqs_queue.queued_builds @@ -96,6 +94,7 @@ module "runners" { enable_organization_runners = var.enable_organization_runners scale_down_schedule_expression = var.scale_down_schedule_expression minimum_running_time_in_minutes = var.minimum_running_time_in_minutes + runner_boot_time_in_minutes = var.runner_boot_time_in_minutes runner_extra_labels = var.runner_extra_labels runner_as_root = var.runner_as_root runners_maximum_count = var.runners_maximum_count @@ -155,6 +154,7 @@ module "runner_binaries" { distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}" + runner_os = var.runner_os runner_architecture = local.runner_architecture runner_allow_prerelease_binaries = var.runner_allow_prerelease_binaries diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.test.ts b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.test.ts index fc17f1fef5..61ff725843 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.test.ts +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.test.ts @@ -310,3 +310,23 @@ describe('Synchronize action distribution for arm64.', () => { await expect(sync()).rejects.toThrow(errorMessage); }); }); + +describe('Synchronize action distribution for windows.', () => { + const errorMessage = 'Cannot find GitHub release asset.'; + beforeEach(() => { + process.env.S3_BUCKET_NAME = bucketName; + process.env.S3_OBJECT_KEY = bucketObjectKey; + process.env.GITHUB_RUNNER_OS = 'win'; + }); + + it('No win asset.', async () => { + mockOctokit.repos.listReleases.mockImplementation(() => ({ + data: listReleases.map((release) => ({ + ...release, + assets: release.assets.filter((asset) => !asset.name.includes('win')), + })), + })); + + await expect(sync()).rejects.toThrow(errorMessage); + }); +}); diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.ts b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.ts index 4787bf7602..a36967e1cb 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.ts +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.ts @@ -34,7 +34,8 @@ interface ReleaseAsset { downloadUrl: string; } -async function getLinuxReleaseAsset( +async function getReleaseAsset( + runnerOs = 'linux', runnerArch = 'x64', fetchPrereleaseBinaries = false, ): Promise { @@ -58,11 +59,11 @@ async function getLinuxReleaseAsset( } else { return undefined; } - const linuxAssets = asset.assets?.filter((a) => a.name?.includes(`actions-runner-linux-${runnerArch}-`)); + const assets = asset.assets?.filter((a: { name?: string }) => + a.name?.includes(`actions-runner-${runnerOs}-${runnerArch}-`), + ); - return linuxAssets?.length === 1 - ? { name: linuxAssets[0].name, downloadUrl: linuxAssets[0].browser_download_url } - : undefined; + return assets?.length === 1 ? { name: assets[0].name, downloadUrl: assets[0].browser_download_url } : undefined; } async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseAsset: ReleaseAsset): Promise { @@ -106,6 +107,7 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA export async function sync(): Promise { const s3 = new AWS.S3(); + const runnerOs = process.env.GITHUB_RUNNER_OS || 'linux'; const runnerArch = process.env.GITHUB_RUNNER_ARCHITECTURE || 'x64'; const fetchPrereleaseBinaries = JSON.parse(process.env.GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES || 'false'); @@ -116,8 +118,7 @@ export async function sync(): Promise { if (!cacheObject.bucket || !cacheObject.key) { throw Error('Please check all mandatory variables are set.'); } - - const actionRunnerReleaseAsset = await getLinuxReleaseAsset(runnerArch, fetchPrereleaseBinaries); + const actionRunnerReleaseAsset = await getReleaseAsset(runnerOs, runnerArch, fetchPrereleaseBinaries); if (actionRunnerReleaseAsset === undefined) { throw Error('Cannot find GitHub release asset.'); } diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock index 7dedb2e533..8e42bbfed9 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock @@ -3365,6 +3365,7 @@ tr46@^2.1.0: resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" integrity sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw== dependencies: + psl "^1.1.28" punycode "^2.1.1" tr46@~0.0.3: diff --git a/modules/runner-binaries-syncer/main.tf b/modules/runner-binaries-syncer/main.tf index 7707d73890..32254b5c0a 100644 --- a/modules/runner-binaries-syncer/main.tf +++ b/modules/runner-binaries-syncer/main.tf @@ -1,5 +1,5 @@ locals { - action_runner_distribution_object_key = "actions-runner-linux.tar.gz" + action_runner_distribution_object_key = "actions-runner-${var.runner_os}.${var.runner_os == "linux" ? "tar.gz" : "zip"}" } resource "aws_s3_bucket" "action_dist" { diff --git a/modules/runner-binaries-syncer/runner-binaries-syncer.tf b/modules/runner-binaries-syncer/runner-binaries-syncer.tf index 04f2cd0c47..8b3f449cae 100644 --- a/modules/runner-binaries-syncer/runner-binaries-syncer.tf +++ b/modules/runner-binaries-syncer/runner-binaries-syncer.tf @@ -18,8 +18,9 @@ resource "aws_lambda_function" "syncer" { environment { variables = { - GITHUB_RUNNER_ARCHITECTURE = var.runner_architecture GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = var.runner_allow_prerelease_binaries + GITHUB_RUNNER_ARCHITECTURE = var.runner_architecture + GITHUB_RUNNER_OS = var.runner_os LOG_LEVEL = var.log_level LOG_TYPE = var.log_type S3_BUCKET_NAME = aws_s3_bucket.action_dist.id diff --git a/modules/runner-binaries-syncer/variables.tf b/modules/runner-binaries-syncer/variables.tf index c70e2a9cd1..c893153e95 100644 --- a/modules/runner-binaries-syncer/variables.tf +++ b/modules/runner-binaries-syncer/variables.tf @@ -54,6 +54,12 @@ variable "role_path" { default = null } +variable "runner_os" { + description = "The operating system for the runner instance (linux, win), defaults to 'linux'" + type = string + default = "linux" +} + variable "runner_architecture" { description = "The platform architecture for the runner instance (x64, arm64), defaults to 'x64'" type = string diff --git a/modules/runners/logging.tf b/modules/runners/logging.tf index e0b19d59a8..3d37ef804f 100644 --- a/modules/runners/logging.tf +++ b/modules/runners/logging.tf @@ -1,5 +1,35 @@ locals { - logfiles = var.enable_cloudwatch_agent ? [for l in var.runner_log_files : { + runner_log_files = ( + var.runner_log_files != null + ? var.runner_log_files + : [ + { + "prefix_log_group" : true, + "file_path" : "/var/log/messages", + "log_group_name" : "messages", + "log_stream_name" : "{instance_id}" + }, + { + "log_group_name" : "user_data", + "prefix_log_group" : true, + "file_path" : var.runner_os == "win" ? "C:/UserData.log" : "/var/log/user-data.log", + "log_stream_name" : "{instance_id}" + }, + { + "log_group_name" : "runner", + "prefix_log_group" : true, + "file_path" : var.runner_os == "win" ? "C:/actions-runner/_diag/Runner_*.log" : "/home/runners/actions-runner/_diag/Runner_**.log", + "log_stream_name" : "{instance_id}" + }, + { + "log_group_name" : "runner-startup", + "prefix_log_group" : true, + "file_path" : var.runner_os == "win" ? "C:/runner-startup.log" : "/var/log/runner-startup.log", + "log_stream_name" : "{instance_id}" + } + ] + ) + logfiles = var.enable_cloudwatch_agent ? [for l in local.runner_log_files : { "log_group_name" : l.prefix_log_group ? "/github-self-hosted-runners/${var.environment}/${l.log_group_name}" : "/${l.log_group_name}" "log_stream_name" : l.log_stream_name "file_path" : l.file_path diff --git a/modules/runners/main.tf b/modules/runners/main.tf index 8a48fece0e..a6deec5b05 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -6,26 +6,44 @@ locals { var.tags, ) - name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] - name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"] - role_path = var.role_path == null ? "/${var.environment}/" : var.role_path - instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path - lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip - userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template - userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" - userdata_install_runner = "${path.module}/templates/install-runner.sh" - userdata_start_runner = "${path.module}/templates/start-runner.sh" - - instance_types = distinct(var.instance_types == null ? [var.instance_type] : var.instance_types) - - kms_key_arn = var.kms_key_arn != null ? var.kms_key_arn : "" + name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] + name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"] + role_path = var.role_path == null ? "/${var.environment}/" : var.role_path + instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path + lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip + userdata_template = var.userdata_template == null ? local.default_userdata_template[var.runner_os] : var.userdata_template + userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" + instance_types = distinct(var.instance_types == null ? [var.instance_type] : var.instance_types) + kms_key_arn = var.kms_key_arn != null ? var.kms_key_arn : "" + + default_ami = { + "win" = { name = ["Windows_Server-20H2-English-Core-ContainersLatest-*"] } + "linux" = var.runner_architecture == "arm64" ? { name = ["amzn2-ami-hvm-2*-arm64-gp2"] } : { name = ["amzn2-ami-hvm-2.*-x86_64-ebs"] } + } + + default_userdata_template = { + "win" = "${path.module}/templates/user-data.ps1" + "linux" = "${path.module}/templates/user-data.sh" + } + + userdata_install_runner = { + "win" = "${path.module}/templates/install-runner.ps1" + "linux" = "${path.module}/templates/install-runner.sh" + } + + userdata_start_runner = { + "win" = "${path.module}/templates/start-runner.ps1" + "linux" = "${path.module}/templates/start-runner.sh" + } + + ami_filter = coalesce(var.ami_filter, local.default_ami[var.runner_os]) } data "aws_ami" "runner" { most_recent = "true" dynamic "filter" { - for_each = var.ami_filter + for_each = local.ami_filter content { name = filter.key values = filter.value @@ -112,12 +130,12 @@ resource "aws_launch_template" "runner" { user_data = var.enabled_userdata ? base64encode(templatefile(local.userdata_template, { pre_install = var.userdata_pre_install - install_runner = templatefile(local.userdata_install_runner, { + install_runner = templatefile(local.userdata_install_runner[var.runner_os], { S3_LOCATION_RUNNER_DISTRIBUTION = var.s3_location_runner_binaries ARM_PATCH = var.runner_architecture == "arm64" ? templatefile(local.userdata_arm_patch, {}) : "" }) post_install = var.userdata_post_install - start_runner = templatefile(local.userdata_start_runner, {}) + start_runner = templatefile(local.userdata_start_runner[var.runner_os], {}) ghes_url = var.ghes_url ghes_ssl_verify = var.ghes_ssl_verify ## retain these for backwards compatibility diff --git a/modules/runners/policies-runner.tf b/modules/runners/policies-runner.tf index 212f9a4b24..2ac1b87454 100644 --- a/modules/runners/policies-runner.tf +++ b/modules/runners/policies-runner.tf @@ -26,8 +26,8 @@ resource "aws_iam_role_policy" "ssm_parameters" { role = aws_iam_role.runner.name policy = templatefile("${path.module}/policies/instance-ssm-parameters-policy.json", { - arn_ssm_parameters_prefix= "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*" - arn_ssm_parameters_path ="arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}/*" + arn_ssm_parameters_prefix = "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*" + arn_ssm_parameters_path = "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}/*" } ) } diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index d3cb5beb37..042535cc7b 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -1,3 +1,10 @@ +locals { + # Windows Runners can take their sweet time to do anything + min_runtime_defaults = { + "win" = 15 + "linux" = 5 + } +} resource "aws_lambda_function" "scale_down" { s3_bucket = var.lambda_s3_bucket != null ? var.lambda_s3_bucket : null s3_key = var.runners_lambda_s3_key != null ? var.runners_lambda_s3_key : null @@ -18,7 +25,7 @@ resource "aws_lambda_function" "scale_down" { GHES_URL = var.ghes_url LOG_LEVEL = var.log_level LOG_TYPE = var.log_type - MINIMUM_RUNNING_TIME_IN_MINUTES = var.minimum_running_time_in_minutes + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.minimum_running_time_in_minutes, local.min_runtime_defaults[var.runner_os]) NODE_TLS_REJECT_UNAUTHORIZED = var.ghes_url != null && !var.ghes_ssl_verify ? 0 : 1 PARAMETER_GITHUB_APP_ID_NAME = var.github_app_parameters.id.name PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name diff --git a/modules/runners/templates/install-runner.ps1 b/modules/runners/templates/install-runner.ps1 new file mode 100644 index 0000000000..61eb727542 --- /dev/null +++ b/modules/runners/templates/install-runner.ps1 @@ -0,0 +1,14 @@ +## install the runner + +Write-Host "Creating actions-runner directory for the GH Action installtion" +New-Item -ItemType Directory -Path C:\actions-runner ; Set-Location C:\actions-runner + +Write-Host "Downloading the GH Action runner from s3 bucket $s3_location" +aws s3 cp ${S3_LOCATION_RUNNER_DISTRIBUTION} actions-runner.zip + +Write-Host "Un-zip action runner" +Expand-Archive -Path actions-runner.zip -DestinationPath . + +Write-Host "Delete zip file" +Remove-Item actions-runner.zip + diff --git a/modules/runners/templates/start-runner.ps1 b/modules/runners/templates/start-runner.ps1 new file mode 100644 index 0000000000..851d5a3d22 --- /dev/null +++ b/modules/runners/templates/start-runner.ps1 @@ -0,0 +1,97 @@ + +## Retrieve instance metadata + +Write-Host "Retrieving TOKEN from AWS API" +$token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"} + +$metadata=Invoke-RestMethod -Uri "http://169.254.169.254/latest/dynamic/instance-identity/document" -Headers @{"X-aws-ec2-metadata-token" = $token} + +$Region = $metadata.region +Write-Host "Reteieved REGION from AWS API ($Region)" + +$InstanceId = $metadata.instanceId +Write-Host "Reteieved InstanceId from AWS API ($InstanceId)" + +$tags=aws ec2 describe-tags --region "$Region" --filters "Name=resource-id,Values=$InstanceId" | ConvertFrom-Json +Write-Host "Retrieved tags from AWS API" + +$environment=$tags.Tags.where( {$_.Key -eq 'ghr:environment'}).value +Write-Host "Reteieved ghr:environment tag - ($environment)" + +$parameters=$(aws ssm get-parameters-by-path --path "/$environment/runner" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json +Write-Host "Retrieved parameters from AWS SSM" + +$run_as=$parameters.where( {$_.Name -eq "/$environment/runner/run-as"}).value +Write-Host "Retrieved /$environment/runner/run-as parameter - ($run_as)" + +$enable_cloudwatch_agent=$parameters.where( {$_.Name -eq "/$environment/runner/enable-cloudwatch"}).value +Write-Host "Retrieved /$environment/runner/enable-cloudwatch parameter - ($enable_cloudwatch_agent)" + +$agent_mode=$parameters.where( {$_.Name -eq "/$environment/runner/agent-mode"}).value +Write-Host "Retrieved /$environment/runner/agent-mode parameter - ($agent_mode)" + +if ($enable_cloudwatch_agent) +{ + Write-Host "Enabling CloudWatch Agent" + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "ssm:$environment-cloudwatch_agent_config_runner" +} + +## Configure the runner + +Write-Host "Get GH Runner config from AWS SSM" +$config = $null +$i = 0 +do { + $config = (aws ssm get-parameters --names "$environment-$InstanceId" --with-decryption --region $Region --query "Parameters[*].{Name:Name,Value:Value}" | ConvertFrom-Json)[0].value + Write-Host "Waiting for GH Runner config to become available in AWS SSM ($i/30)" + Start-Sleep 1 + $i++ +} while (($null -eq $config) -and ($i -lt 30)) + +Write-Host "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$environment-$InstanceId" --region $Region + +# Create or update user +if (-not($run_as)) { + Write-Host "No user specified, using default ec2-user account" + $run_as="ec2-user" +} +Add-Type -AssemblyName "System.Web" +$password = [System.Web.Security.Membership]::GeneratePassword(24, 4) +$securePassword = ConvertTo-SecureString $password -AsPlainText -Force +$username = $run_as +if (!(Get-LocalUser -Name $username -ErrorAction Ignore)) { + New-LocalUser -Name $username -Password $securePassword + Write-Host "Created new user ($username)" +} +else { + Set-LocalUser -Name $username -Password $securePassword + Write-Host "Changed password for user ($username)" +} +# Add user to groups +foreach ($group in @("Administrators", "docker-users")) { + if ((Get-LocalGroup -Name "$group" -ErrorAction Ignore) -and + !(Get-LocalGroupMember -Group "$group" -Member $username -ErrorAction Ignore)) { + Add-LocalGroupMember -Group "$group" -Member $username + Write-Host "Added $username to $group group" + } +} + +# Disable User Access Control (UAC) +# TODO investigate if this is needed or if its overkill - https://github.com/philips-labs/terraform-aws-github-runner/issues/1505 +Set-ItemProperty HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name ConsentPromptBehaviorAdmin -Value 0 -Force +Write-Host "Disabled User Access Control (UAC)" + +$configCmd = ".\config.cmd --unattended --name $InstanceId --work `"_work`" $config" +Write-Host "Configure GH Runner as user $run_as" +Invoke-Expression $configCmd + +Write-Host "Starting the runner as user $run_as" + +Write-Host "Installing the runner as a service" + +$action = New-ScheduledTaskAction -WorkingDirectory "$pwd" -Execute "run.cmd" +$trigger = Get-CimClass "MSFT_TaskRegistrationTrigger" -Namespace "Root/Microsoft/Windows/TaskScheduler" +Register-ScheduledTask -TaskName "runnertask" -Action $action -Trigger $trigger -User $username -Password $password -RunLevel Highest -Force +Write-Host "Starting the runner in persistent mode" +Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))" \ No newline at end of file diff --git a/modules/runners/templates/start-runner.sh b/modules/runners/templates/start-runner.sh index 92e1e34526..38422e6440 100644 --- a/modules/runners/templates/start-runner.sh +++ b/modules/runners/templates/start-runner.sh @@ -37,11 +37,11 @@ fi ## Configure the runner -echo "Get GH Runner token from AWS SSM" +echo "Get GH Runner config from AWS SSM" config=$(aws ssm get-parameters --names "$environment"-"$instance_id" --with-decryption --region "$region" | jq -r ".Parameters | .[0] | .Value") while [[ -z "$config" ]]; do - echo "Waiting for GH Runner token to become available in AWS SSM" + echo "Waiting for GH Runner config to become available in AWS SSM" sleep 1 config=$(aws ssm get-parameters --names "$environment"-"$instance_id" --with-decryption --region "$region" | jq -r ".Parameters | .[0] | .Value") done @@ -50,6 +50,7 @@ echo "Delete GH Runner token from AWS SSM" aws ssm delete-parameter --name "$environment"-"$instance_id" --region "$region" if [ -z "$run_as" ]; then + echo "No user specified, using default ec2-user account" run_as="ec2-user" fi diff --git a/modules/runners/templates/user-data.ps1 b/modules/runners/templates/user-data.ps1 new file mode 100644 index 0000000000..0224c473a4 --- /dev/null +++ b/modules/runners/templates/user-data.ps1 @@ -0,0 +1,47 @@ + +$ErrorActionPreference = "Continue" +$VerbosePreference = "Continue" +Start-Transcript -Path "C:\UserData.log" -Append + +${pre_install} + +# Install Chocolatey +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +$env:chocolateyUseWindowsCompression = 'true' +Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicParsing | Invoke-Expression + +# Add Chocolatey to powershell profile +$ChocoProfileValue = @' +$ChocolateyProfile = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +if (Test-Path($ChocolateyProfile)) { + Import-Module "$ChocolateyProfile" +} + +refreshenv +'@ +# Write it to the $profile location +Set-Content -Path "$PsHome\Microsoft.PowerShell_profile.ps1" -Value $ChocoProfileValue -Force +# Source it +. "$PsHome\Microsoft.PowerShell_profile.ps1" + + +refreshenv + +Write-Host "Installing cloudwatch agent..." +Invoke-WebRequest -Uri https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi -OutFile C:\amazon-cloudwatch-agent.msi +$cloudwatchParams = '/i', 'C:\amazon-cloudwatch-agent.msi', '/qn', '/L*v', 'C:\CloudwatchInstall.log' +Start-Process "msiexec.exe" $cloudwatchParams -Wait -NoNewWindow +Remove-Item C:\amazon-cloudwatch-agent.msi + + +# Install dependent tools +Write-Host "Installing additional development tools" +choco install git awscli -y +refreshenv + +${install_runner} +${post_install} +${start_runner} + +Stop-Transcript + \ No newline at end of file diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index c349557fde..80416b7894 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -57,6 +57,17 @@ variable "market_options" { default = "spot" } +variable "runner_os" { + description = "The EC2 Operating System type to use for action runner instances (linux,win)." + type = string + default = "linux" + + validation { + condition = contains(["linux", "win"], var.runner_os) + error_message = "Valid values for runner_os are (linux, win)." + } +} + variable "instance_type" { description = "[DEPRECATED] See instance_types." type = string @@ -64,7 +75,7 @@ variable "instance_type" { } variable "instance_types" { - description = "List of instance types for the action runner." + description = "List of instance types for the action runner. Defaults are based on runner_os (amzn2 for linux and Windows Server Core for win)." type = list(string) default = null } @@ -72,10 +83,7 @@ variable "instance_types" { variable "ami_filter" { description = "Map of lists used to create the AMI filter for the action runner AMI." type = map(list(string)) - - default = { - name = ["amzn2-ami-hvm-2.*-x86_64-ebs"] - } + default = null } variable "ami_owners" { @@ -134,9 +142,9 @@ variable "scale_down_schedule_expression" { } variable "minimum_running_time_in_minutes" { - description = "The time an ec2 action runner should be running at minimum before terminated if non busy." + description = "The time an ec2 action runner should be running at minimum before terminated if non busy. If not set the default is calculated based on the OS." type = number - default = 5 + default = null } variable "runner_boot_time_in_minutes" { @@ -285,32 +293,7 @@ variable "runner_log_files" { file_path = string log_stream_name = string })) - default = [ - { - "log_group_name" : "messages", - "prefix_log_group" : true, - "file_path" : "/var/log/messages", - "log_stream_name" : "{instance_id}" - }, - { - "log_group_name" : "user_data", - "prefix_log_group" : true, - "file_path" : "/var/log/user-data.log", - "log_stream_name" : "{instance_id}" - }, - { - "log_group_name" : "runner", - "prefix_log_group" : true, - "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log", - "log_stream_name" : "{instance_id}" - }, - { - "log_group_name" : "runner-startup", - "prefix_log_group" : true, - "file_path" : "/var/log/runner-startup.log", - "log_stream_name" : "{instance_id}" - }, - ] + default = null } variable "ghes_url" { diff --git a/variables.tf b/variables.tf index a627a067f2..606f7e310f 100644 --- a/variables.tf +++ b/variables.tf @@ -48,7 +48,7 @@ variable "scale_down_schedule_expression" { variable "minimum_running_time_in_minutes" { description = "The time an ec2 action runner should be running at minimum before terminated if not busy." type = number - default = 5 + default = null } variable "runner_boot_time_in_minutes" { @@ -226,8 +226,7 @@ variable "block_device_mappings" { variable "ami_filter" { description = "List of maps used to create the AMI filter for the action runner AMI. By default amazon linux 2 is used." type = map(list(string)) - - default = {} + default = null } variable "ami_owners" { description = "The list of owners used to select the AMI of action runner instances." @@ -301,32 +300,7 @@ variable "runner_log_files" { file_path = string log_stream_name = string })) - default = [ - { - "log_group_name" : "messages", - "prefix_log_group" : true, - "file_path" : "/var/log/messages", - "log_stream_name" : "{instance_id}" - }, - { - "log_group_name" : "user_data", - "prefix_log_group" : true, - "file_path" : "/var/log/user-data.log", - "log_stream_name" : "{instance_id}" - }, - { - "log_group_name" : "runner", - "prefix_log_group" : true, - "file_path" : "/home/ec2-user/actions-runner/_diag/Runner_**.log", - "log_stream_name" : "{instance_id}" - }, - { - "log_group_name" : "runner-startup", - "prefix_log_group" : true, - "file_path" : "/var/log/runner-startup.log", - "log_stream_name" : "{instance_id}" - }, - ] + default = null } variable "ghes_url" { @@ -378,7 +352,7 @@ variable "volume_size" { } variable "instance_types" { - description = "List of instance types for the action runner." + description = "List of instance types for the action runner. Defaults are based on runner_os (amzn2 for linux and Windows Server Core for win)." type = list(string) default = null } @@ -479,3 +453,14 @@ variable "runner_metadata_options" { } } + +variable "runner_os" { + description = "The Operating System to use for GitHub Actions Runners (linux,win)" + type = string + default = "linux" + + validation { + condition = contains(["linux", "win"], var.runner_os) + error_message = "Valid values for runner_os are (linux, win)." + } +}