diff --git a/README.md b/README.md index 6890e055a0..b38102056b 100644 --- a/README.md +++ b/README.md @@ -356,10 +356,10 @@ In case the setup does not work as intended follow the trace of events: | Name | Source | Version | |------|--------|---------| -| [runner\_binaries](#module\_runner\_binaries) | ./modules/runner-binaries-syncer | | -| [runners](#module\_runners) | ./modules/runners | | -| [ssm](#module\_ssm) | ./modules/ssm | | -| [webhook](#module\_webhook) | ./modules/webhook | | +| [runner\_binaries](#module\_runner\_binaries) | ./modules/runner-binaries-syncer | n/a | +| [runners](#module\_runners) | ./modules/runners | n/a | +| [ssm](#module\_ssm) | ./modules/ssm | n/a | +| [webhook](#module\_webhook) | ./modules/webhook | n/a | ## Resources @@ -373,7 +373,7 @@ In case the setup does not work as intended follow the trace of events: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [ami\_filter](#input\_ami\_filter) | List of maps used to create the AMI filter for the action runner AMI. By default amazon linux 2 is used. | `map(list(string))` | `{}` | no | +| [ami\_filter](#input\_ami\_filter) | List of maps used to create the AMI filter for the action runner AMI. By default amazon linux 2 is used. | `map(list(string))` | `null` | no | | [ami\_owners](#input\_ami\_owners) | The list of owners used to select the AMI of action runner instances. | `list(string)` |
[| no | | [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes | | [block\_device\_mappings](#input\_block\_device\_mappings) | The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops` | `map(string)` | `{}` | no | @@ -384,6 +384,7 @@ In case the setup does not work as intended follow the trace of events: | [enable\_cloudwatch\_agent](#input\_enable\_cloudwatch\_agent) | Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`. | `bool` | `true` | no | | [enable\_organization\_runners](#input\_enable\_organization\_runners) | Register runners to organization, instead of repo level | `bool` | `false` | no | | [enable\_ssm\_on\_runners](#input\_enable\_ssm\_on\_runners) | Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | `false` | no | +| [enabled\_userdata](#input\_enabled\_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no | | [environment](#input\_environment) | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes | | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | @@ -391,10 +392,11 @@ In case the setup does not work as intended follow the trace of events: | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. |
"amazon"
]
list(object({| `[]` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | | [instance\_type](#input\_instance\_type) | [DEPRECATED] See instance\_types. | `string` | `"m5.large"` | no | -| [instance\_types](#input\_instance\_types) | List of instance types for the action runner. | `list(string)` | `null` | no | +| [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (amzn2 for linux and Windows Server Core for win). | `list(string)` | `null` | no | | [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged | `number` | `86400` | no | | [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no | | [kms\_key\_arn](#input\_kms\_key\_arn) | Optional CMK Key ARN to be used for Parameter Store. This key must be in the current account. | `string` | `null` | no | +| [lambda\_principals](#input\_lambda\_principals) | (Optional) add extra principals to the role created for execution of the lambda, e.g. for local testing. |
cron = string
timeZone = string
idleCount = number
}))
list(object({| `[]` | no | | [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no | | [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no | | [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no | @@ -402,7 +404,7 @@ In case the setup does not work as intended follow the trace of events: | [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no | | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [market\_options](#input\_market\_options) | Market options for the action runner instances. Setting the value to `null` let the scaler create on-demand instances instead of spot instances. | `string` | `"spot"` | no | -| [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if not busy. | `number` | `5` | no | +| [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if not busy. | `number` | `null` | no | | [repository\_white\_list](#input\_repository\_white\_list) | List of repositories allowed to use the github app | `list(string)` | `[]` | no | | [role\_path](#input\_role\_path) | The path that will be added to role path for created roles, if not set the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created roles. | `string` | `null` | no | @@ -418,8 +420,9 @@ In case the setup does not work as intended follow the trace of events: | [runner\_extra\_labels](#input\_runner\_extra\_labels) | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no | | [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no | | [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no | -| [runner\_log\_files](#input\_runner\_log\_files) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. |
type = string
identifiers = list(string)
}))
list(object({|
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
}))
[| no | +| [runner\_log\_files](#input\_runner\_log\_files) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. |
{
"file_path": "/var/log/messages",
"log_group_name": "messages",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/var/log/user-data.log",
"log_group_name": "user_data",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
},
{
"file_path": "/home/ec2-user/actions-runner/_diag/Runner_**.log",
"log_group_name": "runner",
"log_stream_name": "{instance_id}",
"prefix_log_group": true
}
]
list(object({| `null` | no | | [runner\_metadata\_options](#input\_runner\_metadata\_options) | Metadata options for the ec2 runner instances. | `map(any)` |
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
}))
{| no | +| [runner\_os](#input\_runner\_os) | The Operating System to use for GitHub Actions Runners (linux,win) | `string` | `"linux"` | no | | [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | | [runners\_lambda\_zip](#input\_runners\_lambda\_zip) | File location of the lambda zip file for scaling runners. | `string` | `null` | no | @@ -432,7 +435,6 @@ In case the setup does not work as intended follow the trace of events: | [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | -| [enabled_userdata](#input\_enabled_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no | | [userdata\_post\_install](#input\_userdata\_post\_install) | Script to be ran after the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no | | [userdata\_pre\_install](#input\_userdata\_pre\_install) | Script to be ran before the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no | | [userdata\_template](#input\_userdata\_template) | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no | diff --git a/docs/test-lambda-local.md b/docs/test-lambda-local.md new file mode 100644 index 0000000000..397884468c --- /dev/null +++ b/docs/test-lambda-local.md @@ -0,0 +1,88 @@ +# Lambda - Test locally + +This README provides guidance for testing the lambda locally / and or in AWS. This guide assumes you are familiar with AWS, lambda and Node. If not mentioned explicitly, comments provided should be executed from the root of the lambda package. + +## Testing in AWS + +Just navigate to the Lambda in the AWS Console and trigger a test event. Provide an event that matches the required input. For lambdas that does not require a specific event, just send any event. + + +## Testing locally + +Testing locally can be done in two ways; using AWS SAM framework or run via a wrapper to simulate the event to invoke the lambda. Both setups require that the mandatory input environment variables be set, and AWS resources on which the lambda depends are available. We advise for testing the lambda locally to first create your own deployment of the whole module to AWS, this will simplify the setup of dependent AWS resources. For example, based on the de [default example](../../../../examples/default/). + +Local test setup instructions are available for the following lambda's: + +- [runner-binary-syncer](./moduele/../../modules/runner-binaries-syncer/lambdas/runner-binaries-syncer) - This lambda does not need any input, no event is required. Supported via SAM and local Node. + +### Extend deployment configuration + +Add the code below to your Terraform deployment to allow your principal to use the Lambda role and retrieve the lambda configuration. Update your Terraform deployment and apply the changes. + +```hcl +data "aws_caller_identity" "current" {} + +module "runners" { + + ... + + # Assume you have a profile with Admin privileges, allow you to switch to the Lambda role + lambda_principals = [{ + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] + }] + +} + +output "development" { + value = { + lambda_syncer = module.runners.binaries_syncer.lambda + } +}``` + +Once you have updated your Terraform deployment you need to read the lambda configuration into your environment. Run the commands below in your Terraform workspace folder. + +```bash +LAMBDA_ENV=$(terraform output -json development | jq -r '.lambda_syncer.environment[].variables' | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]") +for x in $LAMBDA_ENV ; do echo setting $x; export $x; done +``` + +### Testing with SAM + +This setup requires AWS SAM CLI and Docker is installed locally. First update the AWS config (`~/.aws/config`) so you can use easily switch to the role used by the lambda. + +```properties +[profile gh-development] +source_profile=
"http_endpoint": "enabled",
"http_put_response_hop_limit": 1,
"http_tokens": "optional"
}
list(object({| `[]` | no | | [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no | | [lambda\_schedule\_expression](#input\_lambda\_schedule\_expression) | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no | | [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no | @@ -91,6 +92,7 @@ No modules. | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | | [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no | | [runner\_architecture](#input\_runner\_architecture) | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no | +| [runner\_os](#input\_runner\_os) | The operating system for the runner instance (linux, win), defaults to 'linux' | `string` | `"linux"` | no | | [server\_side\_encryption\_configuration](#input\_server\_side\_encryption\_configuration) | Map containing server-side encryption configuration. | `any` | `{}` | no | | [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/README.md b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/README.md new file mode 100644 index 0000000000..a6efc59623 --- /dev/null +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/README.md @@ -0,0 +1,3 @@ +# Lambda runner binary syncer + +For testing the lambda locally check out [this guide](../../../../docs/test-lambda-local.md). \ No newline at end of file diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json index 9b56a39b22..344261538d 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json @@ -26,6 +26,7 @@ "eslint": "^7.32.0", "eslint-plugin-prettier": "4.0.0", "jest": "^27.4.3", + "jest-mock": "^27.4.2", "prettier": "2.5.1", "ts-jest": "^27.1.0", "ts-node-dev": "^1.1.6", @@ -35,4 +36,4 @@ "axios": "^0.24.0", "tslog": "^3.3.0" } -} +} \ No newline at end of file diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.test.ts b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.test.ts index ccc3277f2d..e5bc189aa7 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.test.ts +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.test.ts @@ -1,6 +1,6 @@ import { sync } from './syncer/syncer'; import { handler } from './lambda'; -import { mocked } from 'ts-jest/utils'; +import { mocked } from 'jest-mock'; jest.mock('./syncer/syncer'); diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts index 12d53f26e5..7910f7ac59 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts @@ -9,6 +9,9 @@ export async function handler(event: any, context: any): Promise
type = string
identifiers = list(string)
}))