JSON Logging for Lambdas

.github/workflows/terraform.yml

@@ -13,12 +13,11 @@ env:
   tf_working_dir: "."
   AWS_REGION: eu-west-1
 jobs:
-
   verify_module:
     name: Verify module
     strategy:
       matrix:
-        terraform: [1.0.8]    
+        terraform: [1.0.8]
     runs-on: ubuntu-latest
     container:
       image: hashicorp/terraform:${{ matrix.terraform }}
@@ -43,15 +42,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        terraform: [0.14.1, 0.15.0, 1.0.8]
+        terraform: [0.14.3, 0.15.5, 1.0.8]
         example: ["default", "ubuntu"]
     defaults:
       run:
         working-directory: examples/${{ matrix.example }}
     runs-on: ubuntu-latest
     container:
       image: hashicorp/terraform:${{ matrix.terraform }}
-    steps:   
+    steps:
       - uses: actions/checkout@v2
       - name: terraform init
         run: terraform init -get -backend=false -input=false
@@ -61,5 +60,3 @@ jobs:
         continue-on-error: true
       - name: validate terraform
         run: terraform validate
-
-

main.tf

@@ -64,6 +64,9 @@ module "webhook" {
   role_path                 = var.role_path
   role_permissions_boundary = var.role_permissions_boundary
   repository_white_list     = var.repository_white_list
+
+  log_type  = var.log_type
+  log_level = var.log_level
 }
 
 module "runners" {
@@ -133,6 +136,9 @@ module "runners" {
   ghes_ssl_verify = var.ghes_ssl_verify
 
   kms_key_arn = var.kms_key_arn
+
+  log_type  = var.log_type
+  log_level = var.log_level
 }
 
 module "runner_binaries" {
@@ -156,6 +162,9 @@ module "runner_binaries" {
 
   role_path                 = var.role_path
   role_permissions_boundary = var.role_permissions_boundary
+
+  log_type  = var.log_type
+  log_level = var.log_level
 }
 
 resource "aws_resourcegroups_group" "resourcegroups_group" {

modules/runner-binaries-syncer/README.md

@@ -75,23 +75,25 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes |
-| <a name="input_distribution_bucket_name"></a> [distribution\_bucket\_name](#input\_distribution\_bucket\_name) | Bucket for storing the action runner distribution. | `string` | n/a | yes |
-| <a name="input_environment"></a> [environment](#input\_environment) | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
-| <a name="input_lambda_s3_bucket"></a> [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no |
-| <a name="input_lambda_schedule_expression"></a> [lambda\_schedule\_expression](#input\_lambda\_schedule\_expression) | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no |
-| <a name="input_lambda_security_group_ids"></a> [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
-| <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
-| <a name="input_lambda_timeout"></a> [lambda\_timeout](#input\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `300` | no |
-| <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
-| <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
-| <a name="input_role_path"></a> [role\_path](#input\_role\_path) | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
-| <a name="input_role_permissions_boundary"></a> [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
-| <a name="input_runner_allow_prerelease_binaries"></a> [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
-| <a name="input_runner_architecture"></a> [runner\_architecture](#input\_runner\_architecture) | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
-| <a name="input_syncer_lambda_s3_key"></a> [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
-| <a name="input_syncer_lambda_s3_object_version"></a> [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
-| <a name="input_tags"></a> [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
+| aws\_region | AWS region. | `string` | n/a | yes |
+| distribution\_bucket\_name | Bucket for storing the action runner distribution. | `string` | n/a | yes |
+| environment | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
+| lambda\_s3\_bucket | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `any` | `null` | no |
+| lambda\_schedule\_expression | Scheduler expression for action runner binary syncer. | `string` | `"cron(27 * * * ? *)"` | no |
+| lambda\_security\_group\_ids | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
+| lambda\_subnet\_ids | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
+| lambda\_timeout | Time out of the lambda in seconds. | `number` | `300` | no |
+| lambda\_zip | File location of the lambda zip file. | `string` | `null` | no |
+| log\_level | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
+| log\_type | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
+| logging\_retention\_in\_days | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `7` | no |
+| role\_path | The path that will be added to the role, if not set the environment name will be used. | `string` | `null` | no |
+| role\_permissions\_boundary | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no |
+| runner\_allow\_prerelease\_binaries | Allow the runners to update to prerelease binaries. | `bool` | `false` | no |
+| runner\_architecture | The platform architecture for the runner instance (x64, arm64), defaults to 'x64' | `string` | `"x64"` | no |
+| syncer\_lambda\_s3\_key | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
+| syncer\_lambda\_s3\_object\_version | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
+| tags | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
 
 ## Outputs
 

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

@@ -32,6 +32,7 @@
     "typescript": "^4.4.4"
   },
   "dependencies": {
+    "tslog": "^3.2.2",
     "axios": "^0.24.0"
   }
-}
+}

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/lambda.ts

@@ -1,7 +1,10 @@
 import { handle } from './syncer/handler';
+import { logger } from './syncer/logger';
 
 // eslint-disable-next-line
 export const handler = async (event: any, context: any, callback: any): Promise<void> => {
+  logger.setSettings({ requestId: context.awsRequestId });
+  logger.debug(JSON.stringify(event));
   try {
     await handle();
     callback(null);

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/handler.ts

@@ -3,6 +3,9 @@ import { PassThrough } from 'stream';
 import { S3 } from 'aws-sdk';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { logger as rootLogger } from './logger';
+
+const logger = rootLogger.getChildLogger();
 
 const versionKey = 'name';
 
@@ -22,7 +25,7 @@ async function getCachedVersion(s3: S3, cacheObject: CacheObject): Promise<strin
     const versions = objectTagging.TagSet?.filter((t: S3.Tag) => t.Key === versionKey);
     return versions.length === 1 ? versions[0].Value : undefined;
   } catch (e) {
-    console.debug('No tags found');
+    logger.debug('No tags found');
     return undefined;
   }
 }
@@ -73,7 +76,7 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
     })
     .promise();
 
-  console.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);
+  logger.debug('Start downloading %s and uploading to S3.', actionRunnerReleaseAsset.name);
 
   const readPromise = new Promise<void>((resolve, reject) => {
     axios
@@ -93,9 +96,9 @@ async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseA
   });
 
   await Promise.all([readPromise, writePromise])
-    .then(() => console.info(`The new distribution is uploaded to S3.`))
+    .then(() => logger.info(`The new distribution is uploaded to S3.`))
     .catch((error) => {
-      console.error(`Uploading of the new distribution to S3 failed: ${error}`);
+      logger.error(`Uploading of the new distribution to S3 failed: ${error}`);
       throw error;
     });
 }
@@ -120,10 +123,10 @@ export const handle = async (): Promise<void> => {
   }
 
   const currentVersion = await getCachedVersion(s3, cacheObject);
-  console.debug('latest: ' + currentVersion);
+  logger.debug('latest: ' + currentVersion);
   if (currentVersion === undefined || currentVersion != actionRunnerReleaseAsset.name) {
     uploadToS3(s3, cacheObject, actionRunnerReleaseAsset);
   } else {
-    console.debug('Distribution is up-to-date, no action.');
+    logger.debug('Distribution is up-to-date, no action.');
   }
 };

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/logger.ts

@@ -0,0 +1,10 @@
+import { Logger } from 'tslog';
+
+export const logger = new Logger({
+  colorizePrettyLogs: false,
+  displayInstanceName: false,
+  minLevel: process.env.LOG_LEVEL || 'info',
+  name: 'runner-binaries-syncer',
+  overwriteConsole: true,
+  type: process.env.LOG_TYPE || 'pretty',
+});

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/modules.d.ts

@@ -0,0 +1,6 @@
+declare namespace NodeJS {
+  export interface ProcessEnv {
+    LOG_LEVEL: 'silly' | 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
+    LOG_TYPE: 'json' | 'pretty' | 'hidden';
+  }
+}

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/tsconfig.json

@@ -1,10 +1,13 @@
 {
   "compilerOptions": {
     /* Basic Options */
-    "target": "es6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */,
+    "target": "es2019" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */,
     "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
     "outDir": "build",
-    "lib": ["es2020", "DOM"] /* Specify library files to be included in the compilation. */,
+    "lib": [
+      "es2020",
+      "DOM"
+    ] /* Specify library files to be included in the compilation. */,
     "allowJs": true /* Allow javascript files to be compiled. */,
     // "checkJs": true,                       /* Report errors in .js files. */
     // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
@@ -54,7 +57,10 @@
     /* Experimental Options */
     "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */,
     "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */,
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
+    "sourceMap": true
   },
-  "include": ["src/**/*"]
+  "include": [
+    "src/**/*"
+  ]
 }

modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock

@@ -3148,10 +3148,10 @@ source-map-support@^0.5.12, source-map-support@^0.5.17:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
-source-map-support@^0.5.6:
+source-map-support@^0.5.19, source-map-support@^0.5.6:
   version "0.5.20"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.20.tgz#12166089f8f5e5e8c56926b377633392dd2cb6c9"
-  integrity sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw==
+  integrity sha1-EhZgifj15ejFaSazd2Mzkt0stsk=
   dependencies:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
@@ -3421,6 +3421,13 @@ tslib@^1.8.1:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
+tslog@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/tslog/-/tslog-3.2.2.tgz#5bbaa1fab685c4273e59b38064227321a69a0694"
+  integrity sha1-W7qh+raFxCc+WbOAZCJzIaaaBpQ=
+  dependencies:
+    source-map-support "^0.5.19"
+
 tsutils@^3.21.0:
   version "3.21.0"
   resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"

modules/runner-binaries-syncer/runner-binaries-syncer.tf

@@ -18,10 +18,12 @@ resource "aws_lambda_function" "syncer" {
 
   environment {
     variables = {
-      S3_BUCKET_NAME                          = aws_s3_bucket.action_dist.id
-      S3_OBJECT_KEY                           = local.action_runner_distribution_object_key
       GITHUB_RUNNER_ARCHITECTURE              = var.runner_architecture
       GITHUB_RUNNER_ALLOW_PRERELEASE_BINARIES = var.runner_allow_prerelease_binaries
+      LOG_LEVEL                               = var.log_level
+      LOG_TYPE                                = var.log_type
+      S3_BUCKET_NAME                          = aws_s3_bucket.action_dist.id
+      S3_OBJECT_KEY                           = local.action_runner_distribution_object_key
     }
   }
   dynamic "vpc_config" {

modules/runner-binaries-syncer/variables.tf

@@ -98,3 +98,35 @@ variable "lambda_security_group_ids" {
   type        = list(string)
   default     = []
 }
+
+variable "log_type" {
+  description = "Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. "
+  type        = string
+  default     = "pretty"
+  validation {
+    condition = anytrue([
+      var.log_type == "json",
+      var.log_type == "pretty",
+      var.log_type == "hidden",
+    ])
+    error_message = "`log_type` value not valid. Valid values are 'json', 'pretty', 'hidden'."
+  }
+}
+
+variable "log_level" {
+  description = "Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'."
+  type        = string
+  default     = "info"
+  validation {
+    condition = anytrue([
+      var.log_level == "silly",
+      var.log_level == "trace",
+      var.log_level == "debug",
+      var.log_level == "info",
+      var.log_level == "warn",
+      var.log_level == "error",
+      var.log_level == "fatal",
+    ])
+    error_message = "`log_level` value not valid. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'."
+  }
+}

modules/runners/README.md

@@ -132,6 +132,8 @@ No modules.
 | <a name="input_lambda_timeout_scale_down"></a> [lambda\_timeout\_scale\_down](#input\_lambda\_timeout\_scale\_down) | Time out for the scale down lambda in seconds. | `number` | `60` | no |
 | <a name="input_lambda_timeout_scale_up"></a> [lambda\_timeout\_scale\_up](#input\_lambda\_timeout\_scale\_up) | Time out for the scale up lambda in seconds. | `number` | `60` | no |
 | <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
+| <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
+| <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `"pretty"` | no |
 | <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | <a name="input_market_options"></a> [market\_options](#input\_market\_options) | Market options for the action runner instances. | `string` | `"spot"` | no |
 | <a name="input_minimum_running_time_in_minutes"></a> [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if non busy. | `number` | `5` | no |

modules/runners/lambdas/runners/package.json

@@ -43,6 +43,7 @@
     "@types/node": "^16.11.6",
     "aws-sdk": "^2.1019.0",
     "cron-parser": "^4.1.0",
+    "tslog": "^3.2.2",
     "typescript": "^4.4.4"
   }
 }

modules/runners/lambdas/runners/src/lambda.ts

@@ -1,27 +1,31 @@
-import { scaleUp as scaleUpAction } from './scale-runners/scale-up';
-import { scaleDown as scaleDownAction } from './scale-runners/scale-down';
-import { SQSEvent, ScheduledEvent, Context } from 'aws-lambda';
+import { scaleUp } from './scale-runners/scale-up';
+import { scaleDown } from './scale-runners/scale-down';
+import { SQSEvent, ScheduledEvent, Context, Callback } from 'aws-lambda';
+import { logger } from './scale-runners/logger';
+import 'source-map-support/register';
 
-export const scaleUp = async (event: SQSEvent, context: Context, callback: any): Promise<void> => {
-  console.debug(JSON.stringify(event));
+export async function scaleUpHandler(event: SQSEvent, context: Context, callback: Callback): Promise<void> {
+  logger.setSettings({ requestId: context.awsRequestId });
+  logger.debug(JSON.stringify(event));
   try {
     for (const e of event.Records) {
-      await scaleUpAction(e.eventSource, JSON.parse(e.body));
+      await scaleUp(e.eventSource, JSON.parse(e.body));
     }
 
     callback(null);
   } catch (e) {
-    console.error(e);
+    logger.error(e);
     callback('Failed handling SQS event');
   }
-};
+}
 
-export const scaleDown = async (event: ScheduledEvent, context: Context, callback: any): Promise<void> => {
+export async function scaleDownHandler(event: ScheduledEvent, context: Context, callback: Callback): Promise<void> {
+  logger.setSettings({ requestId: context.awsRequestId });
   try {
-    await scaleDownAction();
+    await scaleDown();
     callback(null);
   } catch (e) {
-    console.error(e);
+    logger.error(e);
     callback('Failed');
   }
-};
+}

modules/runners/lambdas/runners/src/scale-runners/gh-auth.ts

@@ -11,6 +11,9 @@ import {
 } from '@octokit/auth-app/dist-types/types';
 import { OctokitOptions } from '@octokit/core/dist-types/types';
 import { getParameterValue } from './ssm';
+import { logger as rootLogger } from './logger';
+
+const logger = rootLogger.getChildLogger();
 
 export async function createOctoClient(token: string, ghesApiUrl = ''): Promise<Octokit> {
   const ocktokitOptions: OctokitOptions = {
@@ -52,6 +55,7 @@ async function createAuth(installationId: number | undefined, ghesApiUrl: string
   };
   if (installationId) authOptions = { ...authOptions, installationId };
 
+  logger.debug(`GHES API URL: ${ghesApiUrl}`);
   if (ghesApiUrl) {
     authOptions.request = request.defaults({
       baseUrl: ghesApiUrl,