feat: Create node group user data from given template

examples/managed_bottlerocket_node_group/README.md

@@ -0,0 +1,75 @@
+# AWS EKS cluster running Bottlerocket AMI
+
+Configuration in this directory creates EKS cluster with nodes group running [AWS Bottlerocket OS](https://github.com/bottlerocket-os/bottlerocket)
+
+This is a minimalistic example which shows what knobs to turn to make Bottlerocket work.
+
+See [the official documentation](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html) for more details.
+
+## Usage
+
+To run this example you need to execute:
+
+```bash
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.56.0 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | ~> 2.0 |
+| <a name="requirement_local"></a> [local](#requirement\_local) | >= 1.4 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | >= 2.1 |
+| <a name="requirement_tls"></a> [tls](#requirement\_tls) | >= 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.56.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | >= 2.1 |
+| <a name="provider_tls"></a> [tls](#provider\_tls) | >= 2.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_eks"></a> [eks](#module\_eks) | ../.. |  |
+| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_role_policy_attachment.ssm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_key_pair.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource |
+| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
+| [tls_private_key.nodes](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
+| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
+| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
+| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_cluster_endpoint"></a> [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for EKS control plane. |
+| <a name="output_cluster_security_group_id"></a> [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security group ids attached to the cluster control plane. |
+| <a name="output_config_map_aws_auth"></a> [config\_map\_aws\_auth](#output\_config\_map\_aws\_auth) | A kubernetes configuration to authenticate to this EKS cluster. |
+| <a name="output_kubectl_config"></a> [kubectl\_config](#output\_kubectl\_config) | kubectl config as generated by the module. |
+| <a name="output_node_groups"></a> [node\_groups](#output\_node\_groups) | Outputs from node groups |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/managed_bottlerocket_node_group/main.tf

@@ -0,0 +1,163 @@
+provider "aws" {
+  region = local.region
+}
+
+locals {
+  name            = "bottlerocket-${random_string.suffix.result}"
+  cluster_version = "1.20"
+  region          = "eu-west-1"
+}
+
+################################################################################
+# EKS Module
+################################################################################
+
+module "eks" {
+  source = "../.."
+
+  cluster_name    = local.name
+  cluster_version = local.cluster_version
+
+  vpc_id          = module.vpc.vpc_id
+  subnets         = [module.vpc.private_subnets[0], module.vpc.public_subnets[1]]
+  fargate_subnets = [module.vpc.private_subnets[2]]
+
+  cluster_endpoint_private_access = true
+  cluster_endpoint_public_access  = true
+
+  write_kubeconfig = false
+  manage_aws_auth  = true
+
+  node_groups = {
+    bottlerocket = {
+      name             = "bottlerocket-nodes"
+      ami_id           = data.aws_ami.bottlerocket_ami.id
+      instance_types   = ["t3a.small"]
+      desired_capacity = 2
+      key_name         = aws_key_pair.nodes.key_name
+
+      # Since we are using default VPC there is no NAT gateway so we need to
+      # attach public ip to nodes so they can reach k8s API server
+      # do not repeat this at home (i.e. production)
+      public_ip = true
+
+      # This section overrides default userdata template to pass bottlerocket
+      # specific user data and pass additional arguments for userdata template rendering.
+      # It also instructs the module to not wrap the user data as cloudinit config, by
+      # indicating that the data has no MIME type.
+      user_data_mime_type     = ""
+      user_data_template_file = "${path.module}/userdata.toml"
+      user_data_template_extra_args = {
+        enable_admin_container   = false
+        enable_control_container = true
+        aws_region               = data.aws_region.current.name
+      }
+      # example of k8s/kubelet configuration via additional_userdata
+      pre_userdata = <<EOT
+[settings.kubernetes.node-labels]
+ingress = "allowed"
+EOT
+
+      device_name = "/dev/xvdb"
+      disk_type   = "gp3"
+    }
+  }
+
+  tags = {
+    Example    = local.name
+    GithubRepo = "terraform-aws-eks"
+    GithubOrg  = "terraform-aws-modules"
+  }
+}
+
+# SSM policy for bottlerocket control container access
+# https://github.com/bottlerocket-os/bottlerocket/blob/develop/QUICKSTART-EKS.md#enabling-ssm
+resource "aws_iam_role_policy_attachment" "ssm" {
+  role       = module.eks.worker_iam_role_name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+################################################################################
+# Kubernetes provider configuration
+################################################################################
+
+data "aws_eks_cluster" "cluster" {
+  name = module.eks.cluster_id
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  name = module.eks.cluster_id
+}
+
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+}
+
+################################################################################
+# Supporting Resources
+################################################################################
+
+data "aws_region" "current" {}
+
+data "aws_ami" "bottlerocket_ami" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"]
+  }
+}
+
+resource "tls_private_key" "nodes" {
+  algorithm = "RSA"
+}
+
+resource "aws_key_pair" "nodes" {
+  key_name   = "bottlerocket-nodes-${random_string.suffix.result}"
+  public_key = tls_private_key.nodes.public_key_openssh
+}
+
+################################################################################
+# Supporting Resources
+################################################################################
+
+data "aws_availability_zones" "available" {
+}
+
+resource "random_string" "suffix" {
+  length  = 8
+  special = false
+}
+
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "~> 3.0"
+
+  name                 = local.name
+  cidr                 = "10.0.0.0/16"
+  azs                  = data.aws_availability_zones.available.names
+  private_subnets      = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+  public_subnets       = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
+  enable_nat_gateway   = true
+  single_nat_gateway   = true
+  enable_dns_hostnames = true
+
+  public_subnet_tags = {
+    "kubernetes.io/cluster/${local.name}" = "shared"
+    "kubernetes.io/role/elb"              = "1"
+  }
+
+  private_subnet_tags = {
+    "kubernetes.io/cluster/${local.name}" = "shared"
+    "kubernetes.io/role/internal-elb"     = "1"
+  }
+
+  tags = {
+    Example    = local.name
+    GithubRepo = "terraform-aws-eks"
+    GithubOrg  = "terraform-aws-modules"
+  }
+}

examples/managed_bottlerocket_node_group/outputs.tf

@@ -0,0 +1,24 @@
+output "cluster_endpoint" {
+  description = "Endpoint for EKS control plane."
+  value       = module.eks.cluster_endpoint
+}
+
+output "cluster_security_group_id" {
+  description = "Security group ids attached to the cluster control plane."
+  value       = module.eks.cluster_security_group_id
+}
+
+output "kubectl_config" {
+  description = "kubectl config as generated by the module."
+  value       = module.eks.kubeconfig
+}
+
+output "config_map_aws_auth" {
+  description = "A kubernetes configuration to authenticate to this EKS cluster."
+  value       = module.eks.config_map_aws_auth
+}
+
+output "node_groups" {
+  description = "Outputs from node groups"
+  value       = module.eks.node_groups
+}

examples/managed_bottlerocket_node_group/userdata.toml

@@ -0,0 +1,30 @@
+# https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#description-of-settings
+[settings.kubernetes]
+api-server = "${cluster_endpoint}"
+cluster-certificate = "${cluster_auth_base64}"
+cluster-name = "${cluster_name}"
+${pre_userdata}
+
+[settings.kubernetes.node-labels]
+"eks.amazonaws.com/capacityType" = "${capacity_type}"
+%{ for label, value in {for pair in [ for entry in split(",", append_labels) : split("=", entry) ] : pair[0] => pair[1] if length(pair) == 2 } ~}
+"${label}" = "${value}"
+%{ endfor ~}
+
+# Hardening based on https://github.com/bottlerocket-os/bottlerocket/blob/develop/SECURITY_GUIDANCE.md
+
+# Enable kernel lockdown in "integrity" mode.
+# This prevents modifications to the running kernel, even by privileged users.
+[settings.kernel]
+lockdown = "integrity"
+
+# The admin host container provides SSH access and runs with "superpowers".
+# It is disabled by default, but can be disabled explicitly.
+[settings.host-containers.admin]
+enabled = ${enable_admin_container}
+
+# The control host container provides out-of-band access via SSM.
+# It is enabled by default, and can be disabled if you do not expect to use SSM.
+# This could leave you with no way to access the API and change settings on an existing node!
+[settings.host-containers.control]
+enabled = ${enable_control_container}

examples/managed_bottlerocket_node_group/versions.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_version = ">= 0.13.1"
+
+  required_providers {
+    aws        = ">= 3.56.0"
+    local      = ">= 1.4"
+    random     = ">= 2.1"
+    kubernetes = "~> 2.0"
+    tls        = ">= 2.0"
+  }
+}

modules/node_groups/README.md

@@ -55,6 +55,9 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In
 | taints | Kubernetes node taints | list(map) | empty |
 | timeouts | A map of timeouts for create/update/delete operations. | `map(string)` | Provider default behavior |
 | update_default_version | Whether or not to set the new launch template version the Default | bool | `true` |
+| user\_data\_mime\_type | Alternative MIME type for the user data when specifying a cloudinit user data. Explicitly set to the empty string `""` to set the user data as a plain base64 encoded file (such as for use with the Bottlerocket AMI). | `string` | `"text/x-shellscript"` |
+| user\_data\_template\_extra\_args | Additional variables to make available to the user data template. | `map(string)` | `{}` |
+| user\_data\_template\_file | (Required) Alternative template file from which to generate the user data. | `string` | [`"${path.module}/templates/userdata.sh.tpl"`](templates/userdata.sh.tpl) |
 | metadata_http_endpoint | The state of the instance metadata service. Requires `create_launch_template` to be `true` | string | `var.workers_group_defaults[metadata_http_endpoint]` |
 | metadata_http_tokens | If session tokens are required. Requires `create_launch_template` to be `true` | string | `var.workers_group_defaults[metadata_http_tokens]` |
 | metadata_http_put_response_hop_limit | The desired HTTP PUT response hop limit for instance metadata requests. Requires `create_launch_template` to be `true` | number | `var.workers_group_defaults[metadata_http_put_response_hop_limit]` |

modules/node_groups/launch_template.tf

@@ -1,26 +1,16 @@
 data "cloudinit_config" "workers_userdata" {
-  for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] }
+  for_each = {
+    for k, v in local.node_groups_expanded : k => v
+    if v["create_launch_template"] && length(v["user_data_mime_type"]) > 0
+  }
 
   gzip          = false
   base64_encode = true
   boundary      = "//"
 
   part {
-    content_type = "text/x-shellscript"
-    content = templatefile("${path.module}/templates/userdata.sh.tpl",
-      {
-        cluster_name         = var.cluster_name
-        cluster_endpoint     = var.cluster_endpoint
-        cluster_auth_base64  = var.cluster_auth_base64
-        ami_id               = lookup(each.value, "ami_id", "")
-        ami_is_eks_optimized = each.value["ami_is_eks_optimized"]
-        bootstrap_env        = each.value["bootstrap_env"]
-        kubelet_extra_args   = each.value["kubelet_extra_args"]
-        pre_userdata         = each.value["pre_userdata"]
-        capacity_type        = lookup(each.value, "capacity_type", "ON_DEMAND")
-        append_labels        = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
-      }
-    )
+    content_type = each.value["user_data_mime_type"]
+    content      = local.node_groups_userdata[each.key]
   }
 }
 
@@ -38,7 +28,7 @@ resource "aws_launch_template" "workers" {
   update_default_version = lookup(each.value, "update_default_version", true)
 
   block_device_mappings {
-    device_name = "/dev/xvda"
+    device_name = each.value["device_name"]
 
     ebs {
       volume_size           = lookup(each.value, "disk_size", null)
@@ -81,7 +71,11 @@ resource "aws_launch_template" "workers" {
   #
   # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151)
 
-  user_data = data.cloudinit_config.workers_userdata[each.key].rendered
+  user_data = (
+    contains(keys(data.cloudinit_config.workers_userdata), each.key)
+    ? data.cloudinit_config.workers_userdata[each.key].rendered
+    : base64encode(local.node_groups_userdata[each.key])
+  )
 
   key_name = lookup(each.value, "key_name", null)