From a14b3f69e1ca9b69c611d9410136dab74feb1013 Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Tue, 16 Nov 2021 22:35:02 -0300 Subject: [PATCH 01/10] chore: add bottlerocket support for node groups --- modules/node_groups/launch_template.tf | 28 +++++++++++++++++-- .../node_groups/templates/userdata.toml.tpl | 24 ++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 modules/node_groups/templates/userdata.toml.tpl diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 6abe358d5a..e7299077fc 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -1,5 +1,5 @@ data "cloudinit_config" "workers_userdata" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] } + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && v["ami_type"] != "BOTTLEROCKET_x86_64" } gzip = false base64_encode = true @@ -24,6 +24,30 @@ data "cloudinit_config" "workers_userdata" { } } +data "cloudinit_config" "bottlerocket_workers_userdata" { + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && v["ami_type"] == "BOTTLEROCKET_x86_64" } + + gzip = false + base64_encode = true + boundary = "//" + + part { + content_type = "text/x-shellscript" + content = templatefile("${path.module}/templates/userdata.toml", + { + cluster_name = var.cluster_name + endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + enable_admin_container = lookup(each.value, "enable_admin_container", false) + enable_control_container = lookup(each.value, "enable_control_container", true) + additional_userdata = lookup(each.value, "additional_userdata", "") + # capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") + # append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" + } + ) + } +} + # This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) # there are several more options one could set but you probably dont need to modify them # you can take the default and add your custom AMI and/or custom tags @@ -81,7 +105,7 @@ resource "aws_launch_template" "workers" { # # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - user_data = data.cloudinit_config.workers_userdata[each.key].rendered + user_data = each.value["ami_type"] == "BOTTLEROCKET_x86_64" ? data.cloudinit_config.bottlerocket_workers_userdata[each.key].rendered : data.cloudinit_config.workers_userdata[each.key].rendered key_name = lookup(each.value, "key_name", null) diff --git a/modules/node_groups/templates/userdata.toml.tpl b/modules/node_groups/templates/userdata.toml.tpl new file mode 100644 index 0000000000..85019675a6 --- /dev/null +++ b/modules/node_groups/templates/userdata.toml.tpl @@ -0,0 +1,24 @@ +# https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#description-of-settings +[settings.kubernetes] +api-server = "${endpoint}" +cluster-certificate = "${cluster_auth_base64}" +cluster-name = "${cluster_name}" +${additional_userdata} + +# Hardening based on https://github.com/bottlerocket-os/bottlerocket/blob/develop/SECURITY_GUIDANCE.md + +# Enable kernel lockdown in "integrity" mode. +# This prevents modifications to the running kernel, even by privileged users. +[settings.kernel] +lockdown = "integrity" + +# The admin host container provides SSH access and runs with "superpowers". +# It is disabled by default, but can be disabled explicitly. +[settings.host-containers.admin] +enabled = ${enable_admin_container} + +# The control host container provides out-of-band access via SSM. +# It is enabled by default, and can be disabled if you do not expect to use SSM. +# This could leave you with no way to access the API and change settings on an existing node! +[settings.host-containers.control] +enabled = ${enable_control_container} From 964da6a85ed99e1402ff672cef1dd27b9e9d2e09 Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Tue, 16 Nov 2021 22:41:57 -0300 Subject: [PATCH 02/10] fix: template extension --- modules/node_groups/launch_template.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index e7299077fc..d76fdff848 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -33,7 +33,7 @@ data "cloudinit_config" "bottlerocket_workers_userdata" { part { content_type = "text/x-shellscript" - content = templatefile("${path.module}/templates/userdata.toml", + content = templatefile("${path.module}/templates/userdata.toml.tpl", { cluster_name = var.cluster_name endpoint = var.cluster_endpoint From b659c89c717783caabaa4425755f8c625da4b94f Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Tue, 16 Nov 2021 22:52:14 -0300 Subject: [PATCH 03/10] chore: change content_type --- modules/node_groups/launch_template.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index d76fdff848..7a6d6a0338 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -29,10 +29,9 @@ data "cloudinit_config" "bottlerocket_workers_userdata" { gzip = false base64_encode = true - boundary = "//" part { - content_type = "text/x-shellscript" + content_type = "application/toml" content = templatefile("${path.module}/templates/userdata.toml.tpl", { cluster_name = var.cluster_name From 1409e70727fd86a6a0491a65ee195498b606f5f6 Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Tue, 16 Nov 2021 22:59:59 -0300 Subject: [PATCH 04/10] chore: use template_file instead --- modules/node_groups/launch_template.tf | 31 ++++++++++---------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 7a6d6a0338..3c6fc873bf 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -24,26 +24,19 @@ data "cloudinit_config" "workers_userdata" { } } -data "cloudinit_config" "bottlerocket_workers_userdata" { +data "template_file" "bottlerocket_workers_userdata" { for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && v["ami_type"] == "BOTTLEROCKET_x86_64" } - gzip = false - base64_encode = true - - part { - content_type = "application/toml" - content = templatefile("${path.module}/templates/userdata.toml.tpl", - { - cluster_name = var.cluster_name - endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - enable_admin_container = lookup(each.value, "enable_admin_container", false) - enable_control_container = lookup(each.value, "enable_control_container", true) - additional_userdata = lookup(each.value, "additional_userdata", "") - # capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") - # append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" - } - ) + template = templatefile("${path.module}/templates/userdata.toml.tpl") + vars = { + cluster_name = var.cluster_name + endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + enable_admin_container = lookup(each.value, "enable_admin_container", false) + enable_control_container = lookup(each.value, "enable_control_container", true) + additional_userdata = lookup(each.value, "additional_userdata", "") + # capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") + # append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } } @@ -104,7 +97,7 @@ resource "aws_launch_template" "workers" { # # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - user_data = each.value["ami_type"] == "BOTTLEROCKET_x86_64" ? data.cloudinit_config.bottlerocket_workers_userdata[each.key].rendered : data.cloudinit_config.workers_userdata[each.key].rendered + user_data = each.value["ami_type"] == "BOTTLEROCKET_x86_64" ? base64encode(data.template_file.bottlerocket_workers_userdata[each.key].rendered) : data.cloudinit_config.workers_userdata[each.key].rendered key_name = lookup(each.value, "key_name", null) From 64b0e969a56e8dda3f2d3d5860ee369fa67b5bb4 Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Tue, 16 Nov 2021 23:03:59 -0300 Subject: [PATCH 05/10] fix: use file function --- modules/node_groups/launch_template.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 3c6fc873bf..e3c318de4a 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -27,7 +27,7 @@ data "cloudinit_config" "workers_userdata" { data "template_file" "bottlerocket_workers_userdata" { for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && v["ami_type"] == "BOTTLEROCKET_x86_64" } - template = templatefile("${path.module}/templates/userdata.toml.tpl") + template = file("${path.module}/templates/userdata.toml.tpl") vars = { cluster_name = var.cluster_name endpoint = var.cluster_endpoint From aa364431e5f499c3f730e6c5f101d3ba1439d874 Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Wed, 17 Nov 2021 10:30:57 -0300 Subject: [PATCH 06/10] chore: use BOTTLEROCKET as prefix --- modules/node_groups/launch_template.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index e3c318de4a..6edbdf63a4 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -1,5 +1,5 @@ data "cloudinit_config" "workers_userdata" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && v["ami_type"] != "BOTTLEROCKET_x86_64" } + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) == 0 } gzip = false base64_encode = true @@ -25,7 +25,7 @@ data "cloudinit_config" "workers_userdata" { } data "template_file" "bottlerocket_workers_userdata" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && v["ami_type"] == "BOTTLEROCKET_x86_64" } + for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) > 1 } template = file("${path.module}/templates/userdata.toml.tpl") vars = { @@ -97,7 +97,7 @@ resource "aws_launch_template" "workers" { # # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - user_data = each.value["ami_type"] == "BOTTLEROCKET_x86_64" ? base64encode(data.template_file.bottlerocket_workers_userdata[each.key].rendered) : data.cloudinit_config.workers_userdata[each.key].rendered + user_data = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? base64encode(data.template_file.bottlerocket_workers_userdata[each.key].rendered) : data.cloudinit_config.workers_userdata[each.key].rendered key_name = lookup(each.value, "key_name", null) From c9a5d58ddfed7c04c25419e7edb901605f07ac5a Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Mon, 22 Nov 2021 10:01:43 -0300 Subject: [PATCH 07/10] feat: manage xvdb if BOTTLERCKET* --- modules/node_groups/launch_template.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 6edbdf63a4..304c44ebfd 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -35,8 +35,6 @@ data "template_file" "bottlerocket_workers_userdata" { enable_admin_container = lookup(each.value, "enable_admin_container", false) enable_control_container = lookup(each.value, "enable_control_container", true) additional_userdata = lookup(each.value, "additional_userdata", "") - # capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") - # append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } } @@ -54,7 +52,7 @@ resource "aws_launch_template" "workers" { update_default_version = lookup(each.value, "update_default_version", true) block_device_mappings { - device_name = "/dev/xvda" + device_name = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? "/dev/xvdb" : "/dev/xvda" ebs { volume_size = lookup(each.value, "disk_size", null) From 6787a7f126ceb208f8c0c45f26594b369048dd0c Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Mon, 22 Nov 2021 10:54:35 -0300 Subject: [PATCH 08/10] feat: templatefile from local --- modules/node_groups/launch_template.tf | 29 +++++++++++++------------- modules/node_groups/locals.tf | 11 ++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 304c44ebfd..4cc330301c 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -24,19 +24,19 @@ data "cloudinit_config" "workers_userdata" { } } -data "template_file" "bottlerocket_workers_userdata" { - for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) > 1 } - - template = file("${path.module}/templates/userdata.toml.tpl") - vars = { - cluster_name = var.cluster_name - endpoint = var.cluster_endpoint - cluster_auth_base64 = var.cluster_auth_base64 - enable_admin_container = lookup(each.value, "enable_admin_container", false) - enable_control_container = lookup(each.value, "enable_control_container", true) - additional_userdata = lookup(each.value, "additional_userdata", "") - } -} +# data "template_file" "bottlerocket_workers_userdata" { +# for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) > 1 } +# +# template = file("${path.module}/templates/userdata.toml.tpl") +# vars = { +# cluster_name = var.cluster_name +# endpoint = var.cluster_endpoint +# cluster_auth_base64 = var.cluster_auth_base64 +# enable_admin_container = lookup(each.value, "enable_admin_container", false) +# enable_control_container = lookup(each.value, "enable_control_container", true) +# additional_userdata = lookup(each.value, "additional_userdata", "") +# } +# } # This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) # there are several more options one could set but you probably dont need to modify them @@ -95,7 +95,8 @@ resource "aws_launch_template" "workers" { # # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - user_data = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? base64encode(data.template_file.bottlerocket_workers_userdata[each.key].rendered) : data.cloudinit_config.workers_userdata[each.key].rendered + # user_data = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? base64encode(data.template_file.bottlerocket_workers_userdata[each.key].rendered) : data.cloudinit_config.workers_userdata[each.key].rendered + user_data = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? base64encode(local.bottlerocket_workers_userdata[each.key]) : data.cloudinit_config.workers_userdata[each.key].rendered key_name = lookup(each.value, "key_name", null) diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 0a6c7cbffb..849e52d19f 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -48,4 +48,15 @@ locals { join("-", [var.cluster_name, k]) ) ) } + + bottlerocket_workers_userdata = { + for k, v in local.node_groups_expanded : k => templatefile(format("%s/templates/userdata.toml.tpl", path.module), { + cluster_name = var.cluster_name + endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + enable_admin_container = lookup(each.value, "enable_admin_container", false) + enable_control_container = lookup(each.value, "enable_control_container", true) + additional_userdata = lookup(each.value, "additional_userdata", "") + }) if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) > 1 + } } From 716622e026f0cc017cadaaf76a1d32b167ec296d Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Mon, 22 Nov 2021 11:20:26 -0300 Subject: [PATCH 09/10] fix: use object.* instead of each.value.* --- modules/node_groups/launch_template.tf | 14 -------------- modules/node_groups/locals.tf | 6 +++--- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index 4cc330301c..a36e5f3ac7 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -24,20 +24,6 @@ data "cloudinit_config" "workers_userdata" { } } -# data "template_file" "bottlerocket_workers_userdata" { -# for_each = { for k, v in local.node_groups_expanded : k => v if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) > 1 } -# -# template = file("${path.module}/templates/userdata.toml.tpl") -# vars = { -# cluster_name = var.cluster_name -# endpoint = var.cluster_endpoint -# cluster_auth_base64 = var.cluster_auth_base64 -# enable_admin_container = lookup(each.value, "enable_admin_container", false) -# enable_control_container = lookup(each.value, "enable_control_container", true) -# additional_userdata = lookup(each.value, "additional_userdata", "") -# } -# } - # This is based on the LT that EKS would create if no custom one is specified (aws ec2 describe-launch-template-versions --launch-template-id xxx) # there are several more options one could set but you probably dont need to modify them # you can take the default and add your custom AMI and/or custom tags diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 849e52d19f..b88561318d 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -54,9 +54,9 @@ locals { cluster_name = var.cluster_name endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 - enable_admin_container = lookup(each.value, "enable_admin_container", false) - enable_control_container = lookup(each.value, "enable_control_container", true) - additional_userdata = lookup(each.value, "additional_userdata", "") + enable_admin_container = lookup(v, "enable_admin_container", false) + enable_control_container = lookup(v, "enable_control_container", true) + additional_userdata = lookup(v, "additional_userdata", "") }) if v["create_launch_template"] && length(split("BOTTLEROCKET", v["ami_type"])) > 1 } } From 5e64726866fba69f5b8af81493a4bd7410fac6ae Mon Sep 17 00:00:00 2001 From: Pierre Ugaz Date: Mon, 22 Nov 2021 11:24:01 -0300 Subject: [PATCH 10/10] chore: remove unused line --- modules/node_groups/launch_template.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index a36e5f3ac7..742b8d9178 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -81,7 +81,6 @@ resource "aws_launch_template" "workers" { # # (optionally you can use https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config to render the script, example: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/997#issuecomment-705286151) - # user_data = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? base64encode(data.template_file.bottlerocket_workers_userdata[each.key].rendered) : data.cloudinit_config.workers_userdata[each.key].rendered user_data = length(split("BOTTLEROCKET", each.value["ami_type"])) > 1 ? base64encode(local.bottlerocket_workers_userdata[each.key]) : data.cloudinit_config.workers_userdata[each.key].rendered key_name = lookup(each.value, "key_name", null)