From 708d9412cebd4558c7d4eb4ba7bc4a6e92e9c1cc Mon Sep 17 00:00:00 2001
From: pjrm <4622652+pjrm@users.noreply.github.com>
Date: Sun, 12 Dec 2021 21:06:11 +0000
Subject: [PATCH] fix: Correct DNS Server of kubelet service when using custom
 AMI (#1717)

The PR (#1580) is passing the "apiserver-endpoint" and "b64-cluster-ca", which causes the SERVICE_IPV4_CIDR empty (https://github.com/awslabs/amazon-eks-ami/blob/v20211206/files/bootstrap.sh#L366). Because of that, the script fallbacks always to 10.100.0.10 or 172.20.0.10.

Defining the ipv4 cidr ensures that the bootstrap script configures the DNS server correctly on the kubelet service, allowing pods to resolve DNS names.
---
 locals.tf                                     |  1 +
 modules/node_groups/README.md                 |  1 +
 modules/node_groups/launch_template.tf        | 21 ++++++++++---------
 modules/node_groups/templates/userdata.sh.tpl |  4 ++++
 modules/node_groups/variables.tf              |  6 ++++++
 node_groups.tf                                |  7 ++++---
 6 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/locals.tf b/locals.tf
index 8e1b54e49cb..d37fe15422c 100644
--- a/locals.tf
+++ b/locals.tf
@@ -6,6 +6,7 @@ locals {
   cluster_name                      = coalescelist(aws_eks_cluster.this[*].name, [""])[0]
   cluster_endpoint                  = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0]
   cluster_auth_base64               = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]
+  cluster_service_ipv4_cidr         = coalescelist(aws_eks_cluster.this[*].kubernetes_network_config[0].service_ipv4_cidr, [""])[0]
   cluster_oidc_issuer_url           = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc[0].issuer, [""]))[0]
   cluster_primary_security_group_id = coalescelist(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, [""])[0]
 
diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md
index f91ce04ae0f..74a0a8af3ce 100644
--- a/modules/node_groups/README.md
+++ b/modules/node_groups/README.md
@@ -94,6 +94,7 @@ No modules.
 | <a name="input_cluster_auth_base64"></a> [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of parent cluster | `string` | `""` | no |
 | <a name="input_cluster_endpoint"></a> [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of parent cluster | `string` | `""` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | `""` | no |
+| <a name="input_cluster_service_ipv4_cidr"></a> [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no |
 | <a name="input_create_eks"></a> [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no |
 | <a name="input_default_iam_role_arn"></a> [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | `""` | no |
 | <a name="input_ebs_optimized_not_supported"></a> [ebs\_optimized\_not\_supported](#input\_ebs\_optimized\_not\_supported) | List of instance types that do not support EBS optimization | `list(string)` | `[]` | no |
diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf
index 6abe358d5ae..d9dd4dc296b 100644
--- a/modules/node_groups/launch_template.tf
+++ b/modules/node_groups/launch_template.tf
@@ -9,16 +9,17 @@ data "cloudinit_config" "workers_userdata" {
     content_type = "text/x-shellscript"
     content = templatefile("${path.module}/templates/userdata.sh.tpl",
       {
-        cluster_name         = var.cluster_name
-        cluster_endpoint     = var.cluster_endpoint
-        cluster_auth_base64  = var.cluster_auth_base64
-        ami_id               = lookup(each.value, "ami_id", "")
-        ami_is_eks_optimized = each.value["ami_is_eks_optimized"]
-        bootstrap_env        = each.value["bootstrap_env"]
-        kubelet_extra_args   = each.value["kubelet_extra_args"]
-        pre_userdata         = each.value["pre_userdata"]
-        capacity_type        = lookup(each.value, "capacity_type", "ON_DEMAND")
-        append_labels        = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
+        cluster_name              = var.cluster_name
+        cluster_endpoint          = var.cluster_endpoint
+        cluster_auth_base64       = var.cluster_auth_base64
+        cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr
+        ami_id                    = lookup(each.value, "ami_id", "")
+        ami_is_eks_optimized      = each.value["ami_is_eks_optimized"]
+        bootstrap_env             = each.value["bootstrap_env"]
+        kubelet_extra_args        = each.value["kubelet_extra_args"]
+        pre_userdata              = each.value["pre_userdata"]
+        capacity_type             = lookup(each.value, "capacity_type", "ON_DEMAND")
+        append_labels             = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
       }
     )
   }
diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl
index 321c17b4275..1a4c293079c 100644
--- a/modules/node_groups/templates/userdata.sh.tpl
+++ b/modules/node_groups/templates/userdata.sh.tpl
@@ -29,6 +29,10 @@ KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},ek
 ${pre_userdata}
 %{ if length(ami_id) > 0 && ami_is_eks_optimized ~}
 
+# The bootstrap.sh script doesn't allow to pass service ipv4 cidr as an argument.
+# Therefore the environment variable SERVICE_IPV4_CIDR is exported to be used by the script.
+export SERVICE_IPV4_CIDR=${cluster_service_ipv4_cidr}
+
 # Call bootstrap for EKS optimised custom AMI
 /etc/eks/bootstrap.sh ${cluster_name} --apiserver-endpoint "$${API_SERVER_URL}" --b64-cluster-ca "$${B64_CLUSTER_CA}" --kubelet-extra-args "$${KUBELET_EXTRA_ARGS}"
 %{ endif ~}
diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf
index 1aa8cfe26d8..87ee5c407c3 100644
--- a/modules/node_groups/variables.tf
+++ b/modules/node_groups/variables.tf
@@ -22,6 +22,12 @@ variable "cluster_auth_base64" {
   default     = ""
 }
 
+variable "cluster_service_ipv4_cidr" {
+  description = "service ipv4 cidr for the kubernetes cluster"
+  type        = string
+  default     = null
+}
+
 variable "default_iam_role_arn" {
   description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`"
   type        = string
diff --git a/node_groups.tf b/node_groups.tf
index 531a3df480c..85ac208c87f 100644
--- a/node_groups.tf
+++ b/node_groups.tf
@@ -3,9 +3,10 @@ module "node_groups" {
 
   create_eks = var.create_eks
 
-  cluster_name        = local.cluster_name
-  cluster_endpoint    = local.cluster_endpoint
-  cluster_auth_base64 = local.cluster_auth_base64
+  cluster_name              = local.cluster_name
+  cluster_endpoint          = local.cluster_endpoint
+  cluster_auth_base64       = local.cluster_auth_base64
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
 
   default_iam_role_arn                 = coalescelist(aws_iam_role.workers[*].arn, [""])[0]
   ebs_optimized_not_supported          = local.ebs_optimized_not_supported