Launch Template module with Karpenter updates

README.md

@@ -234,10 +234,13 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 | <a name="output_fargate_profiles_aws_auth_config_map"></a> [fargate\_profiles\_aws\_auth\_config\_map](#output\_fargate\_profiles\_aws\_auth\_config\_map) | Fargate profiles AWS auth map |
 | <a name="output_fargate_profiles_iam_role_arns"></a> [fargate\_profiles\_iam\_role\_arns](#output\_fargate\_profiles\_iam\_role\_arns) | IAM role arn's for Fargate Profiles |
 | <a name="output_managed_node_group_aws_auth_config_map"></a> [managed\_node\_group\_aws\_auth\_config\_map](#output\_managed\_node\_group\_aws\_auth\_config\_map) | Managed node groups AWS auth map |
+| <a name="output_managed_node_group_iam_instance_profile_arns"></a> [managed\_node\_group\_iam\_instance\_profile\_arns](#output\_managed\_node\_group\_iam\_instance\_profile\_arns) | IAM instance profile arn's of managed node groups |
+| <a name="output_managed_node_group_iam_instance_profile_id"></a> [managed\_node\_group\_iam\_instance\_profile\_id](#output\_managed\_node\_group\_iam\_instance\_profile\_id) | IAM instance profile id of managed node groups |
 | <a name="output_managed_node_group_iam_role_arns"></a> [managed\_node\_group\_iam\_role\_arns](#output\_managed\_node\_group\_iam\_role\_arns) | IAM role arn's of managed node groups |
 | <a name="output_managed_node_groups"></a> [managed\_node\_groups](#output\_managed\_node\_groups) | Outputs from EKS Managed node groups |
 | <a name="output_self_managed_node_group_autoscaling_groups"></a> [self\_managed\_node\_group\_autoscaling\_groups](#output\_self\_managed\_node\_group\_autoscaling\_groups) | Autoscaling group names of self managed node groups |
 | <a name="output_self_managed_node_group_aws_auth_config_map"></a> [self\_managed\_node\_group\_aws\_auth\_config\_map](#output\_self\_managed\_node\_group\_aws\_auth\_config\_map) | Self managed node groups AWS auth map |
+| <a name="output_self_managed_node_group_iam_instance_profile_id"></a> [self\_managed\_node\_group\_iam\_instance\_profile\_id](#output\_self\_managed\_node\_group\_iam\_instance\_profile\_id) | IAM instance profile id of managed node groups |
 | <a name="output_self_managed_node_group_iam_role_arns"></a> [self\_managed\_node\_group\_iam\_role\_arns](#output\_self\_managed\_node\_group\_iam\_role\_arns) | IAM role arn's of self managed node groups |
 | <a name="output_self_managed_node_groups"></a> [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Outputs from EKS Self-managed node groups |
 | <a name="output_teams"></a> [teams](#output\_teams) | Outputs from EKS Fargate profiles groups |

eks-worker.tf

@@ -38,10 +38,11 @@ module "aws_eks_managed_node_groups" {
   private_subnet_ids = var.private_subnet_ids
   public_subnet_ids  = var.public_subnet_ids
 
-  worker_security_group_id          = module.aws_eks.worker_security_group_id
-  cluster_security_group_id         = module.aws_eks.cluster_security_group_id
-  cluster_primary_security_group_id = module.aws_eks.cluster_primary_security_group_id
-  tags                              = module.eks_tags.tags
+  worker_security_group_id             = module.aws_eks.worker_security_group_id
+  worker_additional_security_group_ids = var.worker_additional_security_group_ids
+  cluster_security_group_id            = module.aws_eks.cluster_security_group_id
+  cluster_primary_security_group_id    = module.aws_eks.cluster_primary_security_group_id
+  tags                                 = module.eks_tags.tags
 
   depends_on = [module.aws_eks, kubernetes_config_map.aws_auth]
 }
@@ -68,10 +69,11 @@ module "aws_eks_self_managed_node_groups" {
   private_subnet_ids = var.private_subnet_ids
   public_subnet_ids  = var.public_subnet_ids
 
-  worker_security_group_id          = module.aws_eks.worker_security_group_id
-  cluster_security_group_id         = module.aws_eks.cluster_security_group_id
-  cluster_primary_security_group_id = module.aws_eks.cluster_primary_security_group_id
-  tags                              = module.eks_tags.tags
+  worker_security_group_id             = module.aws_eks.worker_security_group_id
+  worker_additional_security_group_ids = var.worker_additional_security_group_ids
+  cluster_security_group_id            = module.aws_eks.cluster_security_group_id
+  cluster_primary_security_group_id    = module.aws_eks.cluster_primary_security_group_id
+  tags                                 = module.eks_tags.tags
 
   depends_on = [module.aws_eks, kubernetes_config_map.aws_auth]
 }

examples/eks-cluster-with-karpenter/README.md → examples/karpenter/README.md

@@ -112,6 +112,7 @@ terraform destroy
 |------|--------|---------|
 | <a name="module_aws-eks-accelerator-for-terraform"></a> [aws-eks-accelerator-for-terraform](#module\_aws-eks-accelerator-for-terraform) | ../.. | n/a |
 | <a name="module_aws_vpc"></a> [aws\_vpc](#module\_aws\_vpc) | terraform-aws-modules/vpc/aws | v3.2.0 |
+| <a name="module_karpenter-launch-templates"></a> [karpenter-launch-templates](#module\_karpenter-launch-templates) | ../../modules/launch-templates | n/a |
 | <a name="module_kubernetes-addons"></a> [kubernetes-addons](#module\_kubernetes-addons) | ../../modules/kubernetes-addons | n/a |
 
 ## Resources

examples/eks-cluster-with-karpenter/main.tf → examples/karpenter/main.tf

@@ -125,12 +125,48 @@ module "aws-eks-accelerator-for-terraform" {
     }
   }
 }
+# Creates Launch templates for Karpenter
+# Launch template outputs will be used in Karpenter Provisioners yaml files
+module "karpenter-launch-templates" {
+  source         = "../../modules/launch-templates"
+  eks_cluster_id = module.aws-eks-accelerator-for-terraform.eks_cluster_id
+  tags           = { Name = "karpenter" }
+
+  launch_template_config = {
+    linux = {
+      ami                    = "ami-0adc757be1e4e11a1"
+      launch_template_prefix = "karpenter"
+      iam_instance_profile   = module.aws-eks-accelerator-for-terraform.self_managed_node_group_iam_instance_profile_id[0]
+      vpc_security_group_ids = module.aws-eks-accelerator-for-terraform.worker_security_group_id
+      block_device_mappings = [
+        {
+          device_name = "/dev/xvda"
+          volume_type = "gp2"
+          volume_size = "200"
+        }
+      ]
+    },
+    bottlerocket = {
+      ami                    = "ami-03909df9bfcc1e215"
+      launch_template_os     = "bottlerocket"
+      launch_template_prefix = "bottle"
+      iam_instance_profile   = module.aws-eks-accelerator-for-terraform.self_managed_node_group_iam_instance_profile_id[0]
+      vpc_security_group_ids = module.aws-eks-accelerator-for-terraform.worker_security_group_id
+      block_device_mappings = [
+        {
+          device_name = "/dev/xvda"
+          volume_type = "gp2"
+          volume_size = "200"
+        }
+      ]
+    },
+  }
+}
 
 module "kubernetes-addons" {
   source = "../../modules/kubernetes-addons"
 
   eks_cluster_id = module.aws-eks-accelerator-for-terraform.eks_cluster_id
-
   #K8s Add-ons
   enable_karpenter      = true
   enable_metrics_server = true

examples/karpenter/provisioners/default_provisioner_with_launch_templates.yaml

@@ -0,0 +1,28 @@
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: default
+spec:
+  requirements:
+    - key: "topology.kubernetes.io/zone"
+      operator: In
+      values: ["eu-west-1b"]                                  # Update the correct region and zones
+    - key: "karpenter.sh/capacity-type"
+      operator: In
+      values: ["spot", "on-demand"]
+    - key: "node.kubernetes.io/instance-type"                  #If not included, all instance types are considered
+      operator: In
+      values: ["m5.2xlarge", "m5.4xlarge"]
+    - key: "kubernetes.io/arch"                               #If not included, all architectures are considered
+      operator: In
+      values: ["arm64", "amd64"]
+  limits:
+    resources:
+      cpu: 1000
+  provider:
+    launchTemplate: "karpenter-aws001-preprod-dev-eks"                     # Used by Karpenter Nodes
+    subnetSelector:
+      Name: "*private*"
+    securityGroupSelector:
+      kubernetes.io/cluster/aws-dev-spark-eks: owned
+  ttlSecondsAfterEmpty: 120

examples/eks-cluster-with-karpenter/provisioners/gpu_provisioner.yaml → examples/karpenter/provisioners/gpu_provisioner.yaml

@@ -3,7 +3,6 @@ kind: Provisioner
 metadata:
   name: gpu
 spec:
-  ttlSecondsAfterEmpty: 60
   requirements:
     - key: node.kubernetes.io/instance-type
       operator: In
@@ -14,4 +13,4 @@ spec:
       effect: “NoSchedule”
   provider:
     instanceProfile: "aws001-preprod-dev-eks-self-managed-ondemand" # self-managed IAM Instance profile Name
-  ttlSecondsAfterEmpty: 30
+  ttlSecondsAfterEmpty: 60

main.tf

@@ -62,9 +62,8 @@ module "aws_eks" {
   cluster_endpoint_private_access = var.cluster_endpoint_private_access
   cluster_endpoint_public_access  = var.cluster_endpoint_public_access
 
-  worker_create_security_group         = var.worker_create_security_group
-  worker_additional_security_group_ids = var.worker_additional_security_group_ids
-  cluster_log_retention_in_days        = var.cluster_log_retention_in_days
+  worker_create_security_group  = var.worker_create_security_group
+  cluster_log_retention_in_days = var.cluster_log_retention_in_days
 
   # IRSA
   enable_irsa = var.enable_irsa

modules/aws-eks-fargate-profiles/README.md

@@ -28,7 +28,6 @@ The Fargate profile allows you to declare which pods run on Fargate for Amazon E
         ExtraTag = "Fargate"
       }
     },
-    /*
     multi = {
       fargate_profile_name = "multi-namespaces"
       fargate_profile_namespaces = [{
@@ -57,7 +56,7 @@ The Fargate profile allows you to declare which pods run on Fargate for Amazon E
       additional_tags = {
         ExtraTag = "Fargate"
       }
-    }, */
+    },
   } # END OF FARGATE PROFILES
 ```
 

modules/aws-eks-managed-node-groups/README.md

@@ -206,17 +206,20 @@ No modules.
 | <a name="input_public_subnet_ids"></a> [public\_subnet\_ids](#input\_public\_subnet\_ids) | list of public subnets Id's for the Worker nodes | `list` | `[]` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC Id used in security group creation | `string` | `""` | no |
+| <a name="input_worker_additional_security_group_ids"></a> [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | Worker additional group security IDs | `list(string)` | `[]` | no |
 | <a name="input_worker_security_group_id"></a> [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | Worker group security ID | `string` | `""` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
+| <a name="output_managed_nodegroup_iam_instance_profile_arn"></a> [managed\_nodegroup\_iam\_instance\_profile\_arn](#output\_managed\_nodegroup\_iam\_instance\_profile\_arn) | IAM instance profile arn for EKS Managed Node Group |
+| <a name="output_managed_nodegroup_iam_instance_profile_id"></a> [managed\_nodegroup\_iam\_instance\_profile\_id](#output\_managed\_nodegroup\_iam\_instance\_profile\_id) | IAM instance profile id for EKS Managed Node Group |
 | <a name="output_managed_nodegroup_iam_role_arn"></a> [managed\_nodegroup\_iam\_role\_arn](#output\_managed\_nodegroup\_iam\_role\_arn) | IAM role ARN for EKS Managed Node Group |
-| <a name="output_managed_nodegroup_iam_role_name"></a> [managed\_nodegroup\_iam\_role\_name](#output\_managed\_nodegroup\_iam\_role\_name) | IAM role Names for EKS Managed Node Group |
-| <a name="output_managed_nodegroup_launch_template_arn"></a> [managed\_nodegroup\_launch\_template\_arn](#output\_managed\_nodegroup\_launch\_template\_arn) | launch templated id for EKS Managed Node Group |
-| <a name="output_managed_nodegroup_launch_template_id"></a> [managed\_nodegroup\_launch\_template\_id](#output\_managed\_nodegroup\_launch\_template\_id) | launch templated id for EKS Managed Node Group |
-| <a name="output_managed_nodegroup_launch_template_latest_version"></a> [managed\_nodegroup\_launch\_template\_latest\_version](#output\_managed\_nodegroup\_launch\_template\_latest\_version) | launch templated version for EKS Managed Node Group |
+| <a name="output_managed_nodegroup_iam_role_name"></a> [managed\_nodegroup\_iam\_role\_name](#output\_managed\_nodegroup\_iam\_role\_name) | IAM role name for EKS Managed Node Group |
+| <a name="output_managed_nodegroup_launch_template_arn"></a> [managed\_nodegroup\_launch\_template\_arn](#output\_managed\_nodegroup\_launch\_template\_arn) | Launch Template ARN for EKS Managed Node Group |
+| <a name="output_managed_nodegroup_launch_template_id"></a> [managed\_nodegroup\_launch\_template\_id](#output\_managed\_nodegroup\_launch\_template\_id) | Launch Template ID for EKS Managed Node Group |
+| <a name="output_managed_nodegroup_launch_template_latest_version"></a> [managed\_nodegroup\_launch\_template\_latest\_version](#output\_managed\_nodegroup\_launch\_template\_latest\_version) | Launch Template version for EKS Managed Node Group |
 | <a name="output_managed_nodegroup_sec_group_id"></a> [managed\_nodegroup\_sec\_group\_id](#output\_managed\_nodegroup\_sec\_group\_id) | Security group ID for Managed node group |
 | <a name="output_managed_nodegroups"></a> [managed\_nodegroups](#output\_managed\_nodegroups) | EKS Managed node group id |
 

modules/aws-eks-managed-node-groups/locals.tf

@@ -4,8 +4,8 @@ locals {
     node_group_name = "m4_on_demand" # Max node group length is 40 characters; including the node_group_name_prefix random id it's 63
     instance_types  = ["m4.large"]
     capacity_type   = "ON_DEMAND"  # ON_DEMAND, SPOT
-    ami_type        = "AL2_x86_64" # AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, CUSTOM
-    custom_ami_id   = ""           # Used only with Bottlerocket with custom AMI id
+    ami_type        = "AL2_x86_64" # AL2_x86_64, AL2_x86_64_GPU, AL2_ARM_64, BOTTLEROCKET_x86_64, BOTTLEROCKET_ARM_64
+    custom_ami_id   = ""
     subnet_ids      = []
 
     desired_size    = "3"
@@ -24,17 +24,18 @@ locals {
     k8s_taints      = []
     additional_tags = {}
 
-    create_worker_security_group = false
+    create_worker_security_group         = false
+    worker_additional_security_group_ids = [] # Will use this when create_worker_security_group = true
 
     # LAUNCH TEMPLATES
     create_launch_template  = false
     launch_template_os      = "amazonlinux2eks" # amazonlinux2eks/bottlerocket # Used to identify the launch template
     pre_userdata            = ""
     post_userdata           = ""
-    launch_template_id      = null
-    launch_template_version = "$Latest"
     kubelet_extra_args      = ""
     bootstrap_extra_args    = ""
+    launch_template_id      = null
+    launch_template_version = "$Latest"
 
     # SSH ACCESS
     remote_access           = false
@@ -69,8 +70,8 @@ locals {
     "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy",
     "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly",
     "${local.policy_arn_prefix}/AmazonSSMManagedInstanceCore"],
-    local.managed_node_group["additional_iam_policies"
-  ]))
+    local.managed_node_group["additional_iam_policies"]
+  ))
 
   common_tags = merge(
     var.tags,

modules/aws-eks-managed-node-groups/managed-launch-templates.tf

@@ -40,7 +40,7 @@ resource "aws_launch_template" "managed_node_groups" {
 
   network_interfaces {
     associate_public_ip_address = local.managed_node_group["public_ip"]
-    security_groups             = local.managed_node_group["create_worker_security_group"] == true ? [aws_security_group.managed_ng[0].id] : [var.worker_security_group_id]
+    security_groups             = local.managed_node_group["create_worker_security_group"] == true ? compact(flatten([[aws_security_group.managed_ng[0].id], local.managed_node_group["worker_additional_security_group_ids"]])) : compact(flatten([[var.worker_security_group_id], var.worker_additional_security_group_ids]))
   }
 
   lifecycle {

modules/aws-eks-managed-node-groups/outputs.tf

@@ -10,22 +10,32 @@ output "managed_nodegroup_iam_role_arn" {
 }
 
 output "managed_nodegroup_iam_role_name" {
-  description = "IAM role Names for EKS Managed Node Group"
+  description = "IAM role name for EKS Managed Node Group"
   value       = aws_iam_role.managed_ng[*].name
 }
 
+output "managed_nodegroup_iam_instance_profile_id" {
+  description = "IAM instance profile id for EKS Managed Node Group"
+  value       = aws_iam_instance_profile.managed_ng[*].id
+}
+
+output "managed_nodegroup_iam_instance_profile_arn" {
+  description = "IAM instance profile arn for EKS Managed Node Group"
+  value       = aws_iam_instance_profile.managed_ng[*].arn
+}
+
 output "managed_nodegroup_launch_template_id" {
-  description = "launch templated id for EKS Managed Node Group"
+  description = "Launch Template ID for EKS Managed Node Group"
   value       = aws_launch_template.managed_node_groups[*].id
 }
 
 output "managed_nodegroup_launch_template_arn" {
-  description = "launch templated id for EKS Managed Node Group"
+  description = "Launch Template ARN for EKS Managed Node Group"
   value       = aws_launch_template.managed_node_groups[*].arn
 }
 
 output "managed_nodegroup_launch_template_latest_version" {
-  description = "launch templated version for EKS Managed Node Group"
+  description = "Launch Template version for EKS Managed Node Group"
   value       = aws_launch_template.managed_node_groups[*].default_version
 }
 

modules/aws-eks-managed-node-groups/variables.tf

@@ -37,6 +37,12 @@ variable "worker_security_group_id" {
   default     = ""
 }
 
+variable "worker_additional_security_group_ids" {
+  description = "Worker additional group security IDs"
+  type        = list(string)
+  default     = []
+}
+
 variable "vpc_id" {
   description = "VPC Id used in security group creation"
   type        = string

modules/aws-eks-self-managed-node-groups/README.md

@@ -185,6 +185,7 @@ No modules.
 | <a name="input_self_managed_ng"></a> [self\_managed\_ng](#input\_self\_managed\_ng) | Map of maps of `eks_self_managed_node_groups` to create | `any` | `{}` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC Id used in security group creation | `string` | n/a | yes |
+| <a name="input_worker_additional_security_group_ids"></a> [worker\_additional\_security\_group\_ids](#input\_worker\_additional\_security\_group\_ids) | Worker additional group security IDs | `list(string)` | `[]` | no |
 | <a name="input_worker_security_group_id"></a> [worker\_security\_group\_id](#input\_worker\_security\_group\_id) | Worker group security ID | `string` | `""` | no |
 
 ## Outputs
@@ -196,9 +197,10 @@ No modules.
 | <a name="output_launch_template_latest_versions"></a> [launch\_template\_latest\_versions](#output\_launch\_template\_latest\_versions) | launch templated version for EKS Self Managed Node Group |
 | <a name="output_self_managed_asg_names"></a> [self\_managed\_asg\_names](#output\_self\_managed\_asg\_names) | Self managed group ASG names |
 | <a name="output_self_managed_iam_role_name"></a> [self\_managed\_iam\_role\_name](#output\_self\_managed\_iam\_role\_name) | Self managed groups IAM role names |
-| <a name="output_self_managed_node_group_iam_instance_profile"></a> [self\_managed\_node\_group\_iam\_instance\_profile](#output\_self\_managed\_node\_group\_iam\_instance\_profile) | Self managed groups IAM Instance profile name |
 | <a name="output_self_managed_node_group_iam_role_arns"></a> [self\_managed\_node\_group\_iam\_role\_arns](#output\_self\_managed\_node\_group\_iam\_role\_arns) | Self managed groups IAM role arns |
 | <a name="output_self_managed_node_group_name"></a> [self\_managed\_node\_group\_name](#output\_self\_managed\_node\_group\_name) | EKS Self Managed node group id |
+| <a name="output_self_managed_nodegroup_iam_instance_profile_arn"></a> [self\_managed\_nodegroup\_iam\_instance\_profile\_arn](#output\_self\_managed\_nodegroup\_iam\_instance\_profile\_arn) | IAM Instance Profile arnd for EKS Self Managed Node Group |
+| <a name="output_self_managed_nodegroup_iam_instance_profile_id"></a> [self\_managed\_nodegroup\_iam\_instance\_profile\_id](#output\_self\_managed\_nodegroup\_iam\_instance\_profile\_id) | IAM Instance Profile ID for EKS Self Managed Node Group |
 | <a name="output_self_managed_sec_group_id"></a> [self\_managed\_sec\_group\_id](#output\_self\_managed\_sec\_group\_id) | Self managed group security group id/ids |
 
 <!--- END_TF_DOCS --->