diff --git a/README.md b/README.md index c82ffd5fe0..d003e14724 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Please note that we strive to provide a comprehensive suite of documentation for - [EKS Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - [Self Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/worker.html) - [Fargate Profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate.html) +- Support for creating Karpenter related AWS infrastruture resources (e.g. IAM roles, SQS queue, EventBridge rules, etc.) - Support for custom AMI, custom launch template, and custom user data including custom user data template - Support for Amazon Linux 2 EKS Optimized AMI and Bottlerocket nodes - Windows based node support is limited to a default user data template that is provided due to the lack of Windows support and manual steps required to provision Windows based EKS nodes @@ -60,7 +61,7 @@ module "eks" { version = "~> 18.0" cluster_name = "my-cluster" - cluster_version = "1.22" + cluster_version = "1.24" cluster_endpoint_private_access = true cluster_endpoint_public_access = true @@ -295,7 +296,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | | [cluster\_timeouts](#input\_cluster\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | -| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`) | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`) | `string` | `null` | no | | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane | `list(string)` | `[]` | no | | [create](#input\_create) | Controls if EKS resources should be created (affects nearly all resources) | `bool` | `true` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | @@ -363,8 +364,9 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/docs/compute_resources.md b/docs/compute_resources.md index 51d41025bb..d90cf5f0a8 100644 --- a/docs/compute_resources.md +++ b/docs/compute_resources.md @@ -141,9 +141,9 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 1. The `self-managed-node-group` uses the latest AWS EKS Optimized AMI (Linux) for the given Kubernetes version by default: ```hcl - cluster_version = "1.22" + cluster_version = "1.24" - # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.22 + # This self managed node group will use the latest AWS EKS Optimized AMI for Kubernetes 1.24 self_managed_node_groups = { default = {} } @@ -152,7 +152,7 @@ Refer to the [Self Managed Node Group documentation](https://docs.aws.amazon.com 2. To use Bottlerocket, specify the `platform` as `bottlerocket` and supply a Bottlerocket OS AMI: ```hcl - cluster_version = "1.22" + cluster_version = "1.24" self_managed_node_groups = { bottlerocket = { diff --git a/docs/irsa_integration.md b/docs/irsa_integration.md index 6c78bd9575..cde709fd2c 100644 --- a/docs/irsa_integration.md +++ b/docs/irsa_integration.md @@ -8,7 +8,7 @@ module "eks" { source = "terraform-aws-modules/eks/aws" cluster_name = "example" - cluster_version = "1.22" + cluster_version = "1.24" cluster_addons = { vpc-cni = { diff --git a/examples/complete/README.md b/examples/complete/README.md index 6961891591..66f3e8fdf1 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -81,8 +81,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index a34761e687..04817fa78b 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -10,7 +10,7 @@ provider "kubernetes" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } @@ -267,7 +267,7 @@ module "eks_managed_node_group" { source = "../../modules/eks-managed-node-group" name = "separate-eks-mng" - cluster_name = module.eks.cluster_id + cluster_name = module.eks.cluster_name cluster_version = module.eks.cluster_version vpc_id = module.vpc.vpc_id @@ -298,7 +298,7 @@ module "self_managed_node_group" { source = "../../modules/self-managed-node-group" name = "separate-self-mng" - cluster_name = module.eks.cluster_id + cluster_name = module.eks.cluster_name cluster_version = module.eks.cluster_version cluster_endpoint = module.eks.cluster_endpoint cluster_auth_base64 = module.eks.cluster_certificate_authority_data @@ -319,7 +319,7 @@ module "fargate_profile" { source = "../../modules/fargate-profile" name = "separate-fargate-profile" - cluster_name = module.eks.cluster_id + cluster_name = module.eks.cluster_name subnet_ids = module.vpc.private_subnets selectors = [{ diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf index c6b06de812..f94c629541 100644 --- a/examples/complete/outputs.tf +++ b/examples/complete/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index 9014bc660b..449777a52a 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -115,8 +115,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 12db52be62..050a508810 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -10,13 +10,13 @@ provider "kubernetes" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" tags = { diff --git a/examples/eks_managed_node_group/outputs.tf b/examples/eks_managed_node_group/outputs.tf index 6e31908292..67de532779 100644 --- a/examples/eks_managed_node_group/outputs.tf +++ b/examples/eks_managed_node_group/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index dddbc5755e..2f9cf7da1b 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -68,8 +68,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/fargate_profile/main.tf b/examples/fargate_profile/main.tf index a7b3f8f903..0bf15a7442 100644 --- a/examples/fargate_profile/main.tf +++ b/examples/fargate_profile/main.tf @@ -11,14 +11,14 @@ provider "helm" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } } locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" tags = { @@ -104,7 +104,7 @@ module "eks" { ################################################################################ data "aws_eks_cluster_auth" "this" { - name = module.eks.cluster_id + name = module.eks.cluster_name } locals { @@ -113,7 +113,7 @@ locals { kind = "Config" current-context = "terraform" clusters = [{ - name = module.eks.cluster_id + name = module.eks.cluster_name cluster = { certificate-authority-data = module.eks.cluster_certificate_authority_data server = module.eks.cluster_endpoint @@ -122,7 +122,7 @@ locals { contexts = [{ name = "terraform" context = { - cluster = module.eks.cluster_id + cluster = module.eks.cluster_name user = "terraform" } }] diff --git a/examples/fargate_profile/outputs.tf b/examples/fargate_profile/outputs.tf index 6e31908292..67de532779 100644 --- a/examples/fargate_profile/outputs.tf +++ b/examples/fargate_profile/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 761299aa54..7b865797a4 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -1,6 +1,6 @@ # Karpenter Example -Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances. ## Usage @@ -55,6 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des | [aws](#requirement\_aws) | >= 3.72 | | [helm](#requirement\_helm) | >= 2.4 | | [kubectl](#requirement\_kubectl) | >= 1.14 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [null](#requirement\_null) | >= 3.0 | ## Providers @@ -63,24 +65,29 @@ Note that this example may create resources which cost money. Run `terraform des | [aws](#provider\_aws) | >= 3.72 | | [helm](#provider\_helm) | >= 2.4 | | [kubectl](#provider\_kubectl) | >= 1.14 | +| [null](#provider\_null) | >= 3.0 | ## Modules | Name | Source | Version | |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | -| [karpenter\_irsa](#module\_karpenter\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 4.21.1 | +| [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | ## Resources | Name | Type | |------|------| -| [aws_iam_instance_profile.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [helm_release.coredns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [null_resource.modify_kube_dns](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.remove_default_coredns_deployment](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | ## Inputs @@ -100,8 +107,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | @@ -112,6 +120,20 @@ No inputs. | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | | [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | +| [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes | +| [karpenter\_instance\_profile\_arn](#output\_karpenter\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [karpenter\_instance\_profile\_id](#output\_karpenter\_instance\_profile\_id) | Instance profile's ID | +| [karpenter\_instance\_profile\_name](#output\_karpenter\_instance\_profile\_name) | Name of the instance profile | +| [karpenter\_instance\_profile\_unique](#output\_karpenter\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [karpenter\_irsa\_arn](#output\_karpenter\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts | +| [karpenter\_irsa\_name](#output\_karpenter\_irsa\_name) | The name of the IAM role for service accounts | +| [karpenter\_irsa\_unique\_id](#output\_karpenter\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts | +| [karpenter\_queue\_arn](#output\_karpenter\_queue\_arn) | The ARN of the SQS queue | +| [karpenter\_queue\_name](#output\_karpenter\_queue\_name) | The name of the created Amazon SQS queue | +| [karpenter\_queue\_url](#output\_karpenter\_queue\_url) | The URL for the created Amazon SQS queue | +| [karpenter\_role\_arn](#output\_karpenter\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [karpenter\_role\_name](#output\_karpenter\_role\_name) | The name of the IAM role | +| [karpenter\_role\_unique\_id](#output\_karpenter\_role\_unique\_id) | Stable and unique string identifying the IAM role | | [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group | | [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 3f43d80b2a..3634132ee2 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -2,13 +2,50 @@ provider "aws" { region = local.region } -data "aws_partition" "current" {} +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } + } +} + +provider "kubectl" { + apply_retry_count = 5 + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" - partition = data.aws_partition.current.partition tags = { Example = local.name @@ -32,29 +69,35 @@ module "eks" { vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets - node_security_group_additional_rules = { - # Control plane invoke Karpenter webhook - ingress_karpenter_webhook_tcp = { - description = "Control plane invoke Karpenter webhook" - protocol = "tcp" - from_port = 8443 - to_port = 8443 - type = "ingress" - source_cluster_security_group = true + # Fargate profiles use the cluster primary security group so these are not utilized + create_cluster_security_group = false + create_node_security_group = false + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.karpenter.role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + + fargate_profiles = { + kube_system = { + name = "kube-system" + selectors = [ + { namespace = "kube-system" } + ] } - } - eks_managed_node_groups = { karpenter = { - instance_types = ["t3.medium"] - - min_size = 1 - max_size = 2 - desired_size = 1 - - iam_role_additional_policies = [ - # Required by Karpenter - "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + name = "karpenter" + selectors = [ + { namespace = "karpenter" } ] } } @@ -71,60 +114,14 @@ module "eks" { # Karpenter ################################################################################ -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] - } - } -} - -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] - } -} - -module "karpenter_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.21.1" - - role_name = "karpenter-controller-${local.name}" - attach_karpenter_controller_policy = true - karpenter_controller_cluster_id = module.eks.cluster_id - karpenter_controller_ssm_parameter_arns = [ - "arn:${local.partition}:ssm:*:*:parameter/aws/service/*" - ] - karpenter_controller_node_iam_role_arns = [ - module.eks.eks_managed_node_groups["karpenter"].iam_role_arn - ] +module "karpenter" { + source = "../../modules/karpenter" - oidc_providers = { - ex = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] - } - } -} + cluster_name = module.eks.cluster_name + irsa_oidc_provider_arn = module.eks.oidc_provider_arn -resource "aws_iam_instance_profile" "karpenter" { - name = "KarpenterNodeInstanceProfile-${local.name}" - role = module.eks.eks_managed_node_groups["karpenter"].iam_role_name + tags = local.tags } resource "helm_release" "karpenter" { @@ -132,54 +129,73 @@ resource "helm_release" "karpenter" { create_namespace = true name = "karpenter" - repository = "https://charts.karpenter.sh" + repository = "oci://public.ecr.aws/karpenter" chart = "karpenter" - version = "0.8.2" + version = "v0.19.1" set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter_irsa.iam_role_arn + name = "settings.aws.clusterName" + value = module.eks.cluster_name } set { - name = "clusterName" - value = module.eks.cluster_id + name = "settings.aws.clusterEndpoint" + value = module.eks.cluster_endpoint } set { - name = "clusterEndpoint" - value = module.eks.cluster_endpoint + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.karpenter.irsa_arn } set { - name = "aws.defaultInstanceProfile" - value = aws_iam_instance_profile.karpenter.name + name = "settings.aws.defaultInstanceProfile" + value = module.karpenter.instance_profile_name + } + + set { + name = "settings.aws.interruptionQueueName" + value = module.karpenter.queue_name } } -# Workaround - https://github.com/hashicorp/terraform-provider-kubernetes/issues/1380#issuecomment-967022975 resource "kubectl_manifest" "karpenter_provisioner" { yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner - metadata: - name: default - spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - provider: + apiVersion: karpenter.sh/v1alpha5 + kind: Provisioner + metadata: + name: default + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + resources: + cpu: 1000 + providerRef: + name: default + ttlSecondsAfterEmpty: 30 + YAML + + depends_on = [ + helm_release.karpenter + ] +} + +resource "kubectl_manifest" "karpenter_node_template" { + yaml_body = <<-YAML + apiVersion: karpenter.k8s.aws/v1alpha1 + kind: AWSNodeTemplate + metadata: + name: default + spec: subnetSelector: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${module.eks.cluster_name} securityGroupSelector: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${module.eks.cluster_name} tags: - karpenter.sh/discovery: ${local.name} - ttlSecondsAfterEmpty: 30 + karpenter.sh/discovery: ${module.eks.cluster_name} YAML depends_on = [ @@ -191,27 +207,27 @@ resource "kubectl_manifest" "karpenter_provisioner" { # and starts with zero replicas resource "kubectl_manifest" "karpenter_example_deployment" { yaml_body = <<-YAML - apiVersion: apps/v1 - kind: Deployment - metadata: - name: inflate - spec: - replicas: 0 - selector: - matchLabels: - app: inflate - template: - metadata: - labels: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: inflate + spec: + replicas: 0 + selector: + matchLabels: app: inflate - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 - resources: - requests: - cpu: 1 + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 YAML depends_on = [ @@ -219,6 +235,129 @@ resource "kubectl_manifest" "karpenter_example_deployment" { ] } +################################################################################ +# Modify EKS CoreDNS Deployment +################################################################################ + +data "aws_eks_cluster_auth" "this" { + name = module.eks.cluster_name +} + +locals { + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = module.eks.cluster_name + cluster = { + certificate-authority-data = module.eks.cluster_certificate_authority_data + server = module.eks.cluster_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = module.eks.cluster_name + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = data.aws_eks_cluster_auth.this.token + } + }] + }) +} + +# Separate resource so that this is only ever executed once +resource "null_resource" "remove_default_coredns_deployment" { + triggers = {} + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(local.kubeconfig) + } + + # We are removing the deployment provided by the EKS service and replacing it through the self-managed CoreDNS Helm addon + # However, we are maintaing the existing kube-dns service and annotating it for Helm to assume control + command = <<-EOT + kubectl --namespace kube-system delete deployment coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } +} + +resource "null_resource" "modify_kube_dns" { + triggers = {} + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(local.kubeconfig) + } + + # We are maintaing the existing kube-dns service and annotating it for Helm to assume control + command = <<-EOT + echo "Setting implicit dependency on ${module.eks.fargate_profiles["kube_system"].fargate_profile_pod_execution_role_arn}" + kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-name=coredns --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl --namespace kube-system annotate --overwrite service kube-dns meta.helm.sh/release-namespace=kube-system --kubeconfig <(echo $KUBECONFIG | base64 --decode) + kubectl --namespace kube-system label --overwrite service kube-dns app.kubernetes.io/managed-by=Helm --kubeconfig <(echo $KUBECONFIG | base64 --decode) + EOT + } + + depends_on = [ + null_resource.remove_default_coredns_deployment + ] +} + +################################################################################ +# CoreDNS Helm Chart (self-managed) +################################################################################ + +data "aws_eks_addon_version" "this" { + for_each = toset(["coredns"]) + + addon_name = each.value + kubernetes_version = module.eks.cluster_version + most_recent = true +} + +resource "helm_release" "coredns" { + name = "coredns" + namespace = "kube-system" + create_namespace = false + description = "CoreDNS is a DNS server that chains plugins and provides Kubernetes DNS Services" + chart = "coredns" + version = "1.19.4" + repository = "https://coredns.github.io/helm" + + # For EKS image repositories https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html + values = [ + <<-EOT + image: + repository: 602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns + tag: ${data.aws_eks_addon_version.this["coredns"].version} + deployment: + name: coredns + annotations: + eks.amazonaws.com/compute-type: fargate + service: + name: kube-dns + annotations: + eks.amazonaws.com/compute-type: fargate + podAnnotations: + eks.amazonaws.com/compute-type: fargate + EOT + ] + + depends_on = [ + # Need to ensure the CoreDNS updates are peformed before provisioning + null_resource.modify_kube_dns + ] +} + ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index dbbec2334e..39d418bf37 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } @@ -185,3 +190,93 @@ output "aws_auth_configmap_yaml" { description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles" value = module.eks.aws_auth_configmap_yaml } + +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +output "karpenter_irsa_name" { + description = "The name of the IAM role for service accounts" + value = module.karpenter.irsa_name +} + +output "karpenter_irsa_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts" + value = module.karpenter.irsa_arn +} + +output "karpenter_irsa_unique_id" { + description = "Stable and unique string identifying the IAM role for service accounts" + value = module.karpenter.irsa_unique_id +} + +################################################################################ +# Node Termination Queue +################################################################################ + +output "karpenter_queue_arn" { + description = "The ARN of the SQS queue" + value = module.karpenter.queue_arn +} + +output "karpenter_queue_name" { + description = "The name of the created Amazon SQS queue" + value = module.karpenter.queue_name +} + +output "karpenter_queue_url" { + description = "The URL for the created Amazon SQS queue" + value = module.karpenter.queue_url +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +output "karpenter_event_rules" { + description = "Map of the event rules created and their attributes" + value = module.karpenter.event_rules +} + +################################################################################ +# Node IAM Role +################################################################################ + +output "karpenter_role_name" { + description = "The name of the IAM role" + value = module.karpenter.role_name +} + +output "karpenter_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = module.karpenter.role_arn +} + +output "karpenter_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = module.karpenter.role_unique_id +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +output "karpenter_instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = module.karpenter.instance_profile_arn +} + +output "karpenter_instance_profile_id" { + description = "Instance profile's ID" + value = module.karpenter.instance_profile_id +} + +output "karpenter_instance_profile_name" { + description = "Name of the instance profile" + value = module.karpenter.instance_profile_name +} + +output "karpenter_instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = module.karpenter.instance_profile_unique +} diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index fe18abab81..92e85aacf6 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -6,6 +6,10 @@ terraform { source = "hashicorp/aws" version = ">= 3.72" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } helm = { source = "hashicorp/helm" version = ">= 2.4" @@ -14,5 +18,9 @@ terraform { source = "gavinbunney/kubectl" version = ">= 1.14" } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } } } diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index a543d6454b..9608cdb8cd 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -77,8 +77,9 @@ No inputs. | [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster | | [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster | | [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | -| [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready | +| [cluster\_id](#output\_cluster\_id) | The id of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled | +| [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster. Will block on cluster creation until the cluster is really ready | | [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider | | [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster | | [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console | diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index 116dbc2553..27e82eebb3 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -10,13 +10,13 @@ provider "kubernetes" { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] } } locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.22" + cluster_version = "1.24" region = "eu-west-1" tags = { diff --git a/examples/self_managed_node_group/outputs.tf b/examples/self_managed_node_group/outputs.tf index 6e31908292..67de532779 100644 --- a/examples/self_managed_node_group/outputs.tf +++ b/examples/self_managed_node_group/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = module.eks.cluster_endpoint } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = module.eks.cluster_name +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = module.eks.cluster_id } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index eb8f35dbd3..bf8d6de605 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -10,7 +10,7 @@ module "eks_managed_node_group" { name = "separate-eks-mng" cluster_name = "my-cluster" - cluster_version = "1.22" + cluster_version = "1.24" vpc_id = "vpc-1234556abcdef" subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md new file mode 100644 index 0000000000..1797c6d2e8 --- /dev/null +++ b/modules/karpenter/README.md @@ -0,0 +1,195 @@ +# Karpenter Module + +Configuration in this directory creates the AWS resources required by Karpenter + +## Usage + +### All Resources (Default) + +In the following example, the Karpenter module will create: +- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize +- An IAM role and instance profile for the nodes created by Karpenter to utilize + - Note: This IAM role ARN will need to be added to the `aws-auth` configmap for nodes to join the cluster successfully +- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. + +This setup is great for running Karpenter on EKS Fargate: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks" + + # Shown just for connection between cluster and Karpenter sub-module below + manage_aws_auth_configmap = true + aws_auth_roles = [ + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.karpenter.role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] + ... +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + + cluster_name = module.eks.cluster_name + + irsa_oidc_provider_arn = module.eks.oidc_provider_arn + irsa_namespace_service_accounts = ["karpenter:karpenter"] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### External Node IAM Role (Default) + +In the following example, the Karpenter module will create: +- An IAM role for service accounts (IRSA) with a narrowly scoped IAM policy for the Karpenter controller to utilize +- An IAM instance profile for the nodes created by Karpenter to utilize + - Note: This setup will utilize the existing IAM role created by the EKS Managed Node group which means the role is already populated in the `aws-auth` configmap and no further updates are required. +- An SQS queue and Eventbridge event rules for Karpenter to utilize for spot termination handling, capacity rebalancing, etc. + +In this scenario, Karpenter would run atop the EKS Managed Node group and scale out nodes as needed from there: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks" + + # Shown just for connection between cluster and Karpenter sub-module below + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 3 + desired_size = 1 + } + } + ... +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + + cluster_name = module.eks.cluster_name + + irsa_oidc_provider_arn = module.eks.oidc_provider_arn + irsa_namespace_service_accounts = ["karpenter:karpenter"] + + create_iam_role = false + iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 3.72 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.72 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.irsa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.irsa_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `null` | no | +| [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | +| [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | +| [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | +| [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `list(string)` | `[]` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | +| [irsa\_description](#input\_irsa\_description) | IAM role for service accounts description | `string` | `"Karpenter IAM role for service account"` | no | +| [irsa\_max\_session\_duration](#input\_irsa\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [irsa\_name](#input\_irsa\_name) | Name of IAM role for service accounts | `string` | `null` | no | +| [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` | 
[
  "karpenter:karpenter"
]
| no | +| [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | +| [irsa\_path](#input\_irsa\_path) | Path of IAM role for service accounts | `string` | `"/"` | no | +| [irsa\_permissions\_boundary\_arn](#input\_irsa\_permissions\_boundary\_arn) | Permissions boundary ARN to use for IAM role for service accounts | `string` | `null` | no | +| [irsa\_ssm\_parameter\_arns](#input\_irsa\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` | 
[
  "arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | +| [irsa\_subnet\_account\_id](#input\_irsa\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | +| [irsa\_tag\_key](#input\_irsa\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | +| [irsa\_tags](#input\_irsa\_tags) | A map of additional tags to add the the IAM role for service accounts | `map(any)` | `{}` | no | +| [irsa\_use\_name\_prefix](#input\_irsa\_use\_name\_prefix) | Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix | `bool` | `true` | no | +| [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no | +| [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | +| [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | +| [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [event\_rules](#output\_event\_rules) | Map of the event rules created and their attributes | +| [instance\_profile\_arn](#output\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | +| [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | +| [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [irsa\_arn](#output\_irsa\_arn) | The Amazon Resource Name (ARN) specifying the IAM role for service accounts | +| [irsa\_name](#output\_irsa\_name) | The name of the IAM role for service accounts | +| [irsa\_unique\_id](#output\_irsa\_unique\_id) | Stable and unique string identifying the IAM role for service accounts | +| [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | +| [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | +| [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | +| [role\_arn](#output\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [role\_name](#output\_role\_name) | The name of the IAM role | +| [role\_unique\_id](#output\_role\_unique\_id) | Stable and unique string identifying the IAM role | + diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf new file mode 100644 index 0000000000..c7ae5cc35d --- /dev/null +++ b/modules/karpenter/main.tf @@ -0,0 +1,359 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +locals { + account_id = data.aws_caller_identity.current.account_id + partition = data.aws_partition.current.partition + dns_suffix = data.aws_partition.current.dns_suffix +} + +################################################################################ +# IAM Role for Service Account (IRSA) +# This is used by the Karpenter controller +################################################################################ + +locals { + create_irsa = var.create && var.create_irsa + irsa_name = coalesce(var.irsa_name, "KarpenterIRSA-${var.cluster_name}") + + irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "") +} + +data "aws_iam_policy_document" "irsa_assume_role" { + count = local.create_irsa ? 1 : 0 + + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [var.irsa_oidc_provider_arn] + } + + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:sub" + values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"] + } + + # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:aud" + values = ["sts.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "irsa" { + count = local.create_irsa ? 1 : 0 + + name = var.irsa_use_name_prefix ? null : local.irsa_name + name_prefix = var.irsa_use_name_prefix ? "${local.irsa_name}-" : null + path = var.irsa_path + description = var.irsa_description + + assume_role_policy = data.aws_iam_policy_document.irsa_assume_role[0].json + max_session_duration = var.irsa_max_session_duration + permissions_boundary = var.irsa_permissions_boundary_arn + force_detach_policies = true + + tags = merge(var.tags, var.irsa_tags) +} + +data "aws_iam_policy_document" "irsa" { + count = local.create_irsa ? 1 : 0 + + statement { + actions = [ + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:CreateTags", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeSpotPriceHistory", + "pricing:GetProducts", + ] + + resources = ["*"] + } + + statement { + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/${var.irsa_tag_key}" + values = [var.cluster_name] + } + } + + statement { + actions = ["ec2:RunInstances"] + resources = [ + "arn:${local.partition}:ec2:*:${local.account_id}:launch-template/*", + ] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/${var.irsa_tag_key}" + values = [var.cluster_name] + } + } + + statement { + actions = ["ec2:RunInstances"] + resources = [ + "arn:${local.partition}:ec2:*::image/*", + "arn:${local.partition}:ec2:*:${local.account_id}:instance/*", + "arn:${local.partition}:ec2:*:${local.account_id}:spot-instances-request/*", + "arn:${local.partition}:ec2:*:${local.account_id}:security-group/*", + "arn:${local.partition}:ec2:*:${local.account_id}:volume/*", + "arn:${local.partition}:ec2:*:${local.account_id}:network-interface/*", + "arn:${local.partition}:ec2:*:${coalesce(var.irsa_subnet_account_id, local.account_id)}:subnet/*", + ] + } + + statement { + actions = ["ssm:GetParameter"] + resources = var.irsa_ssm_parameter_arns + } + + statement { + actions = ["iam:PassRole"] + resources = [var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn] + } + + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage", + ] + resources = [aws_sqs_queue.this[0].arn] + } + } +} + +resource "aws_iam_policy" "irsa" { + count = var.create_irsa ? 1 : 0 + + name_prefix = "${local.irsa_name}-" + path = var.irsa_path + description = var.irsa_description + policy = data.aws_iam_policy_document.irsa[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "irsa" { + count = local.create_irsa ? 1 : 0 + + role = aws_iam_role.irsa[0].name + policy_arn = aws_iam_policy.irsa[0].arn +} + +################################################################################ +# Node Termination Queue +################################################################################ + +locals { + enable_spot_termination = var.create && var.enable_spot_termination + + queue_name = coalesce(var.queue_name, "Karpenter-${var.cluster_name}") +} + +resource "aws_sqs_queue" "this" { + count = local.enable_spot_termination ? 1 : 0 + + name = local.queue_name + message_retention_seconds = 300 + sqs_managed_sse_enabled = var.queue_managed_sse_enabled + kms_master_key_id = var.queue_kms_master_key_id + kms_data_key_reuse_period_seconds = var.queue_kms_data_key_reuse_period_seconds + + tags = var.tags +} + +data "aws_iam_policy_document" "queue" { + count = local.enable_spot_termination ? 1 : 0 + + statement { + sid = "SqsWrite" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.this[0].arn] + + principals { + type = "Service" + identifiers = [ + "events.${local.dns_suffix}", + "sqs.${local.dns_suffix}", + ] + } + + } +} + +resource "aws_sqs_queue_policy" "this" { + count = local.enable_spot_termination ? 1 : 0 + + queue_url = aws_sqs_queue.this[0].url + policy = data.aws_iam_policy_document.queue[0].json +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +locals { + events = { + health_event = { + name = "HealthEvent" + description = "Karpenter interrupt - AWS health event" + event_pattern = { + source = ["aws.health"] + detail-type = ["AWS Health Event"] + } + } + spot_interupt = { + name = "SpotInterrupt" + description = "Karpenter interrupt - EC2 spot instance interruption warning" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Spot Instance Interruption Warning"] + } + } + instance_rebalance = { + name = "InstanceRebalance" + description = "Karpenter interrupt - EC2 instance rebalance recommendation" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Instance Rebalance Recommendation"] + } + } + instance_state_change = { + name = "InstanceStateChange" + description = "Karpenter interrupt - EC2 instance state-change notification" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Instance State-change Notification"] + } + } + } +} + +resource "aws_cloudwatch_event_rule" "this" { + for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + + name = "Karpenter${each.value.name}-${var.cluster_name}" + description = each.value.description + event_pattern = jsonencode(each.value.event_pattern) + + tags = var.tags +} + +resource "aws_cloudwatch_event_target" "this" { + for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + + rule = aws_cloudwatch_event_rule.this[each.key].name + target_id = "KarpenterInterruptionQueueTarget" + arn = aws_sqs_queue.this[0].arn +} + +################################################################################ +# Node IAM Role +# This is used by the nodes launched by Karpenter +################################################################################ + +locals { + create_iam_role = var.create && var.create_iam_role + + iam_role_name = coalesce(var.iam_role_name, "Karpenter-${var.cluster_name}") + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" + cni_policy = var.cluster_ip_family == "ipv6" ? "${local.iam_role_policy_prefix}/AmazonEKS_CNI_IPv6_Policy" : "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" +} + +data "aws_iam_policy_document" "assume_role" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.${local.dns_suffix}"] + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role[0].json + max_session_duration = var.iam_role_max_session_duration + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in toset(compact([ + "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy", + "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly", + var.iam_role_attach_cni_policy ? local.cni_policy : "", + ])) : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# Node IAM Instance Profile +# This is used by the nodes launched by Karpenter +################################################################################ + +locals { + external_role_name = try(replace(var.iam_role_arn, "/^(.*role/)/", ""), null) +} + +resource "aws_iam_instance_profile" "this" { + count = var.create && var.create_instance_profile ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + role = var.create_iam_role ? aws_iam_role.this[0].name : local.external_role_name + + tags = merge(var.tags, var.iam_role_tags) +} diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf new file mode 100644 index 0000000000..947de39bfd --- /dev/null +++ b/modules/karpenter/outputs.tf @@ -0,0 +1,89 @@ +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +output "irsa_name" { + description = "The name of the IAM role for service accounts" + value = try(aws_iam_role.irsa[0].name, null) +} + +output "irsa_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts" + value = try(aws_iam_role.irsa[0].arn, null) +} + +output "irsa_unique_id" { + description = "Stable and unique string identifying the IAM role for service accounts" + value = try(aws_iam_role.irsa[0].unique_id, null) +} + +################################################################################ +# Node Termination Queue +################################################################################ + +output "queue_arn" { + description = "The ARN of the SQS queue" + value = try(aws_sqs_queue.this[0].arn, null) +} + +output "queue_name" { + description = "The name of the created Amazon SQS queue" + value = try(aws_sqs_queue.this[0].name, null) +} + +output "queue_url" { + description = "The URL for the created Amazon SQS queue" + value = try(aws_sqs_queue.this[0].url, null) +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +output "event_rules" { + description = "Map of the event rules created and their attributes" + value = aws_cloudwatch_event_rule.this +} + +################################################################################ +# Node IAM Role +################################################################################ + +output "role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, var.iam_role_arn) +} + +output "role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +output "instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = try(aws_iam_instance_profile.this[0].arn, null) +} + +output "instance_profile_id" { + description = "Instance profile's ID" + value = try(aws_iam_instance_profile.this[0].id, null) +} + +output "instance_profile_name" { + description = "Name of the instance profile" + value = try(aws_iam_instance_profile.this[0].name, null) +} + +output "instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = try(aws_iam_instance_profile.this[0].unique_id, null) +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf new file mode 100644 index 0000000000..a6327ba932 --- /dev/null +++ b/modules/karpenter/variables.tf @@ -0,0 +1,226 @@ +variable "create" { + description = "Determines whether to create EKS managed node group or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "cluster_name" { + description = "The name of the EKS cluster" + type = string + default = "" +} + +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +variable "create_irsa" { + description = "Determines whether an IAM role for service accounts is created" + type = bool + default = true +} + +variable "irsa_name" { + description = "Name of IAM role for service accounts" + type = string + default = null +} + +variable "irsa_use_name_prefix" { + description = "Determines whether the IAM role for service accounts name (`irsa_name`) is used as a prefix" + type = bool + default = true +} + +variable "irsa_path" { + description = "Path of IAM role for service accounts" + type = string + default = "/" +} + +variable "irsa_description" { + description = "IAM role for service accounts description" + type = string + default = "Karpenter IAM role for service account" +} + +variable "irsa_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "irsa_permissions_boundary_arn" { + description = "Permissions boundary ARN to use for IAM role for service accounts" + type = string + default = null +} + +variable "irsa_tags" { + description = "A map of additional tags to add the the IAM role for service accounts" + type = map(any) + default = {} +} + +variable "irsa_tag_key" { + description = "Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner" + type = string + default = "karpenter.sh/discovery" +} + +variable "irsa_ssm_parameter_arns" { + description = "List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter" + type = list(string) + # https://github.com/aws/karpenter/blob/ed9473a9863ca949b61b9846c8b9f33f35b86dbd/pkg/cloudprovider/aws/ami.go#L105-L123 + default = ["arn:aws:ssm:*:*:parameter/aws/service/*"] +} + +variable "irsa_subnet_account_id" { + description = "Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account" + type = string + default = "" +} + +variable "irsa_oidc_provider_arn" { + description = "OIDC provider arn used in trust policy for IAM role for service accounts" + type = string + default = "" +} + +variable "irsa_namespace_service_accounts" { + description = "List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts" + type = list(string) + default = ["karpenter:karpenter"] +} + +variable "irsa_assume_role_condition_test" { + description = "Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role" + type = string + default = "StringEquals" +} + +################################################################################ +# Node Termination Queue +################################################################################ + +variable "enable_spot_termination" { + description = "Determines whether to enable native spot termination handling" + type = bool + default = true +} + +variable "queue_name" { + description = "Name of the SQS queue" + type = string + default = null +} + +variable "queue_managed_sse_enabled" { + description = "Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys" + type = bool + default = true +} + +variable "queue_kms_master_key_id" { + description = "The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK" + type = string + default = null +} + +variable "queue_kms_data_key_reuse_period_seconds" { + description = "The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again" + type = number + default = null +} + +################################################################################ +# Node IAM Role & Instance Profile +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = null +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = "/" +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = list(string) + default = [] +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +variable "create_instance_profile" { + description = "Whether to create an IAM instance profile" + type = bool + default = true +} diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf new file mode 100644 index 0000000000..22e8d7265f --- /dev/null +++ b/modules/karpenter/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + } +} diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index eb0b484443..4c392362f6 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -10,7 +10,7 @@ module "self_managed_node_group" { name = "separate-self-mng" cluster_name = "my-cluster" - cluster_version = "1.22" + cluster_version = "1.24" cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" diff --git a/outputs.tf b/outputs.tf index 7fb6d8ce98..eadf2d0cd2 100644 --- a/outputs.tf +++ b/outputs.tf @@ -17,8 +17,13 @@ output "cluster_endpoint" { value = try(aws_eks_cluster.this[0].endpoint, "") } +output "cluster_name" { + description = "The name of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = try(aws_eks_cluster.this[0].name, "") +} + output "cluster_id" { - description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + description = "The id of the EKS cluster. Will block on cluster creation until the cluster is really ready" value = try(aws_eks_cluster.this[0].id, "") } diff --git a/variables.tf b/variables.tf index df2ee51638..6bce0560e2 100644 --- a/variables.tf +++ b/variables.tf @@ -27,7 +27,7 @@ variable "cluster_name" { } variable "cluster_version" { - description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.22`)" + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.24`)" type = string default = null }