Feat(aws-ebs-csi-driver): add capability to deploy into self managed …

…mode (aws-ia#896) Co-authored-by: Bryant Biggs <[email protected]> Co-authored-by: j_menan <[email protected]>
allamand · Dec 15, 2022 · a32dd33 · a32dd33
1 parent 6afff21
commit a32dd33
Show file tree

Hide file tree

Showing 7 changed files with 155 additions and 9 deletions.
diff --git a/examples/stateful/main.tf b/examples/stateful/main.tf
@@ -53,6 +53,7 @@ module "eks_blueprints" {
       node_group_name    = "velero"
       launch_template_os = "amazonlinux2eks"
       subnet_ids         = module.vpc.private_subnets
+      k8s_taints         = [{ key = "VeleroOnly", value = "true", effect = "NO_SCHEDULE" }]
     }
   }
 
@@ -72,6 +73,15 @@ module "eks_blueprints_kubernetes_addons" {
 
   enable_aws_efs_csi_driver = true
 
+  enable_self_managed_aws_ebs_csi_driver = true
+  self_managed_aws_ebs_csi_driver_helm_config = {
+    set_values = [
+      {
+        name  = "node.tolerateAllTaints"
+        value = "true"
+    }]
+  }
+
   tags = local.tags
 }
 
@@ -191,3 +201,19 @@ resource "aws_security_group" "efs" {
 
   tags = local.tags
 }
+
+resource "kubernetes_storage_class" "storage_class" {
+  metadata {
+    name = "gp3"
+  }
+
+  storage_provisioner    = "ebs.csi.aws.com"
+  allow_volume_expansion = true
+  reclaim_policy         = "Delete"
+  volume_binding_mode    = "WaitForFirstConsumer"
+  parameters = {
+    encrypted = true
+    fsType    = "ext4"
+    type      = "gp3"
+  }
+}
diff --git a/modules/kubernetes-addons/README.md b/modules/kubernetes-addons/README.md
@@ -142,7 +142,7 @@
 | <a name="input_enable_agones"></a> [enable\_agones](#input\_enable\_agones) | Enable Agones GamServer add-on | `bool` | `false` | no |
 | <a name="input_enable_airflow"></a> [enable\_airflow](#input\_enable\_airflow) | Enable Airflow add-on | `bool` | `false` | no |
 | <a name="input_enable_amazon_eks_adot"></a> [enable\_amazon\_eks\_adot](#input\_enable\_amazon\_eks\_adot) | Enable Amazon EKS ADOT addon | `bool` | `false` | no |
-| <a name="input_enable_amazon_eks_aws_ebs_csi_driver"></a> [enable\_amazon\_eks\_aws\_ebs\_csi\_driver](#input\_enable\_amazon\_eks\_aws\_ebs\_csi\_driver) | Enable EKS Managed AWS EBS CSI Driver add-on | `bool` | `false` | no |
+| <a name="input_enable_amazon_eks_aws_ebs_csi_driver"></a> [enable\_amazon\_eks\_aws\_ebs\_csi\_driver](#input\_enable\_amazon\_eks\_aws\_ebs\_csi\_driver) | Enable EKS Managed AWS EBS CSI Driver add-on; enable\_amazon\_eks\_aws\_ebs\_csi\_driver and enable\_self\_managed\_aws\_ebs\_csi\_driver are mutually exclusive | `bool` | `false` | no |
 | <a name="input_enable_amazon_eks_coredns"></a> [enable\_amazon\_eks\_coredns](#input\_enable\_amazon\_eks\_coredns) | Enable Amazon EKS CoreDNS add-on | `bool` | `false` | no |
 | <a name="input_enable_amazon_eks_kube_proxy"></a> [enable\_amazon\_eks\_kube\_proxy](#input\_enable\_amazon\_eks\_kube\_proxy) | Enable Kube Proxy add-on | `bool` | `false` | no |
 | <a name="input_enable_amazon_eks_vpc_cni"></a> [enable\_amazon\_eks\_vpc\_cni](#input\_enable\_amazon\_eks\_vpc\_cni) | Enable VPC CNI add-on | `bool` | `false` | no |
@@ -179,6 +179,7 @@
 | <a name="input_enable_promtail"></a> [enable\_promtail](#input\_enable\_promtail) | Enable Promtail add-on | `bool` | `false` | no |
 | <a name="input_enable_secrets_store_csi_driver"></a> [enable\_secrets\_store\_csi\_driver](#input\_enable\_secrets\_store\_csi\_driver) | Enable CSI Secrets Store Provider | `bool` | `false` | no |
 | <a name="input_enable_secrets_store_csi_driver_provider_aws"></a> [enable\_secrets\_store\_csi\_driver\_provider\_aws](#input\_enable\_secrets\_store\_csi\_driver\_provider\_aws) | Enable AWS CSI Secrets Store Provider | `bool` | `false` | no |
+| <a name="input_enable_self_managed_aws_ebs_csi_driver"></a> [enable\_self\_managed\_aws\_ebs\_csi\_driver](#input\_enable\_self\_managed\_aws\_ebs\_csi\_driver) | Enable self-managed aws-ebs-csi-driver add-on; enable\_self\_managed\_aws\_ebs\_csi\_driver and enable\_amazon\_eks\_aws\_ebs\_csi\_driver are mutually exclusive | `bool` | `false` | no |
 | <a name="input_enable_self_managed_coredns"></a> [enable\_self\_managed\_coredns](#input\_enable\_self\_managed\_coredns) | Enable self-managed CoreDNS add-on | `bool` | `false` | no |
 | <a name="input_enable_spark_history_server"></a> [enable\_spark\_history\_server](#input\_enable\_spark\_history\_server) | Enable Spark History Server add-on | `bool` | `false` | no |
 | <a name="input_enable_spark_k8s_operator"></a> [enable\_spark\_k8s\_operator](#input\_enable\_spark\_k8s\_operator) | Enable Spark on K8s Operator add-on | `bool` | `false` | no |
@@ -224,6 +225,7 @@
 | <a name="input_prometheus_helm_config"></a> [prometheus\_helm\_config](#input\_prometheus\_helm\_config) | Community Prometheus Helm Chart config | `any` | `{}` | no |
 | <a name="input_promtail_helm_config"></a> [promtail\_helm\_config](#input\_promtail\_helm\_config) | Promtail Helm Chart config | `any` | `{}` | no |
 | <a name="input_secrets_store_csi_driver_helm_config"></a> [secrets\_store\_csi\_driver\_helm\_config](#input\_secrets\_store\_csi\_driver\_helm\_config) | CSI Secrets Store Provider Helm Configurations | `any` | `null` | no |
+| <a name="input_self_managed_aws_ebs_csi_driver_helm_config"></a> [self\_managed\_aws\_ebs\_csi\_driver\_helm\_config](#input\_self\_managed\_aws\_ebs\_csi\_driver\_helm\_config) | Self-managed aws-ebs-csi-driver Helm chart config | `any` | `{}` | no |
 | <a name="input_self_managed_coredns_helm_config"></a> [self\_managed\_coredns\_helm\_config](#input\_self\_managed\_coredns\_helm\_config) | Self-managed CoreDNS Helm chart config | `any` | `{}` | no |
 | <a name="input_spark_history_server_helm_config"></a> [spark\_history\_server\_helm\_config](#input\_spark\_history\_server\_helm\_config) | Spark History Server Helm Chart config | `any` | `{}` | no |
 | <a name="input_spark_history_server_irsa_policies"></a> [spark\_history\_server\_irsa\_policies](#input\_spark\_history\_server\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |

diff --git a/modules/kubernetes-addons/aws-ebs-csi-driver/README.md b/modules/kubernetes-addons/aws-ebs-csi-driver/README.md
@@ -3,6 +3,37 @@
 [aws-ebs-csi-driver](https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html)
 The EBS CSI driver provides a CSI interface used by container orchestrators to manage the lifecycle of Amazon EBS volumes. Availability in EKS add-ons in preview enables a simple experience for attaching persistent storage to an EKS cluster. The EBS CSI driver can now be installed, managed, and updated directly through the EKS console, CLI, and API
 
+This addons supports managing AWS-EBS-CSI-DRIVER through either the EKS managed addon or a self-managed addon via Helm.
+
+## EKS Managed AWS-EBS-CSI-DRIVER Addon
+
+To enable and modify the EKS managed addon for aws-ebs-csi-driver, you can reference the following configuration and tailor to suit:
+
+```hcl
+  enable_amazon_eks_aws_ebs_csi_driver = true
+  amazon_eks_aws_ebs_csi_driver_config = {
+    resolve_conflicts = "OVERWRITE"
+    ...
+  }
+```
+
+## Self Managed AWS-EBS-CSI-DRIVER Addon
+
+Official [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) helm chart will be deploy in this mode
+
+you must use this mode if you need to change the configuration of the ebs-csi-driver as this is not possible with the EKS managed mode
+
+See the [`stateful`](https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/main/examples/stateful) example where the self-managed aws-ebs-csi-driver addon is used to provision the ebs-csi-driver on a EKS cluster
+
+To provision the self managed addon for aws-ebs-csi-driver, you can reference the following configuration and tailor to suit:
+
+ ```hcl
+   enable_self_managed_aws_ebs_csi_driver = true
+   self_managed_aws_ebs_csi_driver_helm_config = {
+    ...
+   }
+  ```
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
 
@@ -21,6 +52,7 @@ The EBS CSI driver provides a CSI interface used by container orchestrators to m
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_helm_addon"></a> [helm\_addon](#module\_helm\_addon) | ../helm-addon | n/a |
 | <a name="module_irsa_addon"></a> [irsa\_addon](#module\_irsa\_addon) | ../../../modules/irsa | n/a |
 
 ## Resources
@@ -37,6 +69,9 @@ The EBS CSI driver provides a CSI interface used by container orchestrators to m
 |------|-------------|------|---------|:--------:|
 | <a name="input_addon_config"></a> [addon\_config](#input\_addon\_config) | Amazon EKS Managed Add-on config for EBS CSI Driver | `any` | `{}` | no |
 | <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br>    aws_caller_identity_account_id = string<br>    aws_caller_identity_arn        = string<br>    aws_eks_cluster_endpoint       = string<br>    aws_partition_id               = string<br>    aws_region_name                = string<br>    eks_cluster_id                 = string<br>    eks_oidc_issuer_url            = string<br>    eks_oidc_provider_arn          = string<br>    tags                           = map(string)<br>    irsa_iam_role_path             = string<br>    irsa_iam_permissions_boundary  = string<br>  })</pre> | n/a | yes |
+| <a name="input_enable_amazon_eks_aws_ebs_csi_driver"></a> [enable\_amazon\_eks\_aws\_ebs\_csi\_driver](#input\_enable\_amazon\_eks\_aws\_ebs\_csi\_driver) | Enable EKS Managed AWS EBS CSI Driver add-on | `bool` | `false` | no |
+| <a name="input_enable_self_managed_aws_ebs_csi_driver"></a> [enable\_self\_managed\_aws\_ebs\_csi\_driver](#input\_enable\_self\_managed\_aws\_ebs\_csi\_driver) | Enable self-managed aws-ebs-csi-driver add-on | `bool` | `false` | no |
+| <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Self-managed aws-ebs-csi-driver Helm chart config | `any` | `{}` | no |
 
 ## Outputs
 

diff --git a/modules/kubernetes-addons/aws-ebs-csi-driver/main.tf b/modules/kubernetes-addons/aws-ebs-csi-driver/main.tf
@@ -1,8 +1,11 @@
 locals {
   create_irsa = try(var.addon_config.service_account_role_arn == "", true)
+  name        = try(var.helm_config.name, "aws-ebs-csi-driver")
+  namespace   = try(var.helm_config.namespace, "kube-system")
 }
 
 resource "aws_eks_addon" "aws_ebs_csi_driver" {
+  count                    = var.enable_amazon_eks_aws_ebs_csi_driver && !var.enable_self_managed_aws_ebs_csi_driver ? 1 : 0
   cluster_name             = var.addon_context.eks_cluster_id
   addon_name               = "aws-ebs-csi-driver"
   addon_version            = try(var.addon_config.addon_version, null)
@@ -19,11 +22,11 @@ resource "aws_eks_addon" "aws_ebs_csi_driver" {
 module "irsa_addon" {
   source = "../../../modules/irsa"
 
-  count = local.create_irsa ? 1 : 0
+  count = local.create_irsa && !var.enable_self_managed_aws_ebs_csi_driver ? 1 : 0
 
   create_kubernetes_namespace       = false
   create_kubernetes_service_account = false
-  kubernetes_namespace              = "kube-system"
+  kubernetes_namespace              = local.namespace
   kubernetes_service_account        = "ebs-csi-controller-sa"
   irsa_iam_policies                 = concat([aws_iam_policy.aws_ebs_csi_driver[0].arn], try(var.addon_config.additional_iam_policies, []))
   irsa_iam_role_path                = var.addon_context.irsa_iam_role_path
@@ -33,7 +36,7 @@ module "irsa_addon" {
 }
 
 resource "aws_iam_policy" "aws_ebs_csi_driver" {
-  count = local.create_irsa ? 1 : 0
+  count = local.create_irsa || var.enable_self_managed_aws_ebs_csi_driver ? 1 : 0
 
   name        = "${var.addon_context.eks_cluster_id}-aws-ebs-csi-driver-irsa"
   description = "IAM Policy for AWS EBS CSI Driver"
@@ -45,3 +48,45 @@ resource "aws_iam_policy" "aws_ebs_csi_driver" {
     try(var.addon_config.tags, {})
   )
 }
+
+module "helm_addon" {
+  source = "../helm-addon"
+  count  = var.enable_self_managed_aws_ebs_csi_driver && !var.enable_amazon_eks_aws_ebs_csi_driver ? 1 : 0
+
+  helm_config = merge({
+    name        = local.name
+    description = "The Amazon Elastic Block Store Container Storage Interface (CSI) Driver provides a CSI interface used by Container Orchestrators to manage the lifecycle of Amazon EBS volumes."
+    chart       = "aws-ebs-csi-driver"
+    version     = "2.10.1"
+    repository  = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"
+    namespace   = local.namespace
+    values = [
+      <<-EOT
+      image:
+        repository: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver
+      controller:
+        k8sTagClusterId: ${var.addon_context.eks_cluster_id}
+      EOT
+    ]
+    },
+    var.helm_config
+  )
+
+  set_values = [
+    {
+      name  = "controller.serviceAccount.create"
+      value = "false"
+    }
+  ]
+
+  irsa_config = {
+    create_kubernetes_namespace       = try(var.helm_config.create_namespace, false)
+    kubernetes_namespace              = local.namespace
+    create_kubernetes_service_account = true
+    kubernetes_service_account        = "ebs-csi-controller-sa"
+    irsa_iam_policies                 = concat([aws_iam_policy.aws_ebs_csi_driver[0].arn], try(var.helm_config.additional_iam_policies, []))
+  }
+
+  # Blueprints
+  addon_context = var.addon_context
+}
diff --git a/modules/kubernetes-addons/aws-ebs-csi-driver/variables.tf b/modules/kubernetes-addons/aws-ebs-csi-driver/variables.tf
@@ -20,3 +20,21 @@ variable "addon_context" {
     irsa_iam_permissions_boundary  = string
   })
 }
+
+variable "enable_amazon_eks_aws_ebs_csi_driver" {
+  description = "Enable EKS Managed AWS EBS CSI Driver add-on"
+  type        = bool
+  default     = false
+}
+
+variable "enable_self_managed_aws_ebs_csi_driver" {
+  description = "Enable self-managed aws-ebs-csi-driver add-on"
+  type        = bool
+  default     = false
+}
+
+variable "helm_config" {
+  description = "Self-managed aws-ebs-csi-driver Helm chart config"
+  type        = any
+  default     = {}
+}
diff --git a/modules/kubernetes-addons/main.tf b/modules/kubernetes-addons/main.tf
@@ -46,10 +46,18 @@ module "aws_kube_proxy" {
 }
 
 module "aws_ebs_csi_driver" {
-  count         = var.enable_amazon_eks_aws_ebs_csi_driver ? 1 : 0
-  source        = "./aws-ebs-csi-driver"
-  addon_config  = var.amazon_eks_aws_ebs_csi_driver_config
-  addon_context = local.addon_context
+  source = "./aws-ebs-csi-driver"
+
+  count = var.enable_amazon_eks_aws_ebs_csi_driver || var.enable_self_managed_aws_ebs_csi_driver ? 1 : 0
+
+  # Amazon EKS aws-ebs-csi-driver addon
+  enable_amazon_eks_aws_ebs_csi_driver = var.enable_amazon_eks_aws_ebs_csi_driver
+  addon_config                         = var.amazon_eks_aws_ebs_csi_driver_config
+  addon_context                        = local.addon_context
+
+  # Self-managed aws-ebs-csi-driver addon via Helm chart
+  enable_self_managed_aws_ebs_csi_driver = var.enable_self_managed_aws_ebs_csi_driver
+  helm_config                            = var.self_managed_aws_ebs_csi_driver_helm_config
 }
 
 #-----------------Kubernetes Add-ons----------------------

diff --git a/modules/kubernetes-addons/variables.tf b/modules/kubernetes-addons/variables.tf
@@ -119,11 +119,23 @@ variable "enable_amazon_eks_kube_proxy" {
 }
 
 variable "enable_amazon_eks_aws_ebs_csi_driver" {
-  description = "Enable EKS Managed AWS EBS CSI Driver add-on"
+  description = "Enable EKS Managed AWS EBS CSI Driver add-on; enable_amazon_eks_aws_ebs_csi_driver and enable_self_managed_aws_ebs_csi_driver are mutually exclusive"
   type        = bool
   default     = false
 }
 
+variable "enable_self_managed_aws_ebs_csi_driver" {
+  description = "Enable self-managed aws-ebs-csi-driver add-on; enable_self_managed_aws_ebs_csi_driver and enable_amazon_eks_aws_ebs_csi_driver are mutually exclusive"
+  type        = bool
+  default     = false
+}
+
+variable "self_managed_aws_ebs_csi_driver_helm_config" {
+  description = "Self-managed aws-ebs-csi-driver Helm chart config"
+  type        = any
+  default     = {}
+}
+
 variable "custom_image_registry_uri" {
   description = "Custom image registry URI map of `{region = dkr.endpoint }`"
   type        = map(string)