Merge pull request #6714 from ministryofjustice/feat/apc-karpenter

✨ APC Karpenter

terraform/environments/analytical-platform-compute/eks-cluster.tf

@@ -62,6 +62,10 @@ module "eks" {
     }
   }
 
+  node_security_group_tags = {
+    "karpenter.sh/discovery" = local.eks_cluster_name
+  }
+
   eks_managed_node_group_defaults = {
     ami_release_version = local.environment_configuration.eks_node_version
     ami_type            = "BOTTLEROCKET_x86_64"
@@ -82,7 +86,7 @@ module "eks" {
           iops                  = 3000
           throughput            = 150
           encrypted             = true
-          kms_key_id            = module.ebs_kms.key_arn
+          kms_key_id            = module.eks_ebs_kms.key_arn
           delete_on_termination = true
         }
       }
@@ -126,7 +130,7 @@ module "eks" {
             iops                  = 3000
             throughput            = 250
             encrypted             = true
-            kms_key_id            = module.ebs_kms.key_arn
+            kms_key_id            = module.eks_ebs_kms.key_arn
             delete_on_termination = true
           }
         }
@@ -161,3 +165,37 @@ module "eks" {
 
   tags = local.tags
 }
+
+module "karpenter" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/eks/aws//modules/karpenter"
+  version = "20.14.0"
+
+  cluster_name = module.eks.cluster_name
+
+  enable_pod_identity             = true
+  create_pod_identity_association = true
+
+  namespace = kubernetes_namespace.karpenter.metadata[0].name
+
+  queue_name                = "${module.eks.cluster_name}-karpenter"
+  queue_kms_master_key_id   = module.karpenter_sqs_kms.key_arn
+  queue_managed_sse_enabled = false
+
+  iam_policy_name = "karpenter"
+  iam_role_name   = "karpenter"
+  iam_role_policies = {
+    KarpenterSQSKMSAccess = module.karpenter_sqs_kms_access_iam_policy.arn
+  }
+
+  node_iam_role_name = "karpenter"
+  node_iam_role_additional_policies = {
+    AmazonSSMManagedInstanceCore  = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+    CloudWatchAgentServerPolicy   = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+    EKSClusterLogsKMSAccessPolicy = module.eks_cluster_logs_kms_access_iam_policy.arn
+  }
+
+  tags = local.tags
+}

terraform/environments/analytical-platform-compute/helm-charts-system.tf

@@ -110,6 +110,52 @@ resource "helm_release" "cluster_autoscaler" {
   depends_on = [module.cluster_autoscaler_iam_role]
 }
 
+/* Karpenter */
+resource "helm_release" "karpenter" {
+  /* https://github.com/aws/karpenter-provider-aws/releases */
+  name       = "karpenter"
+  repository = "oci://public.ecr.aws/karpenter"
+  chart      = "karpenter"
+  version    = "0.37.0"
+  namespace  = kubernetes_namespace.karpenter.metadata[0].name
+
+  values = [
+    templatefile(
+      "${path.module}/src/helm/values/karpenter/values.yml.tftpl",
+      {
+        service_account_name = module.karpenter.service_account
+        cluster_name         = module.eks.cluster_name
+        cluster_endpoint     = module.eks.cluster_endpoint
+        interruption_queue   = module.karpenter.queue_name
+      }
+    )
+  ]
+  depends_on = [
+    aws_iam_service_linked_role.spot,
+    module.karpenter
+  ]
+}
+
+resource "helm_release" "karpenter_configuration" {
+  name      = "karpenter-configuration"
+  chart     = "./src/helm/charts/karpenter-configuration"
+  namespace = kubernetes_namespace.karpenter.metadata[0].name
+
+  values = [
+    templatefile(
+      "${path.module}/src/helm/values/karpenter-configuration/values.yml.tftpl",
+      {
+        cluster_name    = module.eks.cluster_name
+        cluster_version = module.eks.cluster_version
+        ebs_kms_key_id  = module.eks_ebs_kms.key_id
+        node_role       = module.karpenter.node_iam_role_name
+        node_version    = local.environment_configuration.eks_node_version
+      }
+    )
+  ]
+  depends_on = [helm_release.karpenter]
+}
+
 /* External DNS */
 resource "helm_release" "external_dns" {
   /* https://artifacthub.io/packages/helm/external-dns/external-dns */

terraform/environments/analytical-platform-compute/iam-policies.tf

@@ -25,6 +25,33 @@ module "eks_cluster_logs_kms_access_iam_policy" {
   policy = data.aws_iam_policy_document.eks_cluster_logs_kms_access.json
 }
 
+data "aws_iam_policy_document" "karpenter_sqs_kms_access" {
+  statement {
+    sid    = "AllowKMS"
+    effect = "Allow"
+    actions = [
+      "kms:Encrypt*",
+      "kms:Decrypt*",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Describe*"
+    ]
+    resources = [module.karpenter_sqs_kms.key_arn]
+  }
+}
+
+module "karpenter_sqs_kms_access_iam_policy" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
+  version = "5.39.1"
+
+  name_prefix = "karpenter-sqs-kms-access"
+
+  policy = data.aws_iam_policy_document.karpenter_sqs_kms_access.json
+}
+
 data "aws_iam_policy_document" "amazon_prometheus_proxy" {
   statement {
     sid    = "AllowAPS"

terraform/environments/analytical-platform-compute/kms-keys.tf

@@ -5,7 +5,7 @@ module "vpc_flow_logs_kms" {
   source  = "terraform-aws-modules/kms/aws"
   version = "3.1.0"
 
-  aliases                 = ["vpc-flow-logs"]
+  aliases                 = ["vpc/flow-logs"]
   description             = "VPC flow logs KMS key"
   enable_default_policy   = true
   deletion_window_in_days = 7
@@ -47,7 +47,7 @@ module "managed_prometheus_kms" {
   source  = "terraform-aws-modules/kms/aws"
   version = "3.1.0"
 
-  aliases                 = ["managed-prometheus"]
+  aliases                 = ["amp/default"]
   description             = "AMP KMS key"
   enable_default_policy   = true
   deletion_window_in_days = 7
@@ -93,7 +93,7 @@ module "managed_prometheus_logs_kms" {
   source  = "terraform-aws-modules/kms/aws"
   version = "3.1.0"
 
-  aliases                 = ["managed-prometheus-logs"]
+  aliases                 = ["amp/logs"]
   description             = "AMP logs KMS key"
   enable_default_policy   = true
   deletion_window_in_days = 7
@@ -135,7 +135,7 @@ module "eks_cluster_logs_kms" {
   source  = "terraform-aws-modules/kms/aws"
   version = "3.1.0"
 
-  aliases                 = ["eks-cluster-logs"]
+  aliases                 = ["eks/cluster-logs"]
   description             = "EKS cluster logs KMS key"
   enable_default_policy   = true
   deletion_window_in_days = 7
@@ -170,25 +170,63 @@ module "eks_cluster_logs_kms" {
   tags = local.tags
 }
 
-module "ebs_kms" {
+module "eks_ebs_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/kms/aws"
   version = "3.1.0"
 
-  aliases                 = ["eks-ebs"]
+  aliases                 = ["eks/ebs"]
   description             = "EKS EBS KMS key"
   enable_default_policy   = true
   deletion_window_in_days = 7
   key_service_roles_for_autoscaling = [
     "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling",
     module.eks.cluster_iam_role_arn
   ]
+  key_statements = [
+    {
+      sid = "AllowEC2"
+      actions = [
+        "kms:Encrypt",
+        "kms:Decrypt",
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:CreateGrant",
+        "kms:DescribeKey"
+      ]
+      resources = ["*"]
+      effect    = "Allow"
+      principals = [
+        {
+          type        = "AWS"
+          identifiers = ["*"]
+        }
+      ]
+      conditions = [
+        {
+          test     = "StringEquals"
+          variable = "kms:ViaService"
+          values   = ["ec2.${data.aws_region.current.name}.amazonaws.com"]
+        },
+        {
+          test     = "StringEquals"
+          variable = "kms:CallerAccount"
+          values   = [data.aws_caller_identity.current.account_id]
+        }
+      ]
+    }
+  ]
 
   tags = local.tags
 }
 
+moved {
+  from = module.ebs_kms
+  to   = module.eks_ebs_kms
+}
+
 module "mlflow_auth_rds_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
@@ -252,3 +290,36 @@ module "common_secrets_manager_kms" {
 
   tags = local.tags
 }
+
+module "karpenter_sqs_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/kms/aws"
+  version = "3.1.0"
+
+  aliases               = ["sqs/karpenter"]
+  description           = "Karpenter SQS KMS key"
+  enable_default_policy = true
+  key_statements = [
+    {
+      sid = "AllowAmazonEventBridge"
+      actions = [
+        "kms:GenerateDataKey",
+        "kms:Decrypt"
+      ]
+      resources = ["*"]
+      effect    = "Allow"
+      principals = [
+        {
+          type        = "Service"
+          identifiers = ["events.amazonaws.com"]
+        }
+      ]
+    }
+  ]
+
+  deletion_window_in_days = 7
+
+  tags = local.tags
+}

terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf

@@ -16,6 +16,12 @@ resource "kubernetes_namespace" "cluster_autoscaler" {
   }
 }
 
+resource "kubernetes_namespace" "karpenter" {
+  metadata {
+    name = "karpenter"
+  }
+}
+
 resource "kubernetes_namespace" "external_dns" {
   metadata {
     name = "external-dns"

terraform/environments/analytical-platform-compute/service-linked-roles.tf

@@ -0,0 +1,3 @@
+resource "aws_iam_service_linked_role" "spot" {
+  aws_service_name = "spot.amazonaws.com"
+}

terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: v2
+name: karpenter-configuration
+description: A Helm chart to deploy Karpenter's configuration
+type: application
+version: 1.0.0

terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: bottlerocket-general
+spec:
+  amiFamily: Bottlerocket
+  role: {{ .Values.nodeRole }}
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: {{ .Values.clusterName }}
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: {{ .Values.clusterName }}
+  amiSelectorTerms:
+    - name: "bottlerocket-aws-k8s-{{ .Values.clusterVersion  }}-x86_64-v{{ .Values.nodeVersion }}"
+  metadataOptions:
+    httpEndpoint: enabled
+    httpPutResponseHopLimit: 1
+    httpTokens: required
+  blockDeviceMappings:
+    - deviceName: /dev/xvdb
+      ebs:
+        volumeSize: 100Gi
+        volumeType: gp3
+        iops: 3000
+        encrypted: true
+        kmsKeyID: {{ .Values.ebsKmsKeyId }}
+        deleteOnTermination: true
+        throughput: 125
+  detailedMonitoring: true

terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml

@@ -0,0 +1,35 @@
+---
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: general-on-demand
+spec:
+  template:
+    metadata:
+      labels:
+        compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "general-on-demand"
+    spec:
+      nodeClassRef:
+        apiVersion: karpenter.k8s.aws/v1beta1
+        kind: EC2NodeClass
+        name: bottlerocket-general
+      taints:
+        - key: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool
+          value: "general-on-demand"
+          effect: NoSchedule
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["on-demand"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]