From 87061712d593d993d7b121a2a08cbda4cb94f0f0 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 16:38:52 +0000 Subject: [PATCH 01/24] Add Karpenter module Signed-off-by: Jacob Woffenden --- .../eks-cluster.tf | 26 +++++++++++++++++++ .../analytical-platform-compute/kms-keys.tf | 16 ++++++++++++ .../kubernetes-namespaces.tf | 6 +++++ 3 files changed, 48 insertions(+) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index ac0211e491e..de2959fdfa1 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -161,3 +161,29 @@ module "eks" { tags = local.tags } + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + version = "20.14.0" + + cluster_name = module.eks.cluster_name + + enable_pod_identity = true + create_pod_identity_association = true + + iam_policy_name_prefix = "karpenter" + iam_role_name = "karpenter" + + namespace = kubernetes_namespace.karpenter.metadata[0].name + + queue_name = "${module.eks.cluster_name}-karpenter" + queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn + + node_iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + CloudWatchAgentServerPolicy = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + EKSClusterLogsKMSAccessPolicy = module.eks_cluster_logs_kms_access_iam_policy.arn + } + + tags = local.tags +} diff --git a/terraform/environments/analytical-platform-compute/kms-keys.tf b/terraform/environments/analytical-platform-compute/kms-keys.tf index 7827a4bcf29..512791a133e 100644 --- a/terraform/environments/analytical-platform-compute/kms-keys.tf +++ b/terraform/environments/analytical-platform-compute/kms-keys.tf @@ -252,3 +252,19 @@ module "common_secrets_manager_kms" { tags = local.tags } + +module "karpenter_sqs_kms" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + #checkov:skip=CKV_TF_2:Module registry does not support tags for versions + + source = "terraform-aws-modules/kms/aws" + version = "3.1.0" + + aliases = ["sqs/karpenter"] + description = "Karpenter SQS KMS key" + enable_default_policy = true + + deletion_window_in_days = 7 + + tags = local.tags +} diff --git a/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf b/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf index 68e6856f80e..ed619ac40e7 100644 --- a/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf +++ b/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf @@ -16,6 +16,12 @@ resource "kubernetes_namespace" "cluster_autoscaler" { } } +resource "kubernetes_namespace" "karpenter" { + metadata { + name = "karpernter" + } +} + resource "kubernetes_namespace" "external_dns" { metadata { name = "external-dns" From f7f3457161b7176043f0aa5ee9ffea915ce0b64e Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 16:42:11 +0000 Subject: [PATCH 02/24] fix policy var Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index de2959fdfa1..22f559937ba 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -171,8 +171,8 @@ module "karpenter" { enable_pod_identity = true create_pod_identity_association = true - iam_policy_name_prefix = "karpenter" - iam_role_name = "karpenter" + iam_policy_name = "karpenter" + iam_role_name = "karpenter" namespace = kubernetes_namespace.karpenter.metadata[0].name From d62bf679bfff50e8190081399b54ae0beb3dfcd9 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 17:46:18 +0000 Subject: [PATCH 03/24] don't use prefix Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 22f559937ba..a8919276678 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -179,6 +179,8 @@ module "karpenter" { queue_name = "${module.eks.cluster_name}-karpenter" queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn + node_iam_role_use_name_prefix = false + node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" CloudWatchAgentServerPolicy = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" From 03857d70f34ec9b6f82b0d17c593b1439a740b55 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 17:56:37 +0000 Subject: [PATCH 04/24] update node role name Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index a8919276678..1456269d675 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -179,6 +179,7 @@ module "karpenter" { queue_name = "${module.eks.cluster_name}-karpenter" queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn + node_iam_role_name = "karpenter-node-${module.eks.cluster_name}" node_iam_role_use_name_prefix = false node_iam_role_additional_policies = { From 35d4c68aa574ac7cf8e83968fc4434492988a728 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 18:16:38 +0000 Subject: [PATCH 05/24] update SQS Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 1456269d675..8dafdb420d4 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -178,6 +178,7 @@ module "karpenter" { queue_name = "${module.eks.cluster_name}-karpenter" queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn + sqs_managed_sse_enabled = false node_iam_role_name = "karpenter-node-${module.eks.cluster_name}" node_iam_role_use_name_prefix = false From 1fbeca3b5677e4045622f6a6ba23ca1478504081 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 18:21:34 +0000 Subject: [PATCH 06/24] update SQS Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 8dafdb420d4..e1d719abfe1 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -176,9 +176,9 @@ module "karpenter" { namespace = kubernetes_namespace.karpenter.metadata[0].name - queue_name = "${module.eks.cluster_name}-karpenter" - queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn - sqs_managed_sse_enabled = false + queue_name = "${module.eks.cluster_name}-karpenter" + queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn + queue_managed_sse_enabled = false node_iam_role_name = "karpenter-node-${module.eks.cluster_name}" node_iam_role_use_name_prefix = false From 8821b9db9945617d525d84de9205dd3fecebf994 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 18:31:13 +0000 Subject: [PATCH 07/24] update node role name Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/eks-cluster.tf | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index e1d719abfe1..54e0e86a638 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -171,17 +171,15 @@ module "karpenter" { enable_pod_identity = true create_pod_identity_association = true - iam_policy_name = "karpenter" - iam_role_name = "karpenter" - namespace = kubernetes_namespace.karpenter.metadata[0].name queue_name = "${module.eks.cluster_name}-karpenter" queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn queue_managed_sse_enabled = false - node_iam_role_name = "karpenter-node-${module.eks.cluster_name}" - node_iam_role_use_name_prefix = false + iam_policy_name = "karpenter" + iam_role_name = "karpenter" + node_iam_role_name = "karpenter" node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" From 0f57273f03f9c9da5df02453af6b7c34f13fd137 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 18:56:41 +0000 Subject: [PATCH 08/24] Add Karpenter chart Signed-off-by: Jacob Woffenden --- .../eks-cluster.tf | 3 +++ .../helm-charts-system.tf | 23 +++++++++++++++++++ .../helm/values/karpenter/values.yml.tftpl | 8 +++++++ .../src/helm/values/kyverno/values.yml.tftpl | 1 + 4 files changed, 35 insertions(+) create mode 100644 terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 54e0e86a638..3039e309b81 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -163,6 +163,9 @@ module "eks" { } module "karpenter" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + #checkov:skip=CKV_TF_2:Module registry does not support tags for versions + source = "terraform-aws-modules/eks/aws//modules/karpenter" version = "20.14.0" diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf index 831c609ade3..4f74733156c 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-system.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-system.tf @@ -110,6 +110,29 @@ resource "helm_release" "cluster_autoscaler" { depends_on = [module.cluster_autoscaler_iam_role] } +/* Karpenter */ +resource "helm_release" "karpenter" { + /* https://github.com/aws/karpenter-provider-aws/releases */ + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = "0.37.0" + namespace = kubernetes_namespace.karpenter.metadata[0].name + + values = [ + templatefile( + "${path.module}/src/helm/values/karpenter/values.yml.tftpl", + { + service_account_name = module.karpenter.service_account + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + interruption_queue = module.karpenter.queue_name + } + ) + ] + depends_on = [module.karpenter] +} + /* External DNS */ resource "helm_release" "external_dns" { /* https://artifacthub.io/packages/helm/external-dns/external-dns */ diff --git a/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl new file mode 100644 index 00000000000..3ad63b6f66e --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl @@ -0,0 +1,8 @@ +--- +serviceAccount: + name: ${service_account_name} + +settings: + clusterName: ${cluster_name} + clusterEndpoint: ${cluster_endpoint} + interruptionQueue: ${interruption_queue} diff --git a/terraform/environments/analytical-platform-compute/src/helm/values/kyverno/values.yml.tftpl b/terraform/environments/analytical-platform-compute/src/helm/values/kyverno/values.yml.tftpl index 58b11b44965..eca53c97d35 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/values/kyverno/values.yml.tftpl +++ b/terraform/environments/analytical-platform-compute/src/helm/values/kyverno/values.yml.tftpl @@ -22,6 +22,7 @@ config: - amazon-guardduty - aws-observability - cluster-autoscaler + - karpenter - external-dns - cert-manager - ingress-nginx From d75c62cbba921ad4cb5dd5b92ad6c7f030a398d1 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 19:06:05 +0000 Subject: [PATCH 09/24] Enable Karpenter's service monitor Signed-off-by: Jacob Woffenden --- .../src/helm/values/karpenter/values.yml.tftpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl index 3ad63b6f66e..59a679cdff4 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl +++ b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter/values.yml.tftpl @@ -2,6 +2,9 @@ serviceAccount: name: ${service_account_name} +serviceMonitor: + enabled: true + settings: clusterName: ${cluster_name} clusterEndpoint: ${cluster_endpoint} From 3fa47437f671e9f97243d78cb9304aa4677dd363 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 19:10:47 +0000 Subject: [PATCH 10/24] Tag EKS resources Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 3039e309b81..07b196486ce 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -159,7 +159,9 @@ module "eks" { } } - tags = local.tags + tags = merge(local.tags, { + "karpenter.sh/discovery" = local.eks_cluster_name + }) } module "karpenter" { From 947675dc87782c6916033f8293c7a987094d2d4a Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 19:11:35 +0000 Subject: [PATCH 11/24] Tag private subnets Signed-off-by: Jacob Woffenden --- terraform/environments/analytical-platform-compute/vpc.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/environments/analytical-platform-compute/vpc.tf b/terraform/environments/analytical-platform-compute/vpc.tf index 6d6e08472d5..2c2daf4653b 100644 --- a/terraform/environments/analytical-platform-compute/vpc.tf +++ b/terraform/environments/analytical-platform-compute/vpc.tf @@ -37,6 +37,7 @@ module "vpc" { private_subnet_tags = { "kubernetes.io/role/internal-elb" = 1 + "karpenter.sh/discovery" = local.eks_cluster_name } tags = local.tags From be82ead7c5209cc6127f8a3dcd6674d3e2de3be1 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 24 Jun 2024 23:59:19 +0000 Subject: [PATCH 12/24] Update SQS KMS policy Fix NS name Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/kms-keys.tf | 17 +++++++++++++++++ .../kubernetes-namespaces.tf | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/kms-keys.tf b/terraform/environments/analytical-platform-compute/kms-keys.tf index 512791a133e..d5c756ca4ec 100644 --- a/terraform/environments/analytical-platform-compute/kms-keys.tf +++ b/terraform/environments/analytical-platform-compute/kms-keys.tf @@ -263,6 +263,23 @@ module "karpenter_sqs_kms" { aliases = ["sqs/karpenter"] description = "Karpenter SQS KMS key" enable_default_policy = true + key_statements = [ + { + sid = "AllowAmazonEventBridge" + actions = [ + "kms:GenerateDataKey", + "kms:Decrypt" + ] + resources = ["*"] + effect = "Allow" + principals = [ + { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + ] + } + ] deletion_window_in_days = 7 diff --git a/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf b/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf index ed619ac40e7..d7fa8e10248 100644 --- a/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf +++ b/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf @@ -18,7 +18,7 @@ resource "kubernetes_namespace" "cluster_autoscaler" { resource "kubernetes_namespace" "karpenter" { metadata { - name = "karpernter" + name = "karpenter" } } From cf0bf78c527a1b6fa40debcc6cd64a78974dcafd Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 06:05:33 +0000 Subject: [PATCH 13/24] Update EBS KMS Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/kms-keys.tf | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/terraform/environments/analytical-platform-compute/kms-keys.tf b/terraform/environments/analytical-platform-compute/kms-keys.tf index d5c756ca4ec..0f2a3821d57 100644 --- a/terraform/environments/analytical-platform-compute/kms-keys.tf +++ b/terraform/environments/analytical-platform-compute/kms-keys.tf @@ -185,6 +185,39 @@ module "ebs_kms" { "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", module.eks.cluster_iam_role_arn ] + key_statements = [ + { + sid = "AllowEC2" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:CreateGrant", + "kms:DescribeKey" + ] + resources = ["*"] + effect = "Allow" + principals = [ + { + type = "AWS" + identifiers = ["*"] + } + ] + conditions = [ + { + test = "StringEquals" + variable = "kms:ViaService" + values = ["ec2.${data.aws_region.current.name}.amazonaws.com"] + }, + { + test = "StringEquals" + variable = "kms:CallerAccount" + values = [data.aws_caller_identity.current.account_id] + } + ] + } + ] tags = local.tags } From 75835a55bd5894fe03076c6b25a10693ee6dd097 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 06:13:00 +0000 Subject: [PATCH 14/24] Add service linked roles for AWS spot Update KMS aliases Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/eks-cluster.tf | 4 ++-- .../analytical-platform-compute/kms-keys.tf | 17 +++++++++++------ .../service-linked-roles.tf | 3 +++ 3 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 terraform/environments/analytical-platform-compute/service-linked-roles.tf diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 07b196486ce..bb184fbf01c 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -82,7 +82,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = module.ebs_kms.key_arn + kms_key_id = module.eks_ebs_kms.key_arn delete_on_termination = true } } @@ -126,7 +126,7 @@ module "eks" { iops = 3000 throughput = 250 encrypted = true - kms_key_id = module.ebs_kms.key_arn + kms_key_id = module.eks_ebs_kms.key_arn delete_on_termination = true } } diff --git a/terraform/environments/analytical-platform-compute/kms-keys.tf b/terraform/environments/analytical-platform-compute/kms-keys.tf index 0f2a3821d57..e4e8b4f0927 100644 --- a/terraform/environments/analytical-platform-compute/kms-keys.tf +++ b/terraform/environments/analytical-platform-compute/kms-keys.tf @@ -5,7 +5,7 @@ module "vpc_flow_logs_kms" { source = "terraform-aws-modules/kms/aws" version = "3.1.0" - aliases = ["vpc-flow-logs"] + aliases = ["vpc/flow-logs"] description = "VPC flow logs KMS key" enable_default_policy = true deletion_window_in_days = 7 @@ -47,7 +47,7 @@ module "managed_prometheus_kms" { source = "terraform-aws-modules/kms/aws" version = "3.1.0" - aliases = ["managed-prometheus"] + aliases = ["amp/default"] description = "AMP KMS key" enable_default_policy = true deletion_window_in_days = 7 @@ -93,7 +93,7 @@ module "managed_prometheus_logs_kms" { source = "terraform-aws-modules/kms/aws" version = "3.1.0" - aliases = ["managed-prometheus-logs"] + aliases = ["amp/logs"] description = "AMP logs KMS key" enable_default_policy = true deletion_window_in_days = 7 @@ -135,7 +135,7 @@ module "eks_cluster_logs_kms" { source = "terraform-aws-modules/kms/aws" version = "3.1.0" - aliases = ["eks-cluster-logs"] + aliases = ["eks/cluster-logs"] description = "EKS cluster logs KMS key" enable_default_policy = true deletion_window_in_days = 7 @@ -170,14 +170,14 @@ module "eks_cluster_logs_kms" { tags = local.tags } -module "ebs_kms" { +module "eks_ebs_kms" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/kms/aws" version = "3.1.0" - aliases = ["eks-ebs"] + aliases = ["eks/ebs"] description = "EKS EBS KMS key" enable_default_policy = true deletion_window_in_days = 7 @@ -222,6 +222,11 @@ module "ebs_kms" { tags = local.tags } +moved { + from = module.ebs_kms + to = module.eks_ebs_kms +} + module "mlflow_auth_rds_kms" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions #checkov:skip=CKV_TF_2:Module registry does not support tags for versions diff --git a/terraform/environments/analytical-platform-compute/service-linked-roles.tf b/terraform/environments/analytical-platform-compute/service-linked-roles.tf new file mode 100644 index 00000000000..579180bae08 --- /dev/null +++ b/terraform/environments/analytical-platform-compute/service-linked-roles.tf @@ -0,0 +1,3 @@ +resource "aws_iam_service_linked_role" "spot" { + aws_service_name = "spot.amazonaws.com" +} From f374dedbd8f473ed666ddd587b43047cb90b2658 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 06:13:18 +0000 Subject: [PATCH 15/24] update eks ebs kms ref Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index bb184fbf01c..07b196486ce 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -82,7 +82,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = module.eks_ebs_kms.key_arn + kms_key_id = module.ebs_kms.key_arn delete_on_termination = true } } @@ -126,7 +126,7 @@ module "eks" { iops = 3000 throughput = 250 encrypted = true - kms_key_id = module.eks_ebs_kms.key_arn + kms_key_id = module.ebs_kms.key_arn delete_on_termination = true } } From 555eac3572c9f941aae7141e1f9fb2f6c00701ff Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 06:16:18 +0000 Subject: [PATCH 16/24] fixes Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-compute/eks-cluster.tf | 4 ++-- .../analytical-platform-compute/helm-charts-system.tf | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 07b196486ce..bb184fbf01c 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -82,7 +82,7 @@ module "eks" { iops = 3000 throughput = 150 encrypted = true - kms_key_id = module.ebs_kms.key_arn + kms_key_id = module.eks_ebs_kms.key_arn delete_on_termination = true } } @@ -126,7 +126,7 @@ module "eks" { iops = 3000 throughput = 250 encrypted = true - kms_key_id = module.ebs_kms.key_arn + kms_key_id = module.eks_ebs_kms.key_arn delete_on_termination = true } } diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf index 4f74733156c..21765afa8d8 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-system.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-system.tf @@ -130,7 +130,10 @@ resource "helm_release" "karpenter" { } ) ] - depends_on = [module.karpenter] + depends_on = [ + aws_iam_service_linked_role.spot, + module.karpenter + ] } /* External DNS */ From 235d945a2b67b632f4aebc077b94075d3a73dc68 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 07:29:41 +0000 Subject: [PATCH 17/24] remove karpenter tag from cluster, mng didn't like it Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/eks-cluster.tf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index bb184fbf01c..48e25eb8375 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -62,6 +62,10 @@ module "eks" { } } + node_security_group_tags = { + "karpenter.sh/discovery" = local.eks_cluster_name + } + eks_managed_node_group_defaults = { ami_release_version = local.environment_configuration.eks_node_version ami_type = "BOTTLEROCKET_x86_64" @@ -159,9 +163,7 @@ module "eks" { } } - tags = merge(local.tags, { - "karpenter.sh/discovery" = local.eks_cluster_name - }) + tags = local.tags } module "karpenter" { From 34f33d394d6e7292f6dc4df6c47d42d15e104a1d Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 12:29:37 +0000 Subject: [PATCH 18/24] Add Karpenter SQS KMS access policy Signed-off-by: Jacob Woffenden --- .../eks-cluster.tf | 9 ++++--- .../iam-policies.tf | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 48e25eb8375..d774bb45141 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -184,10 +184,13 @@ module "karpenter" { queue_kms_master_key_id = module.karpenter_sqs_kms.key_arn queue_managed_sse_enabled = false - iam_policy_name = "karpenter" - iam_role_name = "karpenter" - node_iam_role_name = "karpenter" + iam_policy_name = "karpenter" + iam_role_name = "karpenter" + iam_role_policies = { + KarpenterSQSKMSAccess = module.karpenter_sqs_kms_access_iam_policy.arn + } + node_iam_role_name = "karpenter" node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" CloudWatchAgentServerPolicy = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" diff --git a/terraform/environments/analytical-platform-compute/iam-policies.tf b/terraform/environments/analytical-platform-compute/iam-policies.tf index 735f8911132..5aae2688939 100644 --- a/terraform/environments/analytical-platform-compute/iam-policies.tf +++ b/terraform/environments/analytical-platform-compute/iam-policies.tf @@ -25,6 +25,33 @@ module "eks_cluster_logs_kms_access_iam_policy" { policy = data.aws_iam_policy_document.eks_cluster_logs_kms_access.json } +data "aws_iam_policy_document" "karpenter_sqs_kms_access" { + statement { + sid = "AllowKMS" + effect = "Allow" + actions = [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ] + resources = [module.karpenter_sqs_kms.key_arn] + } +} + +module "karpenter_sqs_kms_access_iam_policy" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + #checkov:skip=CKV_TF_2:Module registry does not support tags for versions + + source = "terraform-aws-modules/iam/aws//modules/iam-policy" + version = "5.39.1" + + name_prefix = "karpenter-sqs-kms-access" + + policy = data.aws_iam_policy_document.karpenter_sqs_kms_access.json +} + data "aws_iam_policy_document" "amazon_prometheus_proxy" { statement { sid = "AllowAPS" From 8b221b310e7df1636e121aab95960bf3b5d9893a Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 13:46:55 +0000 Subject: [PATCH 19/24] Add Karpenter Config chart Signed-off-by: Jacob Woffenden --- .../helm-charts-system.tf | 20 +++++++++++ .../charts/karpenter-configuration/Chart.yaml | 6 ++++ .../ec2-node-class-bottlerocket-general.yaml | 31 +++++++++++++++++ .../node-pool-general-on-demand.yaml | 34 +++++++++++++++++++ .../karpenter-configuration/values.yaml | 8 +++++ .../karpenter-configuration/values.yml.tftpl | 8 +++++ 6 files changed, 107 insertions(+) create mode 100644 terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml create mode 100644 terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml create mode 100644 terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml create mode 100644 terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/values.yaml create mode 100644 terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf index 21765afa8d8..7f39171163d 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-system.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-system.tf @@ -136,6 +136,26 @@ resource "helm_release" "karpenter" { ] } +resource "helm_release" "karpenter_configuration" { + name = "karpenter-configuration" + chart = "./src/helm/charts/arpenter-configuration" + namespace = kubernetes_namespace.karpenter.metadata[0].name + + values = [ + templatefile( + "${path.module}/src/helm/values/karpenter-configuration/values.yml.tftpl", + { + cluster_name = module.eks.cluster_name + cluster_version = module.eks.cluster_version + ebs_kms_key_id = module.eks_ebs_kms.key_arn + node_role = module.karpenter.node_iam_role_name + node_version = local.environment_configuration.eks_node_version + } + ) + ] + depends_on = [helm_release.karpenter] +} + /* External DNS */ resource "helm_release" "external_dns" { /* https://artifacthub.io/packages/helm/external-dns/external-dns */ diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml new file mode 100644 index 00000000000..8570582482b --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/Chart.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v2 +name: karpenter-configuration +description: A Helm chart to deploy Karpenter's configuration +type: application +version: 1.0.0 diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml new file mode 100644 index 00000000000..2b0e1a0af4e --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: bottlerocket-general +spec: + amiFamily: Bottlerocket + role: {{ .Values.nodeRole }} # karpenter-20240624183657747400000001 + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: {{ .Values.clusterName }} # analytical-platform-compute-development + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: {{ .Values.clusterName }} # analytical-platform-compute-development + amiSelectorTerms: + - name: "bottlerocket-aws-k8s-{{ .Values.clusterVersion }}-x86_64-v{{ .Values.nodeVersion }}" # bottlerocket-aws-k8s-1.30-x86_64-v1.20.2-536d69d0 + metadataOptions: + httpEndpoint: enabled + httpPutResponseHopLimit: 1 + httpTokens: required + blockDeviceMappings: + - deviceName: /dev/xvdb + ebs: + volumeSize: 100Gi + volumeType: gp3 + iops: 3000 + encrypted: true + kmsKeyID: {{ .Values.ebsKmsKeyId }} # 30b0a8b7-26d7-4307-9943-f28421dbed7f + deleteOnTermination: true + throughput: 125 + detailedMonitoring: true diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml new file mode 100644 index 00000000000..f20f110c928 --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: general-on-demand +spec: + template: + metadata: + labels: + compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "general-on-demand" + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: bottlerocket + # taints: + # - key: compute.analytical-platform.service.justice.gov.uk/karpenter + # effect: NoSchedule + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/values.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/values.yaml new file mode 100644 index 00000000000..90ad6690fab --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/values.yaml @@ -0,0 +1,8 @@ +--- +clusterName: +clusterVersion: + +ebsKmsKeyId: + +nodeRole: +nodeVersion: diff --git a/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl new file mode 100644 index 00000000000..9a270223f3e --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl @@ -0,0 +1,8 @@ +--- +clusterName: ${cluster_name} +clusterVersion: ${cluster_version} + +ebsKmsKeyId: ${ebs_kms_key_id} + +nodeRole: ${node_role} +nodeVersion: ${node_version} From 5d0390e1631328937548cd5b4966eb0e8c1bb024 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 13:55:12 +0000 Subject: [PATCH 20/24] LOL Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/helm-charts-system.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf index 7f39171163d..5bdc5e423c2 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-system.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-system.tf @@ -138,7 +138,7 @@ resource "helm_release" "karpenter" { resource "helm_release" "karpenter_configuration" { name = "karpenter-configuration" - chart = "./src/helm/charts/arpenter-configuration" + chart = "./src/helm/charts/karpenter-configuration" namespace = kubernetes_namespace.karpenter.metadata[0].name values = [ From ef450f5d9d2a27681f8d06b3dfcb70a9bbd5cd47 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 25 Jun 2024 14:28:29 +0000 Subject: [PATCH 21/24] fix cluster version rendering Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/helm-charts-system.tf | 2 +- .../templates/ec2-node-class-bottlerocket-general.yaml | 2 +- .../src/helm/values/karpenter-configuration/values.yml.tftpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/helm-charts-system.tf b/terraform/environments/analytical-platform-compute/helm-charts-system.tf index 5bdc5e423c2..3b9a55e6fd7 100644 --- a/terraform/environments/analytical-platform-compute/helm-charts-system.tf +++ b/terraform/environments/analytical-platform-compute/helm-charts-system.tf @@ -147,7 +147,7 @@ resource "helm_release" "karpenter_configuration" { { cluster_name = module.eks.cluster_name cluster_version = module.eks.cluster_version - ebs_kms_key_id = module.eks_ebs_kms.key_arn + ebs_kms_key_id = module.eks_ebs_kms.key_id node_role = module.karpenter.node_iam_role_name node_version = local.environment_configuration.eks_node_version } diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml index 2b0e1a0af4e..bd9ffb95c9a 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml @@ -13,7 +13,7 @@ spec: - tags: karpenter.sh/discovery: {{ .Values.clusterName }} # analytical-platform-compute-development amiSelectorTerms: - - name: "bottlerocket-aws-k8s-{{ .Values.clusterVersion }}-x86_64-v{{ .Values.nodeVersion }}" # bottlerocket-aws-k8s-1.30-x86_64-v1.20.2-536d69d0 + - name: "bottlerocket-aws-k8s-{{ .Values.clusterVersion }}-x86_64-v{{ .Values.nodeVersion }}" # bottlerocket-aws-k8s-1.30-x86_64-v1.20.2-536d69d0 metadataOptions: httpEndpoint: enabled httpPutResponseHopLimit: 1 diff --git a/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl index 9a270223f3e..ad38577f6cd 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl +++ b/terraform/environments/analytical-platform-compute/src/helm/values/karpenter-configuration/values.yml.tftpl @@ -1,6 +1,6 @@ --- clusterName: ${cluster_name} -clusterVersion: ${cluster_version} +clusterVersion: "${cluster_version}" ebsKmsKeyId: ${ebs_kms_key_id} From 273f9130eccfcb8bce884b7747ee2af49827f74e Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 26 Jun 2024 10:05:53 +0000 Subject: [PATCH 22/24] Remove old hardcoded vals Add spot Add taint Signed-off-by: Jacob Woffenden --- .../ec2-node-class-bottlerocket-general.yaml | 10 +++--- .../node-pool-general-on-demand.yaml | 9 ++--- .../templates/node-pool-general-spot.yaml | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml index bd9ffb95c9a..710a0e0f9f4 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/ec2-node-class-bottlerocket-general.yaml @@ -5,15 +5,15 @@ metadata: name: bottlerocket-general spec: amiFamily: Bottlerocket - role: {{ .Values.nodeRole }} # karpenter-20240624183657747400000001 + role: {{ .Values.nodeRole }} subnetSelectorTerms: - tags: - karpenter.sh/discovery: {{ .Values.clusterName }} # analytical-platform-compute-development + karpenter.sh/discovery: {{ .Values.clusterName }} securityGroupSelectorTerms: - tags: - karpenter.sh/discovery: {{ .Values.clusterName }} # analytical-platform-compute-development + karpenter.sh/discovery: {{ .Values.clusterName }} amiSelectorTerms: - - name: "bottlerocket-aws-k8s-{{ .Values.clusterVersion }}-x86_64-v{{ .Values.nodeVersion }}" # bottlerocket-aws-k8s-1.30-x86_64-v1.20.2-536d69d0 + - name: "bottlerocket-aws-k8s-{{ .Values.clusterVersion }}-x86_64-v{{ .Values.nodeVersion }}" metadataOptions: httpEndpoint: enabled httpPutResponseHopLimit: 1 @@ -25,7 +25,7 @@ spec: volumeType: gp3 iops: 3000 encrypted: true - kmsKeyID: {{ .Values.ebsKmsKeyId }} # 30b0a8b7-26d7-4307-9943-f28421dbed7f + kmsKeyID: {{ .Values.ebsKmsKeyId }} deleteOnTermination: true throughput: 125 detailedMonitoring: true diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml index f20f110c928..4115df60be4 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml @@ -13,9 +13,10 @@ spec: apiVersion: karpenter.k8s.aws/v1beta1 kind: EC2NodeClass name: bottlerocket - # taints: - # - key: compute.analytical-platform.service.justice.gov.uk/karpenter - # effect: NoSchedule + taints: + - key: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool + value: "general-on-demand" + effect: NoSchedule requirements: - key: kubernetes.io/arch operator: In @@ -25,7 +26,7 @@ spec: values: ["linux"] - key: karpenter.sh/capacity-type operator: In - values: ["spot"] + values: ["on-demand"] - key: karpenter.k8s.aws/instance-category operator: In values: ["c", "m", "r"] diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml new file mode 100644 index 00000000000..ab7d117c300 --- /dev/null +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: general-spot +spec: + template: + metadata: + labels: + compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool: "general-spot" + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: bottlerocket + taints: + - key: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool + value: "general-spot" + effect: NoSchedule + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] From 13625b19ab78863b917d6dba2108e3e4a215e526 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 26 Jun 2024 10:24:42 +0000 Subject: [PATCH 23/24] fix nodeclass ref Signed-off-by: Jacob Woffenden --- .../templates/node-pool-general-spot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml index ab7d117c300..57ec2829e18 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-spot.yaml @@ -12,7 +12,7 @@ spec: nodeClassRef: apiVersion: karpenter.k8s.aws/v1beta1 kind: EC2NodeClass - name: bottlerocket + name: bottlerocket-general taints: - key: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool value: "general-spot" From 390829bb51952b3642639cbf439fdf02b7276078 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 26 Jun 2024 10:28:34 +0000 Subject: [PATCH 24/24] fix nodeclass ref 2 Signed-off-by: Jacob Woffenden --- .../templates/node-pool-general-on-demand.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml index 4115df60be4..4d1f2abb596 100644 --- a/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml +++ b/terraform/environments/analytical-platform-compute/src/helm/charts/karpenter-configuration/templates/node-pool-general-on-demand.yaml @@ -12,7 +12,7 @@ spec: nodeClassRef: apiVersion: karpenter.k8s.aws/v1beta1 kind: EC2NodeClass - name: bottlerocket + name: bottlerocket-general taints: - key: compute.analytical-platform.service.justice.gov.uk/karpenter-node-pool value: "general-on-demand"