From 40a42011d98af377103a6416f21476d4d97da15d Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 6 Jun 2024 10:19:29 +0000 Subject: [PATCH 1/4] Add namespace and role Signed-off-by: Jacob Woffenden --- .../kubernetes-namespaces.tf | 14 ++++++-- .../kubernetes-roles.tf | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 terraform/environments/analytical-platform-compute/kubernetes-roles.tf diff --git a/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf b/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf index 041024eb1b0..1990fe31f4c 100644 --- a/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf +++ b/terraform/environments/analytical-platform-compute/kubernetes-namespaces.tf @@ -40,8 +40,18 @@ resource "kubernetes_namespace" "actions_runners" { metadata { name = "actions-runners" labels = { - "pod-security.kubernetes.io/enforce" = "baseline" - "pod-security.kubernetes.io/enforce-version" = "v${local.environment_configuration.eks_cluster_version}" + "pod-security.kubernetes.io/enforce" = "baseline" + "compute.analytical-platform.service.justice.gov.uk/workload" = "actions-runners" + } + } +} + +resource "kubernetes_namespace" "airflow" { + metadata { + name = "airflow" + labels = { + "pod-security.kubernetes.io/enforce" = "restricted" + "compute.analytical-platform.service.justice.gov.uk/workload" = "airflow" } } } diff --git a/terraform/environments/analytical-platform-compute/kubernetes-roles.tf b/terraform/environments/analytical-platform-compute/kubernetes-roles.tf new file mode 100644 index 00000000000..b8e84430b78 --- /dev/null +++ b/terraform/environments/analytical-platform-compute/kubernetes-roles.tf @@ -0,0 +1,34 @@ +// Derived from https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-eks-example.html#eksctl-role +resource "kubernetes_role" "airflow_execution" { + metadata { + name = "airflow-execution" + namespace = kubernetes_namespace.airflow.metadata[0].name + } + rule { + api_groups = [ + "", + "apps", + "batch", + "extensions", + ] + resources = [ + "jobs", + "pods", + "pods/attach", + "pods/exec", + "pods/log", + "pods/portforward", + "secrets", + "services" + ] + verbs = [ + "create", + "delete", + "describe", + "get", + "list", + "patch", + "update" + ] + } +} From 2b924b906a4af369a14dfb9f851068187e0e41ca Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 6 Jun 2024 10:34:34 +0000 Subject: [PATCH 2/4] Update EFS CSI Signed-off-by: Jacob Woffenden --- .../environment-configuration.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf index 03041607868..98fe1c77634 100644 --- a/terraform/environments/analytical-platform-compute/environment-configuration.tf +++ b/terraform/environments/analytical-platform-compute/environment-configuration.tf @@ -45,7 +45,7 @@ locals { coredns = "v1.11.1-eksbuild.9" kube_proxy = "v1.30.0-eksbuild.3" aws_ebs_csi_driver = "v1.31.0-eksbuild.1" - aws_efs_csi_driver = "v2.0.2-eksbuild.1" + aws_efs_csi_driver = "v2.0.3-eksbuild.1" aws_guardduty_agent = "v1.6.1-eksbuild.1" eks_pod_identity_agent = "v1.2.0-eksbuild.1" vpc_cni = "v1.18.1-eksbuild.3" @@ -80,7 +80,7 @@ locals { coredns = "v1.11.1-eksbuild.9" kube_proxy = "v1.30.0-eksbuild.3" aws_ebs_csi_driver = "v1.31.0-eksbuild.1" - aws_efs_csi_driver = "v2.0.2-eksbuild.1" + aws_efs_csi_driver = "v2.0.3-eksbuild.1" aws_guardduty_agent = "v1.6.1-eksbuild.1" eks_pod_identity_agent = "v1.2.0-eksbuild.1" vpc_cni = "v1.18.1-eksbuild.3" @@ -115,7 +115,7 @@ locals { coredns = "v1.11.1-eksbuild.9" kube_proxy = "v1.30.0-eksbuild.3" aws_ebs_csi_driver = "v1.31.0-eksbuild.1" - aws_efs_csi_driver = "v2.0.2-eksbuild.1" + aws_efs_csi_driver = "v2.0.3-eksbuild.1" aws_guardduty_agent = "v1.6.1-eksbuild.1" eks_pod_identity_agent = "v1.2.0-eksbuild.1" vpc_cni = "v1.18.1-eksbuild.3" From ea7924c6c41192f534dc6beeb04e758a7d4c272d Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 6 Jun 2024 10:42:59 +0000 Subject: [PATCH 3/4] Update EKS module Add access entry for airflow Signed-off-by: Jacob Woffenden --- .../analytical-platform-compute/eks-cluster.tf | 7 ++++++- .../environment-configuration.tf | 9 +++++++++ .../kubernetes-role-bindings.tf | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 terraform/environments/analytical-platform-compute/kubernetes-role-bindings.tf diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index 0239d962596..acfec3f1224 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -6,7 +6,7 @@ module "eks" { #checkov:skip=CKV_TF_2:Module registry does not support tags for versions source = "terraform-aws-modules/eks/aws" - version = "20.12.0" + version = "20.13.1" cluster_name = local.eks_cluster_name cluster_version = local.environment_configuration.eks_cluster_version @@ -116,6 +116,11 @@ module "eks" { } } } + data-engineering-airflow = { + principal_arn = local.environment_configuration.data_engineering_airflow_execution_role_arn + username = "data-engineering-airflow" + kubernetes_groups = ["airflow"] + } } tags = local.tags diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf index 98fe1c77634..bd5e907a47c 100644 --- a/terraform/environments/analytical-platform-compute/environment-configuration.tf +++ b/terraform/environments/analytical-platform-compute/environment-configuration.tf @@ -51,6 +51,9 @@ locals { vpc_cni = "v1.18.1-eksbuild.3" } + /* Data Engineering Airflow */ + data_engineering_airflow_execution_role_arn = "arn:aws:iam::593291632749:role/airflow-dev-execution-role" + /* Observability Platform */ observability_platform = "development" @@ -89,6 +92,9 @@ locals { /* Observability Platform */ observability_platform = "development" + /* Data Engineering Airflow */ + data_engineering_airflow_execution_role_arn = "arn:aws:iam::593291632749:role/airflow-dev-execution-role" + /* QuickSight */ quicksight_notification_email = "analytical-platform@digital.justice.gov.uk" } @@ -121,6 +127,9 @@ locals { vpc_cni = "v1.18.1-eksbuild.3" } + /* Data Engineering Airflow */ + data_engineering_airflow_execution_role_arn = "arn:aws:iam::593291632749:role/airflow-prod-execution-role" + /* Observability Platform */ observability_platform = "production" diff --git a/terraform/environments/analytical-platform-compute/kubernetes-role-bindings.tf b/terraform/environments/analytical-platform-compute/kubernetes-role-bindings.tf new file mode 100644 index 00000000000..bf1e3b21833 --- /dev/null +++ b/terraform/environments/analytical-platform-compute/kubernetes-role-bindings.tf @@ -0,0 +1,16 @@ +resource "kubernetes_role_binding" "airflow_execution" { + metadata { + name = "airflow-execution" + namespace = kubernetes_namespace.airflow.metadata[0].name + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = kubernetes_role.airflow_execution.metadata[0].name + } + subject { + api_group = "rbac.authorization.k8s.io" + kind = "Group" + name = "airflow" + } +} From ac30a5745b3f8219b2eba3547c9eb76390c0d0ed Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 6 Jun 2024 11:05:24 +0000 Subject: [PATCH 4/4] Expand general MNG Add airflow-high-memory Signed-off-by: Jacob Woffenden --- .../eks-cluster.tf | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-compute/eks-cluster.tf b/terraform/environments/analytical-platform-compute/eks-cluster.tf index acfec3f1224..3f2ee2ae364 100644 --- a/terraform/environments/analytical-platform-compute/eks-cluster.tf +++ b/terraform/environments/analytical-platform-compute/eks-cluster.tf @@ -98,10 +98,40 @@ module "eks" { eks_managed_node_groups = { general = { min_size = 1 - max_size = 5 + max_size = 10 desired_size = 3 instance_types = ["t3.xlarge"] } + airflow-high-memory = { + min_size = 0 + max_size = 1 + desired_size = 0 + instance_types = ["r6i.8xlarge"] + labels = { + high-memory = "true" + } + taints = [ + { + key = "high-memory" + value = "true" + effect = "NO_SCHEDULE" + } + ] + block_device_mappings = { + xvdb = { + device_name = "/dev/xvdb" + ebs = { + volume_size = 200 + volume_type = "gp3" + iops = 3000 + throughput = 250 + encrypted = true + kms_key_id = module.ebs_kms.key_arn + delete_on_termination = true + } + } + } + } } access_entries = {