diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/README.md b/README.md index 46c13e2b..041915e2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This project contains Terraform scripts to provision the AWS cloud infrastructur >- Amazon VPC and Security Group >- Managed Amazon Elastic Kubernetes Service (EKS) - >- Self-managed node groups with required labels and taints + >- Amazon EKS managed node groups with required labels and taints >- Infrastructure to deploy the SAS Viya CAS server in SMP or MPP mode >- Amazon Elastic Block Storage (EBS) for NFS >- Amazon Elastic File System (EFS) diff --git a/locals.tf b/locals.tf old mode 100644 new mode 100755 index eb04f6d2..9aa62c9f --- a/locals.tf +++ b/locals.tf @@ -5,7 +5,6 @@ locals { security_group_id = var.security_group_id == null ? aws_security_group.sg[0].id : data.aws_security_group.sg[0].id cluster_security_group_id = var.cluster_security_group_id == null ? aws_security_group.cluster_security_group.0.id : var.cluster_security_group_id workers_security_group_id = var.workers_security_group_id == null ? aws_security_group.workers_security_group.0.id : var.workers_security_group_id - cluster_name = "${var.prefix}-eks" # CIDRs @@ -31,47 +30,92 @@ locals { kubeconfig_path = var.iac_tooling == "docker" ? "/workspace/${local.kubeconfig_filename}" : local.kubeconfig_filename kubeconfig_ca_cert = data.aws_eks_cluster.cluster.certificate_authority.0.data - # Mapping node_pools to worker_groups - default_node_pool = [ - { - name = "default" - instance_type = var.default_nodepool_vm_type - root_volume_size = var.default_nodepool_os_disk_size - root_volume_type = var.default_nodepool_os_disk_type - root_iops = var.default_nodepool_os_disk_iops - asg_desired_capacity = var.default_nodepool_node_count - asg_min_size = var.default_nodepool_min_nodes - asg_max_size = var.default_nodepool_max_nodes - kubelet_extra_args = "--node-labels=${replace(replace(jsonencode(var.default_nodepool_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", var.default_nodepool_taints)}" - additional_userdata = (var.default_nodepool_custom_data != "" ? file(var.default_nodepool_custom_data) : "") - metadata_http_endpoint = var.default_nodepool_metadata_http_endpoint - metadata_http_tokens = var.default_nodepool_metadata_http_tokens - metadata_http_put_response_hop_limit = var.default_nodepool_metadata_http_put_response_hop_limit - + # Mapping node_pools to node_groups + default_node_pool = { + default = { + name = "default" + instance_types = [var.default_nodepool_vm_type] + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_type = var.default_nodepool_os_disk_type + volume_size = var.default_nodepool_os_disk_size + iops = var.default_nodepool_os_disk_iops + } + } + } + desired_size = var.default_nodepool_node_count + min_size = var.default_nodepool_min_nodes + max_size = var.default_nodepool_max_nodes + taints = { for i, taint in var.default_nodepool_taints : "default-${i}"=> { + "key" = split("=", taint)[0], + "value"= split(":", split("=", taint)[1])[0], + "effect"=length(regexall(":No", taint)) > 0 ? upper(replace(split(":", split("=", taint)[1])[1], "No", "NO_")) : upper(replace(split(":", split("=", taint)[1])[1], "No", "_NO_")) + } + } + labels = var.default_nodepool_labels + # User data + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=${replace(replace(jsonencode(var.default_nodepool_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", var.default_nodepool_taints)} ' " + post_bootstrap_user_data = (var.default_nodepool_custom_data != "" ? file(var.default_nodepool_custom_data) : "") + metadata_options = { + http_endpoint = var.default_nodepool_metadata_http_endpoint + http_tokens = var.default_nodepool_metadata_http_tokens + http_put_response_hop_limit = var.default_nodepool_metadata_http_put_response_hop_limit + } + # Launch Template + create_launch_template = true + launch_template_name = "${local.cluster_name}-default-lt" + launch_template_use_name_prefix = true + tags = var.autoscaling_enabled ? merge(var.tags, { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true}) : var.tags } - ] + } - user_node_pool = [ - for np_key, np_value in var.node_pools : - { - name = np_key - instance_type = np_value.vm_type - root_volume_size = np_value.os_disk_size - root_volume_type = np_value.os_disk_type - root_iops = np_value.os_disk_iops - asg_desired_capacity = var.autoscaling_enabled ? np_value.min_nodes == 0 ? 1 : np_value.min_nodes : np_value.min_nodes # TODO - Remove when moving to managed nodes - asg_min_size = np_value.min_nodes - asg_max_size = np_value.max_nodes - kubelet_extra_args = "--node-labels=${replace(replace(jsonencode(np_value.node_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", np_value.node_taints)}" - additional_userdata = (np_value.custom_data != "" ? file(np_value.custom_data) : "") - metadata_http_endpoint = np_value.metadata_http_endpoint - metadata_http_tokens = np_value.metadata_http_tokens - metadata_http_put_response_hop_limit = np_value.metadata_http_put_response_hop_limit + user_node_pool = { + for key, np_value in var.node_pools : + key => { + name = key + instance_types = [np_value.vm_type] + disk_size = np_value.os_disk_size + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_type = np_value.os_disk_type + volume_size = np_value.os_disk_size + iops = np_value.os_disk_iops + } + } + } + desired_size = var.autoscaling_enabled ? np_value.min_nodes == 0 ? 1 : np_value.min_nodes : np_value.min_nodes # TODO - Remove when moving to managed nodes + min_size = np_value.min_nodes + max_size = np_value.max_nodes + # AWS EKS Taints - https://docs.aws.amazon.com/eks/latest/userguide/node-taints-managed-node-groups.html + taints ={ for i, taint in np_value.node_taints: "${key}-${i}"=> { # to handle multiple taints, add index i to key for uniqueness + "key" = split("=", taint)[0], + "value"= split(":", split("=", taint)[1])[0], + "effect"=length(regexall(":No", taint)) > 0 ? upper(replace(split(":", split("=", taint)[1])[1], "No", "NO_")) : upper(replace(split(":", split("=", taint)[1])[1], "No", "_NO_")) + } + } + labels = np_value.node_labels + # User data + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=${replace(replace(jsonencode(np_value.node_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", np_value.node_taints)}' " + post_bootstrap_user_data = (np_value.custom_data != "" ? file(np_value.custom_data) : "") + metadata_options = { + http_endpoint = var.default_nodepool_metadata_http_endpoint + http_tokens = var.default_nodepool_metadata_http_tokens + http_put_response_hop_limit = var.default_nodepool_metadata_http_put_response_hop_limit + } + # Launch Template + create_launch_template = true + launch_template_name = "${local.cluster_name}-${key}-lt" + launch_template_use_name_prefix = true + tags = var.autoscaling_enabled ? merge(var.tags, { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true}) : var.tags } - ] + } # Merging the default_node_pool into the work_groups node pools - worker_groups = concat(local.default_node_pool, local.user_node_pool) + node_groups = merge(local.default_node_pool, local.user_node_pool) # PostgreSQL postgres_servers = var.postgres_servers == null ? {} : { for k, v in var.postgres_servers : k => merge( var.postgres_server_defaults, v, )} diff --git a/main.tf b/main.tf old mode 100644 new mode 100755 index 6b50edf4..045b87a5 --- a/main.tf +++ b/main.tf @@ -5,12 +5,12 @@ # provider "aws" { - region = var.location - profile = var.aws_profile - shared_credentials_file = var.aws_shared_credentials_file - access_key = var.aws_access_key_id - secret_key = var.aws_secret_access_key - token = var.aws_session_token + region = var.location + profile = var.aws_profile + shared_credentials_file = var.aws_shared_credentials_file + access_key = var.aws_access_key_id + secret_key = var.aws_secret_access_key + token = var.aws_session_token } data "aws_eks_cluster" "cluster" { @@ -80,37 +80,79 @@ module "vpc" { # EKS Setup - https://github.com/terraform-aws-modules/terraform-aws-eks module "eks" { source = "terraform-aws-modules/eks/aws" - version = "17.1.0" + version = "18.7.1" cluster_name = local.cluster_name cluster_version = var.kubernetes_version + cluster_enabled_log_types = [] # disable cluster control plan logging + create_cloudwatch_log_group = false cluster_endpoint_private_access = true - cluster_create_endpoint_private_access_sg_rule = true # NOTE: If true cluster_endpoint_private_access_cidrs must always be set - cluster_endpoint_private_access_sg = [local.security_group_id] - cluster_endpoint_private_access_cidrs = local.cluster_endpoint_private_access_cidrs cluster_endpoint_public_access = var.cluster_api_mode == "public" ? true : false cluster_endpoint_public_access_cidrs = local.cluster_endpoint_public_access_cidrs - write_kubeconfig = false - subnets = module.vpc.private_subnets + + subnet_ids = module.vpc.private_subnets vpc_id = module.vpc.vpc_id tags = var.tags enable_irsa = var.autoscaling_enabled - - manage_worker_iam_resources = var.workers_iam_role_name == null ? true : false - workers_role_name = var.workers_iam_role_name - manage_cluster_iam_resources = var.cluster_iam_role_name == null ? true : false - cluster_iam_role_name = var.cluster_iam_role_name - worker_create_security_group = false - worker_security_group_id = local.workers_security_group_id - cluster_create_security_group = false + ################################################################################ + # Cluster Security Group + ################################################################################ + create_cluster_security_group = false # v17: cluster_create_security_group cluster_security_group_id = local.cluster_security_group_id + # Extend cluster security group rules + cluster_security_group_additional_rules = { + egress_nodes_ephemeral_ports_tcp = { + description = "To node 1025-65535" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "egress" + source_node_security_group = true + } + } + + ################################################################################ + # Node Security Group + ################################################################################ + create_node_security_group = false #v17: worker_create_security_group + node_security_group_id = local.workers_security_group_id #v17: worker_security_group_id + # Extend node-to-node security group rules + node_security_group_additional_rules = { + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } - workers_group_defaults = { - tags = var.autoscaling_enabled ? [ { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true} ] : null - metadata_http_tokens = "required" - metadata_http_put_response_hop_limit = 1 - iam_instance_profile_name = var.workers_iam_role_name + ################################################################################ + # Handle BYO IAM policy + ################################################################################ + create_iam_role = var.cluster_iam_role_name == null ? true : false # v17: manage_cluster_iam_resources + iam_role_name = var.cluster_iam_role_name # v17: cluster_iam_role_name + iam_role_additional_policies = [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] + + ## Use this to define any values that are common and applicable to all Node Groups + eks_managed_node_group_defaults = { + create_security_group = false + vpc_security_group_ids = [local.workers_security_group_id] } - worker_groups = local.worker_groups + + ## Any individual Node Group customizations should go here + eks_managed_node_groups = local.node_groups } module "autoscaling" { diff --git a/outputs.tf b/outputs.tf old mode 100644 new mode 100755 index 85ade237..3401e2ef --- a/outputs.tf +++ b/outputs.tf @@ -8,9 +8,6 @@ output "kube_config" { sensitive = true } -output "worker_iam_role_arn" { - value = module.eks.worker_iam_role_arn -} output "cluster_iam_role_arn" { value = module.eks.cluster_iam_role_arn } diff --git a/security.tf b/security.tf index 71099cfc..4954f519 100644 --- a/security.tf +++ b/security.tf @@ -21,6 +21,7 @@ resource "aws_security_group" "sg" { resource "aws_security_group_rule" "vms" { count = ( length(local.vm_public_access_cidrs) > 0 + && var.security_group_id == null && ( (var.create_jump_public_ip && var.create_jump_vm ) || (var.create_nfs_public_ip && var.storage_type == "standard") ) diff --git a/versions.tf b/versions.tf index a062a753..b1f6bf38 100644 --- a/versions.tf +++ b/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "3.43.0" + version = "3.72.0" } random = { source = "hashicorp/random"