Skip to content

Commit

Permalink
(IAC-422) Update EKS Module (#122)
Browse files Browse the repository at this point in the history 
* wip - initial version bumps

* initial working draft

* WIP code changes for EKS module 17 to 18 upgrade

* working version, pared down and hardcoded to a single node group

* switch to EKS managed nodegroup and map more variables to v18 module

* working version with cluster SG/R fixed

* handle all taints cases and fixed typo default node group name

* add custom launch template name

* rename worker_groups to node_groups for consistency with AWS

* update Terraform version to 1.1.6

* revert all versions except aws provider required by EKS module

* Update TFv1.0.0 in readme and remove extraneous default_node_group call

* removed output variable no longer supported

* conditionally create SG rule for BYO SecurityGroup

* Update docs

Co-authored-by: Mano Meenaksh <[email protected]>
  • Loading branch information
@jarpat @manoatsas
jarpat and manoatsas authored Apr 29, 2022
1 parent d7679b7 commit d66d5b9
Show file tree
Hide file tree
Showing 7 changed files with 152 additions and 68 deletions.
Empty file modified Dockerfile
100644 → 100755
View file
Empty file.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ This project contains Terraform scripts to provision the AWS cloud infrastructur

>- Amazon VPC and Security Group
>- Managed Amazon Elastic Kubernetes Service (EKS)
>- Self-managed node groups with required labels and taints
>- Amazon EKS managed node groups with required labels and taints
>- Infrastructure to deploy the SAS Viya CAS server in SMP or MPP mode
>- Amazon Elastic Block Storage (EBS) for NFS
>- Amazon Elastic File System (EFS)
Expand Down
118 changes: 81 additions & 37 deletions locals.tf
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ locals {
security_group_id = var.security_group_id == null ? aws_security_group.sg[0].id : data.aws_security_group.sg[0].id
cluster_security_group_id = var.cluster_security_group_id == null ? aws_security_group.cluster_security_group.0.id : var.cluster_security_group_id
workers_security_group_id = var.workers_security_group_id == null ? aws_security_group.workers_security_group.0.id : var.workers_security_group_id

cluster_name = "${var.prefix}-eks"

# CIDRs
Expand All @@ -31,47 +30,92 @@ locals {
kubeconfig_path = var.iac_tooling == "docker" ? "/workspace/${local.kubeconfig_filename}" : local.kubeconfig_filename
kubeconfig_ca_cert = data.aws_eks_cluster.cluster.certificate_authority.0.data

# Mapping node_pools to worker_groups
default_node_pool = [
{
name = "default"
instance_type = var.default_nodepool_vm_type
root_volume_size = var.default_nodepool_os_disk_size
root_volume_type = var.default_nodepool_os_disk_type
root_iops = var.default_nodepool_os_disk_iops
asg_desired_capacity = var.default_nodepool_node_count
asg_min_size = var.default_nodepool_min_nodes
asg_max_size = var.default_nodepool_max_nodes
kubelet_extra_args = "--node-labels=${replace(replace(jsonencode(var.default_nodepool_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", var.default_nodepool_taints)}"
additional_userdata = (var.default_nodepool_custom_data != "" ? file(var.default_nodepool_custom_data) : "")
metadata_http_endpoint = var.default_nodepool_metadata_http_endpoint
metadata_http_tokens = var.default_nodepool_metadata_http_tokens
metadata_http_put_response_hop_limit = var.default_nodepool_metadata_http_put_response_hop_limit

# Mapping node_pools to node_groups
default_node_pool = {
default = {
name = "default"
instance_types = [var.default_nodepool_vm_type]
block_device_mappings = {
xvda = {
device_name = "/dev/xvda"
ebs = {
volume_type = var.default_nodepool_os_disk_type
volume_size = var.default_nodepool_os_disk_size
iops = var.default_nodepool_os_disk_iops
}
}
}
desired_size = var.default_nodepool_node_count
min_size = var.default_nodepool_min_nodes
max_size = var.default_nodepool_max_nodes
taints = { for i, taint in var.default_nodepool_taints : "default-${i}"=> {
"key" = split("=", taint)[0],
"value"= split(":", split("=", taint)[1])[0],
"effect"=length(regexall(":No", taint)) > 0 ? upper(replace(split(":", split("=", taint)[1])[1], "No", "NO_")) : upper(replace(split(":", split("=", taint)[1])[1], "No", "_NO_"))
}
}
labels = var.default_nodepool_labels
# User data
bootstrap_extra_args = "--kubelet-extra-args '--node-labels=${replace(replace(jsonencode(var.default_nodepool_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", var.default_nodepool_taints)} ' "
post_bootstrap_user_data = (var.default_nodepool_custom_data != "" ? file(var.default_nodepool_custom_data) : "")
metadata_options = {
http_endpoint = var.default_nodepool_metadata_http_endpoint
http_tokens = var.default_nodepool_metadata_http_tokens
http_put_response_hop_limit = var.default_nodepool_metadata_http_put_response_hop_limit
}
# Launch Template
create_launch_template = true
launch_template_name = "${local.cluster_name}-default-lt"
launch_template_use_name_prefix = true
tags = var.autoscaling_enabled ? merge(var.tags, { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true}) : var.tags
}
]
}
user_node_pool = [
for np_key, np_value in var.node_pools :
{
name = np_key
instance_type = np_value.vm_type
root_volume_size = np_value.os_disk_size
root_volume_type = np_value.os_disk_type
root_iops = np_value.os_disk_iops
asg_desired_capacity = var.autoscaling_enabled ? np_value.min_nodes == 0 ? 1 : np_value.min_nodes : np_value.min_nodes # TODO - Remove when moving to managed nodes
asg_min_size = np_value.min_nodes
asg_max_size = np_value.max_nodes
kubelet_extra_args = "--node-labels=${replace(replace(jsonencode(np_value.node_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", np_value.node_taints)}"
additional_userdata = (np_value.custom_data != "" ? file(np_value.custom_data) : "")
metadata_http_endpoint = np_value.metadata_http_endpoint
metadata_http_tokens = np_value.metadata_http_tokens
metadata_http_put_response_hop_limit = np_value.metadata_http_put_response_hop_limit
user_node_pool = {
for key, np_value in var.node_pools :
key => {
name = key
instance_types = [np_value.vm_type]
disk_size = np_value.os_disk_size
block_device_mappings = {
xvda = {
device_name = "/dev/xvda"
ebs = {
volume_type = np_value.os_disk_type
volume_size = np_value.os_disk_size
iops = np_value.os_disk_iops
}
}
}
desired_size = var.autoscaling_enabled ? np_value.min_nodes == 0 ? 1 : np_value.min_nodes : np_value.min_nodes # TODO - Remove when moving to managed nodes
min_size = np_value.min_nodes
max_size = np_value.max_nodes
# AWS EKS Taints - https://docs.aws.amazon.com/eks/latest/userguide/node-taints-managed-node-groups.html
taints ={ for i, taint in np_value.node_taints: "${key}-${i}"=> { # to handle multiple taints, add index i to key for uniqueness
"key" = split("=", taint)[0],
"value"= split(":", split("=", taint)[1])[0],
"effect"=length(regexall(":No", taint)) > 0 ? upper(replace(split(":", split("=", taint)[1])[1], "No", "NO_")) : upper(replace(split(":", split("=", taint)[1])[1], "No", "_NO_"))
}
}
labels = np_value.node_labels
# User data
bootstrap_extra_args = "--kubelet-extra-args '--node-labels=${replace(replace(jsonencode(np_value.node_labels), "/[\"\\{\\}]/", ""), ":", "=")} --register-with-taints=${join(",", np_value.node_taints)}' "
post_bootstrap_user_data = (np_value.custom_data != "" ? file(np_value.custom_data) : "")
metadata_options = {
http_endpoint = var.default_nodepool_metadata_http_endpoint
http_tokens = var.default_nodepool_metadata_http_tokens
http_put_response_hop_limit = var.default_nodepool_metadata_http_put_response_hop_limit
}
# Launch Template
create_launch_template = true
launch_template_name = "${local.cluster_name}-${key}-lt"
launch_template_use_name_prefix = true
tags = var.autoscaling_enabled ? merge(var.tags, { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true}) : var.tags
}
]
}
# Merging the default_node_pool into the work_groups node pools
worker_groups = concat(local.default_node_pool, local.user_node_pool)
node_groups = merge(local.default_node_pool, local.user_node_pool)
# PostgreSQL
postgres_servers = var.postgres_servers == null ? {} : { for k, v in var.postgres_servers : k => merge( var.postgres_server_defaults, v, )}
Expand Down
94 changes: 68 additions & 26 deletions main.tf
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
#

provider "aws" {
region = var.location
profile = var.aws_profile
shared_credentials_file = var.aws_shared_credentials_file
access_key = var.aws_access_key_id
secret_key = var.aws_secret_access_key
token = var.aws_session_token
region = var.location
profile = var.aws_profile
shared_credentials_file = var.aws_shared_credentials_file
access_key = var.aws_access_key_id
secret_key = var.aws_secret_access_key
token = var.aws_session_token
}

data "aws_eks_cluster" "cluster" {
Expand Down Expand Up @@ -80,37 +80,79 @@ module "vpc" {
# EKS Setup - https://github.com/terraform-aws-modules/terraform-aws-eks
module "eks" {
source = "terraform-aws-modules/eks/aws"
version = "17.1.0"
version = "18.7.1"
cluster_name = local.cluster_name
cluster_version = var.kubernetes_version
cluster_enabled_log_types = [] # disable cluster control plan logging
create_cloudwatch_log_group = false
cluster_endpoint_private_access = true
cluster_create_endpoint_private_access_sg_rule = true # NOTE: If true cluster_endpoint_private_access_cidrs must always be set
cluster_endpoint_private_access_sg = [local.security_group_id]
cluster_endpoint_private_access_cidrs = local.cluster_endpoint_private_access_cidrs
cluster_endpoint_public_access = var.cluster_api_mode == "public" ? true : false
cluster_endpoint_public_access_cidrs = local.cluster_endpoint_public_access_cidrs
write_kubeconfig = false
subnets = module.vpc.private_subnets

subnet_ids = module.vpc.private_subnets
vpc_id = module.vpc.vpc_id
tags = var.tags
enable_irsa = var.autoscaling_enabled

manage_worker_iam_resources = var.workers_iam_role_name == null ? true : false
workers_role_name = var.workers_iam_role_name
manage_cluster_iam_resources = var.cluster_iam_role_name == null ? true : false
cluster_iam_role_name = var.cluster_iam_role_name
worker_create_security_group = false
worker_security_group_id = local.workers_security_group_id
cluster_create_security_group = false
################################################################################
# Cluster Security Group
################################################################################
create_cluster_security_group = false # v17: cluster_create_security_group
cluster_security_group_id = local.cluster_security_group_id
# Extend cluster security group rules
cluster_security_group_additional_rules = {
egress_nodes_ephemeral_ports_tcp = {
description = "To node 1025-65535"
protocol = "tcp"
from_port = 1025
to_port = 65535
type = "egress"
source_node_security_group = true
}
}

################################################################################
# Node Security Group
################################################################################
create_node_security_group = false #v17: worker_create_security_group
node_security_group_id = local.workers_security_group_id #v17: worker_security_group_id
# Extend node-to-node security group rules
node_security_group_additional_rules = {
ingress_self_all = {
description = "Node to node all ports/protocols"
protocol = "-1"
from_port = 0
to_port = 0
type = "ingress"
self = true
}
egress_all = {
description = "Node all egress"
protocol = "-1"
from_port = 0
to_port = 0
type = "egress"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
}

workers_group_defaults = {
tags = var.autoscaling_enabled ? [ { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true} ] : null
metadata_http_tokens = "required"
metadata_http_put_response_hop_limit = 1
iam_instance_profile_name = var.workers_iam_role_name
################################################################################
# Handle BYO IAM policy
################################################################################
create_iam_role = var.cluster_iam_role_name == null ? true : false # v17: manage_cluster_iam_resources
iam_role_name = var.cluster_iam_role_name # v17: cluster_iam_role_name
iam_role_additional_policies = [
"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
]

## Use this to define any values that are common and applicable to all Node Groups
eks_managed_node_group_defaults = {
create_security_group = false
vpc_security_group_ids = [local.workers_security_group_id]
}
worker_groups = local.worker_groups

## Any individual Node Group customizations should go here
eks_managed_node_groups = local.node_groups
}

module "autoscaling" {
Expand Down
3 changes: 0 additions & 3 deletions outputs.tf
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@ output "kube_config" {
sensitive = true
}

output "worker_iam_role_arn" {
value = module.eks.worker_iam_role_arn
}
output "cluster_iam_role_arn" {
value = module.eks.cluster_iam_role_arn
}
Expand Down
1 change: 1 addition & 0 deletions security.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ resource "aws_security_group" "sg" {

resource "aws_security_group_rule" "vms" {
count = ( length(local.vm_public_access_cidrs) > 0
&& var.security_group_id == null
&& ( (var.create_jump_public_ip && var.create_jump_vm )
|| (var.create_nfs_public_ip && var.storage_type == "standard")
)
Expand Down
2 changes: 1 addition & 1 deletion versions.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "3.43.0"
version = "3.72.0"
}
random = {
source = "hashicorp/random"
Expand Down

0 comments on commit d66d5b9

Please sign in to comment.