AWS terraform: add zone label and reserved resources for nodes (#871)

Signed-off-by: Aylei <[email protected]>
pingcap · Sep 6, 2019 · 12cea6a · 12cea6a
1 parent 48dd502
commit 12cea6a
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 4 deletions.
diff --git a/deploy/modules/aws/tidb-cluster/local.tf b/deploy/modules/aws/tidb-cluster/local.tf
@@ -35,14 +35,24 @@ locals {
     placement_group               = ""                         # The name of the placement group into which to launch the instances, if any.
   }
 
+  # 169.254.169.254 is the authoritative AWS metadata server, see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+  aws_zone_getter = "$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)"
+
   tidb_cluster_worker_groups = [
     {
       name                 = "${var.cluster_name}-pd"
       key_name             = var.ssh_key_name
       instance_type        = var.pd_instance_type
       root_volume_size     = "50"
       public_ip            = false
-      kubelet_extra_args   = "--register-with-taints=dedicated=${var.cluster_name}-pd:NoSchedule --node-labels=dedicated=${var.cluster_name}-pd,pingcap.com/aws-local-ssd=true"
+      # the space separator is safe when the extra args is empty or prefixed by spaces (the same hereafter)
+      kubelet_extra_args   = join(" ",
+        [
+          "--register-with-taints=dedicated=${var.cluster_name}-pd:NoSchedule",
+          "--node-labels=dedicated=${var.cluster_name}-pd,pingcap.com/aws-local-ssd=true,zone=${local.aws_zone_getter}",
+          lookup(var.group_kubelet_extra_args, "pd", var.kubelet_extra_args)
+        ]
+      )
       asg_desired_capacity = var.pd_count
       asg_max_size         = var.pd_count + 2
       # additional_userdata  = file("userdata.sh")
@@ -53,7 +63,13 @@ locals {
       instance_type        = var.tikv_instance_type
       root_volume_size     = "50"
       public_ip            = false
-      kubelet_extra_args   = "--register-with-taints=dedicated=${var.cluster_name}-tikv:NoSchedule --node-labels=dedicated=${var.cluster_name}-tikv,pingcap.com/aws-local-ssd=true"
+      kubelet_extra_args   = join(" ",
+        [
+          "--register-with-taints=dedicated=${var.cluster_name}-tikv:NoSchedule",
+          "--node-labels=dedicated=${var.cluster_name}-tikv,pingcap.com/aws-local-ssd=true,zone=${local.aws_zone_getter}",
+          lookup(var.group_kubelet_extra_args, "tikv", var.kubelet_extra_args)
+        ]
+      )
       asg_desired_capacity = var.tikv_count
       asg_max_size         = var.tikv_count + 2
       pre_userdata         = file("${path.module}/pre_userdata")
@@ -66,7 +82,13 @@ locals {
       root_volume_type     = "gp2"
       root_volume_size     = "50"
       public_ip            = false
-      kubelet_extra_args   = "--register-with-taints=dedicated=${var.cluster_name}-tidb:NoSchedule --node-labels=dedicated=${var.cluster_name}-tidb"
+      kubelet_extra_args   = join(" ",
+        [
+          "--register-with-taints=dedicated=${var.cluster_name}-tidb:NoSchedule",
+          "--node-labels=dedicated=${var.cluster_name}-tidb,zone=${local.aws_zone_getter}",
+          lookup(var.group_kubelet_extra_args, "tidb", var.kubelet_extra_args)
+        ]
+      )
       asg_desired_capacity = var.tidb_count
       asg_max_size         = var.tidb_count + 2
     },
@@ -77,6 +99,12 @@ locals {
       root_volume_type     = "gp2"
       root_volume_size     = "50"
       public_ip            = false
+      kubelet_extra_args   = join(" ",
+        [
+          "--node-labels=zone=${local.aws_zone_getter}",
+          lookup(var.group_kubelet_extra_args, "monitor", var.kubelet_extra_args)
+        ]
+      )
       asg_desired_capacity = 1
       asg_max_size         = 3
     }

diff --git a/deploy/modules/aws/tidb-cluster/templates/userdata.sh.tpl b/deploy/modules/aws/tidb-cluster/templates/userdata.sh.tpl
@@ -18,7 +18,7 @@ systemctl daemon-reload
 systemctl restart docker
 
 # Bootstrap and join the cluster
-/etc/eks/bootstrap.sh --b64-cluster-ca '${cluster_auth_base64}' --apiserver-endpoint '${endpoint}' ${bootstrap_extra_args} --kubelet-extra-args '${kubelet_extra_args}' '${cluster_name}'
+/etc/eks/bootstrap.sh --b64-cluster-ca "${cluster_auth_base64}" --apiserver-endpoint "${endpoint}" ${bootstrap_extra_args} --kubelet-extra-args "${kubelet_extra_args}" "${cluster_name}"
 
 # Allow user supplied userdata code
 ${additional_userdata}
diff --git a/deploy/modules/aws/tidb-cluster/variables.tf b/deploy/modules/aws/tidb-cluster/variables.tf
@@ -147,3 +147,14 @@ variable "iam_path" {
   description = "If provided, all IAM roles will be created on this path."
   default     = "/"
 }
+
+variable "kubelet_extra_args" {
+  description = "Extra arguments passed to kubelet"
+  default     = "--kube-reserved memory=0.3Gi,ephemeral-storage=1Gi --system-reserved memory=0.2Gi,ephemeral-storage=1Gi"
+}
+
+variable "group_kubelet_extra_args" {
+  description = "If provided, override the kubelet_extra_args for a specific node group which matches the key of map (e.g. tidb, tikv, pd, monitor)"
+  type        = map(string)
+  default = {}
+}