From ed22caaf6b8ef02fe1d809d6e0029000c63145ba Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 12:06:02 -0800 Subject: [PATCH 01/25] create bastion instance, security groups, container linux config --- aws/container-linux/kubernetes/bastion.tf | 73 +++++++++++++++++++ .../kubernetes/cl/bastion.yaml.tmpl | 6 ++ 2 files changed, 79 insertions(+) create mode 100644 aws/container-linux/kubernetes/bastion.tf create mode 100644 aws/container-linux/kubernetes/cl/bastion.yaml.tmpl diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf new file mode 100644 index 000000000..03ac3853e --- /dev/null +++ b/aws/container-linux/kubernetes/bastion.tf @@ -0,0 +1,73 @@ +data "template_file" "controller_config" { + template = "${file("${path.module}/cl/bastion.yaml.tmpl")}" + + vars = { + ssh_authorized_key = "${var.ssh_authorized_key}" + } +} + +data "ct_config" "bastion_ign" { + content = "${data.template_file.controller_config.rendered}" + pretty_print = false +} + +resource "aws_instance" "bastion" { + tags = { + Name = "${var.cluster_name}-bastion" + } + + instance_type = "${var.bastion_type}" + ami = "${data.aws_ami.coreos.image_id}" + user_data = "${element(data.ct_config.bastion_ign.rendered, count.index)}" + + # network + associate_public_ip_address = true + subnet_id = "${element(aws_subnet.public.*.id, 0)}" + vpc_security_group_ids = ["${aws_security_group.bastion_external.id}"] + + lifecycle { + create_before_destroy = true + ignore_changes = ["ami"] + } +} + +resource "aws_security_group" "bastion_external" { + name_prefix = "${var.cluster_name}-bastion-external-" + description = "Allows access to the bastion from the internet" + vpc_id = "${aws_vpc.network.id}" + + ingress { + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + protocol = -1 + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_security_group" "bastion_internal" { + name_prefix = "${var.cluster_name}-bastion-internal-" + description = "Allows access to a host from the bastion" + vpc_id = "${aws_vpc.network.id}" + + ingress { + protocol = "tcp" + from_port = 22 + to_port = 22 + security_groups = ["${aws_security_group.bastion_external.id}"] + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl b/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl new file mode 100644 index 000000000..283fba7f4 --- /dev/null +++ b/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl @@ -0,0 +1,6 @@ +--- +passwd: + users: + - name: core + ssh_authorized_keys: + - "${ssh_authorized_key}" From d28d0d3dec4b5ad847fed30a6b629149051cc184 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 12:08:27 -0800 Subject: [PATCH 02/25] remove controllers/workers from public subnet --- aws/container-linux/kubernetes/controllers.tf | 1 - aws/container-linux/kubernetes/workers.tf | 3 --- 2 files changed, 4 deletions(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index ff1e571cf..7a4291b88 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -34,7 +34,6 @@ resource "aws_instance" "controllers" { # network associate_public_ip_address = true - subnet_id = "${element(aws_subnet.public.*.id, count.index)}" vpc_security_group_ids = ["${aws_security_group.controller.id}"] lifecycle { diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 49ab684df..efc4cf933 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -9,9 +9,6 @@ resource "aws_autoscaling_group" "workers" { default_cooldown = 30 health_check_grace_period = 30 - # network - vpc_zone_identifier = ["${aws_subnet.public.*.id}"] - # template launch_configuration = "${aws_launch_configuration.worker.name}" From 3264259858121d524744e27e10630995100829ad Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 12:13:15 -0800 Subject: [PATCH 03/25] add controllers + workers to bastion_internal sg --- aws/container-linux/kubernetes/controllers.tf | 5 ++++- aws/container-linux/kubernetes/workers.tf | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 7a4291b88..d9c3ba218 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -34,7 +34,10 @@ resource "aws_instance" "controllers" { # network associate_public_ip_address = true - vpc_security_group_ids = ["${aws_security_group.controller.id}"] + vpc_security_group_ids = [ + "${aws_security_group.controller.id}", + "${aws_security_group.bastion_internal.id}" + ] lifecycle { ignore_changes = ["ami"] diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index efc4cf933..b8ea4c391 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -45,7 +45,10 @@ resource "aws_launch_configuration" "worker" { } # network - security_groups = ["${aws_security_group.worker.id}"] + security_groups = [ + "${aws_security_group.worker.id}", + "${aws_security_group.bastion_internal.id}" + ] lifecycle { // Override the default destroy and replace update behavior From 349b88801b0ea612ce3884eddc49a9a6fc141aed Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 12:16:57 -0800 Subject: [PATCH 04/25] Remove public IP from controllers --- aws/container-linux/kubernetes/controllers.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index d9c3ba218..f952bc6a5 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -33,7 +33,6 @@ resource "aws_instance" "controllers" { } # network - associate_public_ip_address = true vpc_security_group_ids = [ "${aws_security_group.controller.id}", "${aws_security_group.bastion_internal.id}" From 84830f356affbc26f9e73312d2361d9f425ce933 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 12:20:35 -0800 Subject: [PATCH 05/25] add bastion_{host,user} to provisioner connections --- aws/container-linux/kubernetes/ssh.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 4b58c297c..c20ba865f 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -4,8 +4,13 @@ resource "null_resource" "copy-secrets" { connection { type = "ssh" - host = "${element(aws_instance.controllers.*.public_ip, count.index)}" + + host = "${element(aws_instance.controllers.*.private_ip, count.index)}" user = "core" + + bastion_host = "${aws_instance.bastion.public_ip}" + bastion_user = "core" + timeout = "15m" } @@ -73,8 +78,13 @@ resource "null_resource" "bootkube-start" { connection { type = "ssh" + host = "${aws_instance.controllers.0.public_ip}" user = "core" + + bastion_host = "${aws_instance.bastion.public_ip}" + bastion_user = "core" + timeout = "15m" } From 6f0bef42a936c3a93d8dbad9ff40826129a2efd9 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 13:02:13 -0800 Subject: [PATCH 06/25] bastion: fix cl template_file resource name --- aws/container-linux/kubernetes/bastion.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 03ac3853e..5a8a1e406 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -1,4 +1,4 @@ -data "template_file" "controller_config" { +data "template_file" "bastion_config" { template = "${file("${path.module}/cl/bastion.yaml.tmpl")}" vars = { @@ -7,7 +7,7 @@ data "template_file" "controller_config" { } data "ct_config" "bastion_ign" { - content = "${data.template_file.controller_config.rendered}" + content = "${data.template_file.bastion_config.rendered}" pretty_print = false } From 67f64436eab4b0ad784a879efdecdd9ffcf3af9e Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 13:03:14 -0800 Subject: [PATCH 07/25] Add bastion_type var --- aws/container-linux/kubernetes/variables.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 362fb5e69..8ecaa5808 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -114,4 +114,10 @@ variable "oidc_client_id" { variable "oidc_username_claim" { description = "JWT claim to use as the user name." type = "string" +} + +variable "bastion_type" { + type = "string" + default = "t2.micro" + description = "Bastion EC2 instance type" } \ No newline at end of file From efcbdde47465b5f4cd3ac5b04c1b458c6e7016b6 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 13:09:30 -0800 Subject: [PATCH 08/25] bastion: fix user_data --- aws/container-linux/kubernetes/bastion.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 5a8a1e406..7192bb662 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -18,7 +18,7 @@ resource "aws_instance" "bastion" { instance_type = "${var.bastion_type}" ami = "${data.aws_ami.coreos.image_id}" - user_data = "${element(data.ct_config.bastion_ign.rendered, count.index)}" + user_data = "${data.ct_config.bastion_ign.rendered}" # network associate_public_ip_address = true From bf2b1029273b3bc410950988cd5488802f8702e1 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 16:51:22 -0800 Subject: [PATCH 09/25] deploy kube controller and worker components into private subnet --- aws/container-linux/kubernetes/controllers.tf | 3 +- aws/container-linux/kubernetes/elb.tf | 2 +- aws/container-linux/kubernetes/network.tf | 56 ++++++++++++++++++- aws/container-linux/kubernetes/workers.tf | 3 + 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index f952bc6a5..8aa856a63 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -33,7 +33,8 @@ resource "aws_instance" "controllers" { } # network - vpc_security_group_ids = [ + subnet_id = "${element(aws_subnet.private.*.id, count.index)}" + vpc_security_group_ids = [ "${aws_security_group.controller.id}", "${aws_security_group.bastion_internal.id}" ] diff --git a/aws/container-linux/kubernetes/elb.tf b/aws/container-linux/kubernetes/elb.tf index 2a448a698..20dcdb960 100644 --- a/aws/container-linux/kubernetes/elb.tf +++ b/aws/container-linux/kubernetes/elb.tf @@ -16,7 +16,7 @@ resource "aws_route53_record" "apiserver" { # Controller Network Load Balancer resource "aws_elb" "apiserver" { name = "${var.cluster_name}-apiserver" - subnets = ["${aws_subnet.public.*.id}"] + subnets = ["${aws_subnet.private.*.id}"] security_groups = ["${aws_security_group.controller.id}"] listener { diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index 1be5073ba..5048ce753 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -17,7 +17,7 @@ resource "aws_internet_gateway" "gateway" { tags = "${map("Name", "${var.cluster_name}")}" } -resource "aws_route_table" "default" { +resource "aws_route_table" "public" { vpc_id = "${aws_vpc.network.id}" route { @@ -30,7 +30,7 @@ resource "aws_route_table" "default" { gateway_id = "${aws_internet_gateway.gateway.id}" } - tags = "${map("Name", "${var.cluster_name}")}" + tags = "${map("Name", "${var.cluster_name}-public")}" } # Subnets (one per availability zone) @@ -52,6 +52,56 @@ resource "aws_subnet" "public" { resource "aws_route_table_association" "public" { count = "${length(data.aws_availability_zones.all.names)}" - route_table_id = "${aws_route_table.default.id}" + route_table_id = "${aws_route_table.public.id}" subnet_id = "${element(aws_subnet.public.*.id, count.index)}" } + +resource "aws_subnet" "private" { + count = "${length(data.aws_availability_zones.all.names)}" + + vpc_id = "${aws_vpc.network.id}" + availability_zone = "${data.aws_availability_zones.all.names[count.index]}" + + cidr_block = "${cidrsubnet(var.host_cidr, 4, count.index + 10)}" + ipv6_cidr_block = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index + 10)}" + assign_ipv6_address_on_creation = true + + tags = "${map("Name", "${var.cluster_name}-public-${count.index}")}" +} + +resource "aws_route_table" "private" { + vpc_id = "${aws_vpc.network.id}" + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = "${aws_nat_gateway.nat.id}" + } + + route { + ipv6_cidr_block = "::/0" + nat_gateway_id = "${aws_nat_gateway.nat.id}" + } + + tags = "${map("Name", "${var.cluster_name}-private")}" +} + +resource "aws_route_table_association" "private" { + count = "${length(data.aws_availability_zones.all.names)}" + + route_table_id = "${aws_route_table.private.id}" + subnet_id = "${element(aws_subnet.private.*.id, count.index)}" +} + + +resource "aws_eip" "nat" { + vpc = true +} + +resource "aws_nat_gateway" "nat" { + depends_on = [ + "aws_internet_gateway.gateway", + ] + + allocation_id = "${aws_eip.nat.id}" + subnet_id = "${element(aws_subnet.public.*.id, 0)}" +} \ No newline at end of file diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index b8ea4c391..8f245c60e 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -9,6 +9,9 @@ resource "aws_autoscaling_group" "workers" { default_cooldown = 30 health_check_grace_period = 30 + # network + vpc_zone_identifier = ["${aws_subnet.private.*.id}"] + # template launch_configuration = "${aws_launch_configuration.worker.name}" From 58af913794ea8fd5df286be42d3d3d5ae9c5904e Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 16:52:29 -0800 Subject: [PATCH 10/25] add dns and tags to bastion for discoverability --- aws/container-linux/kubernetes/bastion.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 7192bb662..78c716068 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -34,8 +34,13 @@ resource "aws_instance" "bastion" { resource "aws_security_group" "bastion_external" { name_prefix = "${var.cluster_name}-bastion-external-" description = "Allows access to the bastion from the internet" + vpc_id = "${aws_vpc.network.id}" + tags { + Name = "${var.cluster_name}-bastion-external" + } + ingress { protocol = "tcp" from_port = 22 @@ -58,8 +63,13 @@ resource "aws_security_group" "bastion_external" { resource "aws_security_group" "bastion_internal" { name_prefix = "${var.cluster_name}-bastion-internal-" description = "Allows access to a host from the bastion" + vpc_id = "${aws_vpc.network.id}" + tags { + Name = "${var.cluster_name}-bastion-internal" + } + ingress { protocol = "tcp" from_port = 22 @@ -71,3 +81,11 @@ resource "aws_security_group" "bastion_internal" { create_before_destroy = true } } + +resource "aws_route53_record" "bastion" { + zone_id = "${var.dns_zone_id}" + name = "bastion.${var.dns_zone}" + type = "A" + ttl = "300" + records = ["${aws_instance.bastion.public_ip}"] +} \ No newline at end of file From ce83787d33bebbde90698aebe289405f30149a46 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 17:21:21 -0800 Subject: [PATCH 11/25] fix tag name for private subnets --- aws/container-linux/kubernetes/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index 5048ce753..54447a295 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -66,7 +66,7 @@ resource "aws_subnet" "private" { ipv6_cidr_block = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index + 10)}" assign_ipv6_address_on_creation = true - tags = "${map("Name", "${var.cluster_name}-public-${count.index}")}" + tags = "${map("Name", "${var.cluster_name}-private-${count.index}")}" } resource "aws_route_table" "private" { From d3a9913a5cd8340446598b5861d9fb93555c9348 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Tue, 6 Mar 2018 17:22:15 -0800 Subject: [PATCH 12/25] add an egress internet gateway for ipv6 * it's the equivalent of a nat gateway, but for ipv6 * a nat gw will return an error if used as an ipv6 route --- aws/container-linux/kubernetes/network.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index 54447a295..80bfa50c5 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -78,8 +78,8 @@ resource "aws_route_table" "private" { } route { - ipv6_cidr_block = "::/0" - nat_gateway_id = "${aws_nat_gateway.nat.id}" + ipv6_cidr_block = "::/0" + egress_only_gateway_id = "${aws_egress_only_internet_gateway.egress_igw.id}" } tags = "${map("Name", "${var.cluster_name}-private")}" @@ -104,4 +104,8 @@ resource "aws_nat_gateway" "nat" { allocation_id = "${aws_eip.nat.id}" subnet_id = "${element(aws_subnet.public.*.id, 0)}" +} + +resource "aws_egress_only_internet_gateway" "egress_igw" { + vpc_id = "${aws_vpc.network.id}" } \ No newline at end of file From 0f9d958dcfc5c0b90f5db82854fcc52889546a84 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Mar 2018 16:30:51 -0800 Subject: [PATCH 13/25] elb: convert apiserver to internal nlb --- aws/container-linux/kubernetes/elb.tf | 68 ++++++++++++++++++--------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/aws/container-linux/kubernetes/elb.tf b/aws/container-linux/kubernetes/elb.tf index 20dcdb960..b36353d1d 100644 --- a/aws/container-linux/kubernetes/elb.tf +++ b/aws/container-linux/kubernetes/elb.tf @@ -7,37 +7,61 @@ resource "aws_route53_record" "apiserver" { # AWS recommends their special "alias" records for ELBs alias { - name = "${aws_elb.apiserver.dns_name}" - zone_id = "${aws_elb.apiserver.zone_id}" + name = "${aws_lb.apiserver.dns_name}" + zone_id = "${aws_lb.apiserver.zone_id}" evaluate_target_health = true } } -# Controller Network Load Balancer -resource "aws_elb" "apiserver" { - name = "${var.cluster_name}-apiserver" - subnets = ["${aws_subnet.private.*.id}"] - security_groups = ["${aws_security_group.controller.id}"] - - listener { - lb_port = 443 - lb_protocol = "tcp" - instance_port = 443 - instance_protocol = "tcp" +# Network Load Balancer for apiservers +resource "aws_lb" "apiserver" { + name = "${var.cluster_name}-apiserver" + load_balancer_type = "network" + internal = true + + subnets = ["${aws_subnet.private.*.id}"] +} + +# Forward HTTP traffic to controllers +resource "aws_lb_listener" "apiserver-https" { + load_balancer_arn = "${aws_lb.apiserver.arn}" + protocol = "TCP" + port = "443" + + default_action { + type = "forward" + target_group_arn = "${aws_lb_target_group.controllers.arn}" } +} - instances = ["${aws_instance.controllers.*.id}"] +# Target group of controllers +resource "aws_lb_target_group" "controllers" { + name = "${var.cluster_name}-controllers" + vpc_id = "${aws_vpc.network.id}" + target_type = "instance" + + protocol = "TCP" + port = 443 # Kubelet HTTP health check health_check { - target = "SSL:443" - healthy_threshold = 2 - unhealthy_threshold = 4 - timeout = 5 - interval = 6 + protocol = "TCP" + port = 443 + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 } +} + +# Attach controller instances to apiserver NLB +resource "aws_lb_target_group_attachment" "controllers" { + count = "${var.controller_count}" - idle_timeout = 3600 - connection_draining = true - connection_draining_timeout = 300 + target_group_arn = "${aws_lb_target_group.controllers.arn}" + target_id = "${element(aws_instance.controllers.*.id, count.index)}" + port = 443 } From 852baa2b9e7dd95af7927c38fb7d66de6ea7489b Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 8 Mar 2018 16:30:59 -0800 Subject: [PATCH 14/25] ssh: use private_ip --- aws/container-linux/kubernetes/ssh.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index c20ba865f..35bf3441a 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -79,7 +79,7 @@ resource "null_resource" "bootkube-start" { connection { type = "ssh" - host = "${aws_instance.controllers.0.public_ip}" + host = "${aws_instance.controllers.0.private_ip}" user = "core" bastion_host = "${aws_instance.bastion.public_ip}" From 09206447d9a953453f4591db30a25db17e79b6f3 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Fri, 9 Mar 2018 15:13:01 -0800 Subject: [PATCH 15/25] dns: use bastion.$CLUSTER --- aws/container-linux/kubernetes/bastion.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 78c716068..7b444e3b9 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -84,7 +84,7 @@ resource "aws_security_group" "bastion_internal" { resource "aws_route53_record" "bastion" { zone_id = "${var.dns_zone_id}" - name = "bastion.${var.dns_zone}" + name = "bastion.${var.cluster_name}.${var.dns_zone}" type = "A" ttl = "300" records = ["${aws_instance.bastion.public_ip}"] From dcc5a1ee1d0c3479da9c583897837e2a37ef906b Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Mar 2018 12:05:19 -0700 Subject: [PATCH 16/25] replace nlb will classic elb Can't use NLB because: * During bootstrap a single instance is in service * NLB cannot direct traffic to the originating instance * It preserves IPs Can't use ALB because: * Health check is HTTP/HTTPS * apiserver will reject health checks w/ 401 (unauthorized) * https://github.com/kubernetes/kubernetes/issues/43784 --- aws/container-linux/kubernetes/elb.tf | 66 +++++++++------------------ 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/aws/container-linux/kubernetes/elb.tf b/aws/container-linux/kubernetes/elb.tf index b36353d1d..d633660bb 100644 --- a/aws/container-linux/kubernetes/elb.tf +++ b/aws/container-linux/kubernetes/elb.tf @@ -7,61 +7,37 @@ resource "aws_route53_record" "apiserver" { # AWS recommends their special "alias" records for ELBs alias { - name = "${aws_lb.apiserver.dns_name}" - zone_id = "${aws_lb.apiserver.zone_id}" + name = "${aws_elb.apiserver.dns_name}" + zone_id = "${aws_elb.apiserver.zone_id}" evaluate_target_health = true } } # Network Load Balancer for apiservers -resource "aws_lb" "apiserver" { - name = "${var.cluster_name}-apiserver" - load_balancer_type = "network" - internal = true - - subnets = ["${aws_subnet.private.*.id}"] -} - -# Forward HTTP traffic to controllers -resource "aws_lb_listener" "apiserver-https" { - load_balancer_arn = "${aws_lb.apiserver.arn}" - protocol = "TCP" - port = "443" - - default_action { - type = "forward" - target_group_arn = "${aws_lb_target_group.controllers.arn}" +resource "aws_elb" "apiserver" { + name = "${var.cluster_name}-apiserver" + subnets = ["${aws_subnet.public.*.id}"] + security_groups = ["${aws_security_group.controller.id}"] + + listener { + lb_port = 443 + lb_protocol = "tcp" + instance_port = 443 + instance_protocol = "tcp" } -} -# Target group of controllers -resource "aws_lb_target_group" "controllers" { - name = "${var.cluster_name}-controllers" - vpc_id = "${aws_vpc.network.id}" - target_type = "instance" - - protocol = "TCP" - port = 443 + instances = ["${aws_instance.controllers.*.id}"] # Kubelet HTTP health check health_check { - protocol = "TCP" - port = 443 - - # NLBs required to use same healthy and unhealthy thresholds - healthy_threshold = 3 - unhealthy_threshold = 3 - - # Interval between health checks required to be 10 or 30 - interval = 10 + target = "SSL:443" + healthy_threshold = 2 + unhealthy_threshold = 4 + timeout = 5 + interval = 6 } -} - -# Attach controller instances to apiserver NLB -resource "aws_lb_target_group_attachment" "controllers" { - count = "${var.controller_count}" - target_group_arn = "${aws_lb_target_group.controllers.arn}" - target_id = "${element(aws_instance.controllers.*.id, count.index)}" - port = 443 + idle_timeout = 3600 + connection_draining = true + connection_draining_timeout = 300 } From ddb62f4b6638bead0161ad639f53691a9e279052 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Mar 2018 12:24:41 -0700 Subject: [PATCH 17/25] NLB + IP target! Gaurav ftw --- aws/container-linux/kubernetes/elb.tf | 66 ++++++++++++++++++--------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/aws/container-linux/kubernetes/elb.tf b/aws/container-linux/kubernetes/elb.tf index d633660bb..9ab740016 100644 --- a/aws/container-linux/kubernetes/elb.tf +++ b/aws/container-linux/kubernetes/elb.tf @@ -7,37 +7,61 @@ resource "aws_route53_record" "apiserver" { # AWS recommends their special "alias" records for ELBs alias { - name = "${aws_elb.apiserver.dns_name}" - zone_id = "${aws_elb.apiserver.zone_id}" + name = "${aws_lb.apiserver.dns_name}" + zone_id = "${aws_lb.apiserver.zone_id}" evaluate_target_health = true } } # Network Load Balancer for apiservers -resource "aws_elb" "apiserver" { - name = "${var.cluster_name}-apiserver" - subnets = ["${aws_subnet.public.*.id}"] - security_groups = ["${aws_security_group.controller.id}"] - - listener { - lb_port = 443 - lb_protocol = "tcp" - instance_port = 443 - instance_protocol = "tcp" +resource "aws_lb" "apiserver" { + name = "${var.cluster_name}-apiserver" + load_balancer_type = "network" + internal = true + + subnets = ["${aws_subnet.private.*.id}"] +} + +# Forward HTTP traffic to controllers +resource "aws_lb_listener" "apiserver-https" { + load_balancer_arn = "${aws_lb.apiserver.arn}" + protocol = "TCP" + port = "443" + + default_action { + type = "forward" + target_group_arn = "${aws_lb_target_group.controllers.arn}" } +} - instances = ["${aws_instance.controllers.*.id}"] +# Target group of controllers +resource "aws_lb_target_group" "controllers" { + name = "${var.cluster_name}-controllers" + vpc_id = "${aws_vpc.network.id}" + target_type = "ip" + + protocol = "TCP" + port = 443 # Kubelet HTTP health check health_check { - target = "SSL:443" - healthy_threshold = 2 - unhealthy_threshold = 4 - timeout = 5 - interval = 6 + protocol = "TCP" + port = 443 + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 } +} + +# Attach controller instances to apiserver NLB +resource "aws_lb_target_group_attachment" "controllers" { + count = "${var.controller_count}" - idle_timeout = 3600 - connection_draining = true - connection_draining_timeout = 300 + target_group_arn = "${aws_lb_target_group.controllers.arn}" + target_id = "${element(aws_instance.controllers.*.private_ip, count.index)}" + port = 443 } From c65c6b93bb7257014c1524341f9b986a81a65fc2 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 12 Mar 2018 17:17:09 -0700 Subject: [PATCH 18/25] Reduce spacing between public and private subnets --- aws/container-linux/kubernetes/network.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index 80bfa50c5..f32855c3f 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -62,8 +62,8 @@ resource "aws_subnet" "private" { vpc_id = "${aws_vpc.network.id}" availability_zone = "${data.aws_availability_zones.all.names[count.index]}" - cidr_block = "${cidrsubnet(var.host_cidr, 4, count.index + 10)}" - ipv6_cidr_block = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index + 10)}" + cidr_block = "${cidrsubnet(var.host_cidr, 4, count.index + 8)}" + ipv6_cidr_block = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index + 8)}" assign_ipv6_address_on_creation = true tags = "${map("Name", "${var.cluster_name}-private-${count.index}")}" From 609e82868e34e45fd4b1149d8fa25b4d943aeb2f Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 14 Mar 2018 19:18:40 -0700 Subject: [PATCH 19/25] create bastion in load balanced asg with size=1 --- aws/container-linux/kubernetes/bastion.tf | 123 +++++++++++++++++----- aws/container-linux/kubernetes/ssh.tf | 8 +- 2 files changed, 103 insertions(+), 28 deletions(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 7b444e3b9..368391ea7 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -1,40 +1,71 @@ -data "template_file" "bastion_config" { - template = "${file("${path.module}/cl/bastion.yaml.tmpl")}" +resource "aws_autoscaling_group" "bastion" { + name = "${var.cluster_name}-bastion ${aws_launch_configuration.bastion.name}" - vars = { - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} + # count + desired_capacity = 1 + min_size = 1 + max_size = 1 + default_cooldown = 30 + health_check_grace_period = 30 -data "ct_config" "bastion_ign" { - content = "${data.template_file.bastion_config.rendered}" - pretty_print = false -} + # network + vpc_zone_identifier = ["${aws_subnet.private.*.id}"] + + # template + launch_configuration = "${aws_launch_configuration.bastion.name}" + + # target groups to which instances should be added + target_group_arns = [ + "${aws_lb_target_group.bastion.id}" + ] -resource "aws_instance" "bastion" { - tags = { - Name = "${var.cluster_name}-bastion" + lifecycle { + # override the default destroy and replace update behavior + create_before_destroy = true + ignore_changes = ["image_id"] } + tags = [{ + key = "Name" + value = "${var.cluster_name}-bastion" + propagate_at_launch = true + }] +} + +resource "aws_launch_configuration" "bastion" { + image_id = "${data.aws_ami.coreos.image_id}" instance_type = "${var.bastion_type}" - ami = "${data.aws_ami.coreos.image_id}" + user_data = "${data.ct_config.bastion_ign.rendered}" # network - associate_public_ip_address = true - subnet_id = "${element(aws_subnet.public.*.id, 0)}" - vpc_security_group_ids = ["${aws_security_group.bastion_external.id}"] + security_groups = [ + "${aws_security_group.bastion_external.id}" + ] lifecycle { + // Override the default destroy and replace update behavior create_before_destroy = true - ignore_changes = ["ami"] } } +data "template_file" "bastion_config" { + template = "${file("${path.module}/cl/bastion.yaml.tmpl")}" + + vars = { + ssh_authorized_key = "${var.ssh_authorized_key}" + } +} + +data "ct_config" "bastion_ign" { + content = "${data.template_file.bastion_config.rendered}" + pretty_print = false +} + resource "aws_security_group" "bastion_external" { name_prefix = "${var.cluster_name}-bastion-external-" description = "Allows access to the bastion from the internet" - + vpc_id = "${aws_vpc.network.id}" tags { @@ -63,7 +94,7 @@ resource "aws_security_group" "bastion_external" { resource "aws_security_group" "bastion_internal" { name_prefix = "${var.cluster_name}-bastion-internal-" description = "Allows access to a host from the bastion" - + vpc_id = "${aws_vpc.network.id}" tags { @@ -82,10 +113,54 @@ resource "aws_security_group" "bastion_internal" { } } +resource "aws_lb" "bastion" { + name = "${var.cluster_name}-bastion" + load_balancer_type = "network" + + subnets = ["${aws_subnet.public.*.id}"] +} + + +resource "aws_lb_listener" "bastion" { + load_balancer_arn = "${aws_lb.bastion.arn}" + protocol = "TCP" + port = "22" + + default_action { + type = "forward" + target_group_arn = "${aws_lb_target_group.bastion.arn}" + } +} + +resource "aws_lb_target_group" "bastion" { + name = "${var.cluster_name}-bastion" + vpc_id = "${aws_vpc.network.id}" + target_type = "instance" + + protocol = "TCP" + port = 22 + + health_check { + protocol = "TCP" + port = 22 + + healthy_threshold = 3 + unhealthy_threshold = 3 + + interval = 10 + } +} + resource "aws_route53_record" "bastion" { zone_id = "${var.dns_zone_id}" - name = "bastion.${var.cluster_name}.${var.dns_zone}" + + name = "${format("bastion.%s.%s.", var.cluster_name, var.dns_zone)}" type = "A" - ttl = "300" - records = ["${aws_instance.bastion.public_ip}"] -} \ No newline at end of file + + # AWS recommends their special "alias" records for ELBs + alias { + name = "${aws_lb.bastion.dns_name}" + zone_id = "${aws_lb.bastion.zone_id}" + evaluate_target_health = false + } +} diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 35bf3441a..210400c6d 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -4,13 +4,13 @@ resource "null_resource" "copy-secrets" { connection { type = "ssh" - + host = "${element(aws_instance.controllers.*.private_ip, count.index)}" user = "core" - bastion_host = "${aws_instance.bastion.public_ip}" + bastion_host = "${aws_route53_record.bastion.fqdn}" bastion_user = "core" - + timeout = "15m" } @@ -82,7 +82,7 @@ resource "null_resource" "bootkube-start" { host = "${aws_instance.controllers.0.private_ip}" user = "core" - bastion_host = "${aws_instance.bastion.public_ip}" + bastion_host = "${aws_route53_record.bastion.fqdn}" bastion_user = "core" timeout = "15m" From 75bdabaa1c9c3ae6e0688451b66e67d2f8ec4cd9 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 14 Mar 2018 19:23:26 -0700 Subject: [PATCH 20/25] set bastion asg size to var.bastion_count --- aws/container-linux/kubernetes/bastion.tf | 6 +++--- aws/container-linux/kubernetes/variables.tf | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 368391ea7..355558c63 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -2,9 +2,9 @@ resource "aws_autoscaling_group" "bastion" { name = "${var.cluster_name}-bastion ${aws_launch_configuration.bastion.name}" # count - desired_capacity = 1 - min_size = 1 - max_size = 1 + desired_capacity = "${var.bastion_count}" + min_size = "${var.bastion_count}" + max_size = "${var.bastion_count}" default_cooldown = 30 health_check_grace_period = 30 diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 8ecaa5808..11d4db172 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -120,4 +120,10 @@ variable "bastion_type" { type = "string" default = "t2.micro" description = "Bastion EC2 instance type" -} \ No newline at end of file +} + +variable "bastion_count" { + type = "string" + default = "1" + description = "Number of bastion hosts to run" +} From 3f2a94f5295f621daac09b6b52e00a2c46b53889 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Wed, 14 Mar 2018 20:47:02 -0700 Subject: [PATCH 21/25] wait for 1 available bastion before considering asg ready * copy-secrets hangs for ~1 min waiting for an instance * it succeeds but it's more clear to wait for the asg before trying --- aws/container-linux/kubernetes/bastion.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 355558c63..7ff86c339 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -19,6 +19,8 @@ resource "aws_autoscaling_group" "bastion" { "${aws_lb_target_group.bastion.id}" ] + min_elb_capacity = 1 + lifecycle { # override the default destroy and replace update behavior create_before_destroy = true @@ -152,6 +154,8 @@ resource "aws_lb_target_group" "bastion" { } resource "aws_route53_record" "bastion" { + depends_on = ["aws_autoscaling_group.bastion"] + zone_id = "${var.dns_zone_id}" name = "${format("bastion.%s.%s.", var.cluster_name, var.dns_zone)}" From 61424d59fa186e6346a35b6f0cfbcebcc29b8f58 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 19 Mar 2018 17:47:07 -0700 Subject: [PATCH 22/25] remove unnecessary lb alias record comment --- aws/container-linux/kubernetes/bastion.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 7ff86c339..88474d760 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -161,7 +161,6 @@ resource "aws_route53_record" "bastion" { name = "${format("bastion.%s.%s.", var.cluster_name, var.dns_zone)}" type = "A" - # AWS recommends their special "alias" records for ELBs alias { name = "${aws_lb.bastion.dns_name}" zone_id = "${aws_lb.bastion.zone_id}" From 644ef35e3b2b61ac4ca1aa6ae1748a8ad55068d8 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Mon, 19 Mar 2018 18:00:09 -0700 Subject: [PATCH 23/25] newline --- aws/container-linux/kubernetes/network.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/container-linux/kubernetes/network.tf b/aws/container-linux/kubernetes/network.tf index f32855c3f..6c8f5f352 100644 --- a/aws/container-linux/kubernetes/network.tf +++ b/aws/container-linux/kubernetes/network.tf @@ -108,4 +108,4 @@ resource "aws_nat_gateway" "nat" { resource "aws_egress_only_internet_gateway" "egress_igw" { vpc_id = "${aws_vpc.network.id}" -} \ No newline at end of file +} From 4c6ca6164b42477df712095f55ae0e03d3bb23a1 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Thu, 22 Mar 2018 17:56:52 -0700 Subject: [PATCH 24/25] add support for multiple ssh keys (untested) --- aws/container-linux/kubernetes/bastion.tf | 2 +- .../kubernetes/cl/bastion.yaml.tmpl | 2 +- .../kubernetes/cl/controller.yaml.tmpl | 2 +- .../kubernetes/cl/worker.yaml.tmpl | 2 +- aws/container-linux/kubernetes/controllers.tf | 14 +++++++------- aws/container-linux/kubernetes/variables.tf | 6 +++--- aws/container-linux/kubernetes/workers.tf | 16 ++++++++-------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index 88474d760..e10e669dc 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -55,7 +55,7 @@ data "template_file" "bastion_config" { template = "${file("${path.module}/cl/bastion.yaml.tmpl")}" vars = { - ssh_authorized_key = "${var.ssh_authorized_key}" + ssh_authorized_keys_list = "- ${join("\n- ", var.ssh_authorized_keys)}" } } diff --git a/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl b/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl index 283fba7f4..d7013768c 100644 --- a/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl @@ -3,4 +3,4 @@ passwd: users: - name: core ssh_authorized_keys: - - "${ssh_authorized_key}" + ${ssh_authorized_keys_list} diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 390dfe0fd..3f9bce5db 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -167,4 +167,4 @@ passwd: users: - name: core ssh_authorized_keys: - - "${ssh_authorized_key}" + ${ssh_authorized_keys_list} diff --git a/aws/container-linux/kubernetes/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/cl/worker.yaml.tmpl index 994794d66..b4bb379fc 100644 --- a/aws/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -129,4 +129,4 @@ passwd: users: - name: core ssh_authorized_keys: - - "${ssh_authorized_key}" + ${ssh_authorized_keys_list} diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 8aa856a63..71fd774c7 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -58,13 +58,13 @@ data "template_file" "controller_config" { # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}" - k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - kubeconfig_ca_cert = "${module.bootkube.ca_cert}" - kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" - kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" - kubeconfig_server = "${module.bootkube.server}" + k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + ssh_authorized_keys_list = "- ${join("\n- ", var.ssh_authorized_keys)}" + cluster_domain_suffix = "${var.cluster_domain_suffix}" + kubeconfig_ca_cert = "${module.bootkube.ca_cert}" + kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" + kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" + kubeconfig_server = "${module.bootkube.server}" } } diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf index 11d4db172..5699d4f8f 100644 --- a/aws/container-linux/kubernetes/variables.tf +++ b/aws/container-linux/kubernetes/variables.tf @@ -13,9 +13,9 @@ variable "dns_zone_id" { description = "AWS DNS Zone ID (e.g. Z3PAABBCFAKEC0)" } -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key for user 'core'" +variable "ssh_authorized_keys" { + type = "list" + description = "SSH public keys for user 'core'" } variable "os_channel" { diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index 8f245c60e..d409251f3 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -64,14 +64,14 @@ data "template_file" "worker_config" { template = "${file("${path.module}/cl/worker.yaml.tmpl")}" vars = { - k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - kubeconfig_ca_cert = "${module.bootkube.ca_cert}" - kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" - kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" - kubeconfig_server = "${module.bootkube.server}" + k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" + ssh_authorized_keys_list = "- ${join("\n- ", var.ssh_authorized_keys)}" + cluster_domain_suffix = "${var.cluster_domain_suffix}" + kubeconfig_ca_cert = "${module.bootkube.ca_cert}" + kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" + kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" + kubeconfig_server = "${module.bootkube.server}" } } From b90776dbef780a18157c82826d119fca536c30a8 Mon Sep 17 00:00:00 2001 From: Ben Drucker Date: Fri, 23 Mar 2018 15:10:26 -0700 Subject: [PATCH 25/25] use a [] yaml array to specify the list of authorized public keys * using indented arrays means the template call means matching indentation --- aws/container-linux/kubernetes/bastion.tf | 2 +- aws/container-linux/kubernetes/cl/bastion.yaml.tmpl | 3 +-- aws/container-linux/kubernetes/cl/controller.yaml.tmpl | 3 +-- aws/container-linux/kubernetes/cl/worker.yaml.tmpl | 3 +-- aws/container-linux/kubernetes/controllers.tf | 2 +- aws/container-linux/kubernetes/workers.tf | 2 +- 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/aws/container-linux/kubernetes/bastion.tf b/aws/container-linux/kubernetes/bastion.tf index e10e669dc..bef364120 100644 --- a/aws/container-linux/kubernetes/bastion.tf +++ b/aws/container-linux/kubernetes/bastion.tf @@ -55,7 +55,7 @@ data "template_file" "bastion_config" { template = "${file("${path.module}/cl/bastion.yaml.tmpl")}" vars = { - ssh_authorized_keys_list = "- ${join("\n- ", var.ssh_authorized_keys)}" + ssh_authorized_keys_list = "[ ${join(", ", var.ssh_authorized_keys)} ]" } } diff --git a/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl b/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl index d7013768c..b011e7eee 100644 --- a/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/bastion.yaml.tmpl @@ -2,5 +2,4 @@ passwd: users: - name: core - ssh_authorized_keys: - ${ssh_authorized_keys_list} + ssh_authorized_keys: ${ssh_authorized_keys_list} diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index 3f9bce5db..6756ccdfb 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -166,5 +166,4 @@ storage: passwd: users: - name: core - ssh_authorized_keys: - ${ssh_authorized_keys_list} + ssh_authorized_keys: ${ssh_authorized_keys_list} diff --git a/aws/container-linux/kubernetes/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/cl/worker.yaml.tmpl index b4bb379fc..fd43fff95 100644 --- a/aws/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -128,5 +128,4 @@ storage: passwd: users: - name: core - ssh_authorized_keys: - ${ssh_authorized_keys_list} + ssh_authorized_keys: ${ssh_authorized_keys_list} diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 71fd774c7..9e7bd24b9 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -59,7 +59,7 @@ data "template_file" "controller_config" { etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}" k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - ssh_authorized_keys_list = "- ${join("\n- ", var.ssh_authorized_keys)}" + ssh_authorized_keys_list = "[ ${join(", ", var.ssh_authorized_keys)} ]" cluster_domain_suffix = "${var.cluster_domain_suffix}" kubeconfig_ca_cert = "${module.bootkube.ca_cert}" kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index d409251f3..5775c8396 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -66,7 +66,7 @@ data "template_file" "worker_config" { vars = { k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" - ssh_authorized_keys_list = "- ${join("\n- ", var.ssh_authorized_keys)}" + ssh_authorized_keys_list = "[ ${join(", ", var.ssh_authorized_keys)} ]" cluster_domain_suffix = "${var.cluster_domain_suffix}" kubeconfig_ca_cert = "${module.bootkube.ca_cert}" kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"