Skip to content

Commit

Permalink
data/aws: use nlbs instead of elbs
Browse files Browse the repository at this point in the history 
We've noticed an elevated rate of installation failures recently. The
root cause appears to be 50-90 seconds of latency added to traffic going
through the internal ELB on port 49500. This was causing Ignition's
connections to timeout, resulting in the machines never provisioning.

AWS's NLBs don't seem to have this high latency, so we've decided to
move over to them instead. With the move to NLBs, we also get the
ability to add individual health checks for each port instead of just a
single health check for each load balancer. Also, NLBs are cheaper.

This commit drops support for ingress and the console. Since the console
and router aren't currently configured correctly, nobody should notice
that this is gone. It was easier to drop support in this commit rather
than continue to try to plumb through the existing implementation
knowing that it was going to have to change in the future. Once the
router has a strategy for ingress, we'll re-add this functionality using
the new NLBs.

This also drop support for the `<cluster-name>-k8s` DNS entry. We aren't
aware of any consumers and it was going to be tedious to keep this
working.
  • Loading branch information
@crawford
crawford committed Nov 2, 2018
1 parent 8823176 commit 16dfbb3
Show file tree
Hide file tree
Showing 9 changed files with 245 additions and 302 deletions.
16 changes: 12 additions & 4 deletions data/data/aws/bootstrap/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,16 @@ resource "aws_instance" "bootstrap" {
volume_tags = "${var.tags}"
}

resource "aws_elb_attachment" "bootstrap" {
count = "${var.elbs_length}"
elb = "${var.elbs[count.index]}"
instance = "${aws_instance.bootstrap.id}"
resource "aws_lb_target_group_attachment" "public" {
count = "${var.public_target_group_arns_length}"

target_group_arn = "${var.public_target_group_arns[count.index]}"
target_id = "${aws_instance.bootstrap.private_ip}"
}

resource "aws_lb_target_group_attachment" "private" {
count = "${var.private_target_group_arns_length}"

target_group_arn = "${var.private_target_group_arns[count.index]}"
target_id = "${aws_instance.bootstrap.private_ip}"
}
30 changes: 20 additions & 10 deletions data/data/aws/bootstrap/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,6 @@ variable "cluster_name" {
description = "The name of the cluster."
}

variable "elbs" {
type = "list"
default = []
description = "Elastic load balancer IDs to attach to the bootstrap node."
}

variable "elbs_length" {
description = "The length of the 'elbs' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
}

variable "iam_role" {
type = "string"
default = ""
Expand All @@ -45,6 +35,26 @@ variable "instance_type" {
description = "The EC2 instance type for the bootstrap node."
}

variable "private_target_group_arns" {
type = "list"
default = []
description = "The list of target group ARNs for the private load balancer."
}

variable "private_target_group_arns_length" {
description = "The length of the 'private_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
}

variable "public_target_group_arns" {
type = "list"
default = []
description = "The list of target group ARNs for the public load balancer."
}

variable "public_target_group_arns_length" {
description = "The length of the 'public_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
}

variable "subnet_id" {
type = "string"
description = "The subnet ID for the bootstrap node."
Expand Down
90 changes: 45 additions & 45 deletions data/data/aws/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,18 @@ provider "aws" {
module "bootstrap" {
source = "./bootstrap"

ami = "${var.tectonic_aws_ec2_ami_override}"
associate_public_ip_address = "${var.tectonic_aws_endpoints != "private"}"
bucket = "${aws_s3_bucket.bootstrap.id}"
cluster_name = "${var.tectonic_cluster_name}"
elbs = "${module.vpc.aws_lbs}"
elbs_length = "${module.vpc.aws_lbs_length}"
iam_role = "${var.tectonic_aws_master_iam_role_name}"
ignition = "${var.ignition_bootstrap}"
subnet_id = "${module.vpc.master_subnet_ids[0]}"
vpc_security_group_ids = ["${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"]
ami = "${var.tectonic_aws_ec2_ami_override}"
associate_public_ip_address = "${var.tectonic_aws_endpoints != "private"}"
bucket = "${aws_s3_bucket.bootstrap.id}"
cluster_name = "${var.tectonic_cluster_name}"
public_target_group_arns = "${module.vpc.aws_lb_public_target_group_arns}"
public_target_group_arns_length = "${module.vpc.aws_lb_public_target_group_arns_length}"
private_target_group_arns = "${module.vpc.aws_lb_private_target_group_arns}"
private_target_group_arns_length = "${module.vpc.aws_lb_private_target_group_arns_length}"
iam_role = "${var.tectonic_aws_master_iam_role_name}"
ignition = "${var.ignition_bootstrap}"
subnet_id = "${module.vpc.master_subnet_ids[0]}"
vpc_security_group_ids = ["${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"]

tags = "${merge(map(
"Name", "${var.tectonic_cluster_name}-bootstrap",
Expand All @@ -37,25 +39,26 @@ module "bootstrap" {
module "masters" {
source = "./master"

elb_api_internal_id = "${module.vpc.aws_elb_api_internal_id}"
elb_api_external_id = "${module.vpc.aws_elb_api_external_id}"
elb_console_id = "${module.vpc.aws_elb_console_id}"
base_domain = "${var.tectonic_base_domain}"
cluster_id = "${var.tectonic_cluster_id}"
cluster_name = "${var.tectonic_cluster_name}"
ec2_type = "${var.tectonic_aws_master_ec2_type}"
extra_tags = "${var.tectonic_aws_extra_tags}"
instance_count = "${var.tectonic_master_count}"
master_iam_role = "${var.tectonic_aws_master_iam_role_name}"
master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
private_endpoints = "${local.private_endpoints}"
public_endpoints = "${local.public_endpoints}"
root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}"
root_volume_size = "${var.tectonic_aws_master_root_volume_size}"
root_volume_type = "${var.tectonic_aws_master_root_volume_type}"
subnet_ids = "${module.vpc.master_subnet_ids}"
ec2_ami = "${var.tectonic_aws_ec2_ami_override}"
user_data_ign = "${var.ignition_master}"
public_target_group_arns = "${module.vpc.aws_lb_public_target_group_arns}"
public_target_group_arns_length = "${module.vpc.aws_lb_public_target_group_arns_length}"
private_target_group_arns = "${module.vpc.aws_lb_private_target_group_arns}"
private_target_group_arns_length = "${module.vpc.aws_lb_private_target_group_arns_length}"
base_domain = "${var.tectonic_base_domain}"
cluster_id = "${var.tectonic_cluster_id}"
cluster_name = "${var.tectonic_cluster_name}"
ec2_type = "${var.tectonic_aws_master_ec2_type}"
extra_tags = "${var.tectonic_aws_extra_tags}"
instance_count = "${var.tectonic_master_count}"
master_iam_role = "${var.tectonic_aws_master_iam_role_name}"
master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
private_endpoints = "${local.private_endpoints}"
public_endpoints = "${local.public_endpoints}"
root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}"
root_volume_size = "${var.tectonic_aws_master_root_volume_size}"
root_volume_type = "${var.tectonic_aws_master_root_volume_type}"
subnet_ids = "${module.vpc.master_subnet_ids}"
ec2_ami = "${var.tectonic_aws_ec2_ami_override}"
user_data_ign = "${var.ignition_master}"
}

module "iam" {
Expand All @@ -68,22 +71,19 @@ module "iam" {
module "dns" {
source = "./route53"

api_external_elb_dns_name = "${module.vpc.aws_elb_api_external_dns_name}"
api_external_elb_zone_id = "${module.vpc.aws_elb_api_external_zone_id}"
api_internal_elb_dns_name = "${module.vpc.aws_elb_api_internal_dns_name}"
api_internal_elb_zone_id = "${module.vpc.aws_elb_api_internal_zone_id}"
api_ip_addresses = "${module.vpc.aws_lbs}"
base_domain = "${var.tectonic_base_domain}"
cluster_name = "${var.tectonic_cluster_name}"
console_elb_dns_name = "${module.vpc.aws_console_dns_name}"
console_elb_zone_id = "${module.vpc.aws_elb_console_zone_id}"
elb_alias_enabled = true
master_count = "${var.tectonic_master_count}"
private_zone_id = "${local.private_zone_id}"
external_vpc_id = "${module.vpc.vpc_id}"
extra_tags = "${var.tectonic_aws_extra_tags}"
private_endpoints = "${local.private_endpoints}"
public_endpoints = "${local.public_endpoints}"
api_external_lb_dns_name = "${module.vpc.aws_lb_api_external_dns_name}"
api_external_lb_zone_id = "${module.vpc.aws_lb_api_external_zone_id}"
api_internal_lb_dns_name = "${module.vpc.aws_lb_api_internal_dns_name}"
api_internal_lb_zone_id = "${module.vpc.aws_lb_api_internal_zone_id}"
base_domain = "${var.tectonic_base_domain}"
cluster_name = "${var.tectonic_cluster_name}"
elb_alias_enabled = true
master_count = "${var.tectonic_master_count}"
private_zone_id = "${local.private_zone_id}"
external_vpc_id = "${module.vpc.vpc_id}"
extra_tags = "${var.tectonic_aws_extra_tags}"
private_endpoints = "${local.private_endpoints}"
public_endpoints = "${local.public_endpoints}"
}

module "vpc" {
Expand Down
22 changes: 9 additions & 13 deletions data/data/aws/master/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,20 +113,16 @@ resource "aws_instance" "master" {
), var.extra_tags)}"
}

resource "aws_elb_attachment" "masters_internal" {
count = "${var.private_endpoints ? var.instance_count : 0}"
elb = "${var.elb_api_internal_id}"
instance = "${aws_instance.master.*.id[count.index]}"
}
resource "aws_lb_target_group_attachment" "public" {
count = "${var.public_endpoints ? var.instance_count * var.public_target_group_arns_length : 0}"

resource "aws_elb_attachment" "masters_external" {
count = "${var.public_endpoints ? var.instance_count : 0}"
elb = "${var.elb_api_external_id}"
instance = "${aws_instance.master.*.id[count.index]}"
target_group_arn = "${var.public_target_group_arns[count.index % var.public_target_group_arns_length]}"
target_id = "${aws_instance.master.*.private_ip[count.index / var.public_target_group_arns_length]}"
}

resource "aws_elb_attachment" "masters_console" {
count = "${var.instance_count}"
elb = "${var.elb_console_id}"
instance = "${aws_instance.master.*.id[count.index]}"
resource "aws_lb_target_group_attachment" "private" {
count = "${var.private_endpoints ? var.instance_count * var.private_target_group_arns_length : 0}"

target_group_arn = "${var.private_target_group_arns[count.index % var.private_target_group_arns_length]}"
target_id = "${aws_instance.master.*.private_ip[count.index / var.private_target_group_arns_length]}"
}
24 changes: 16 additions & 8 deletions data/data/aws/master/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,29 @@ variable "private_endpoints" {
default = true
}

variable "private_target_group_arns" {
type = "list"
default = []
description = "The list of target group ARNs for the private load balancer."
}

variable "private_target_group_arns_length" {
description = "The length of the 'private_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
}

variable "public_endpoints" {
description = "If set to true, public-facing ingress resources are created."
default = true
}

variable "elb_api_internal_id" {
type = "string"
}

variable "elb_api_external_id" {
type = "string"
variable "public_target_group_arns" {
type = "list"
default = []
description = "The list of target group ARNs for the public load balancer."
}

variable "elb_console_id" {
type = "string"
variable "public_target_group_arns_length" {
description = "The length of the 'public_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
}

variable "root_volume_iops" {
Expand Down
88 changes: 9 additions & 79 deletions data/data/aws/route53/tectonic.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,103 +10,33 @@ data "aws_route53_zone" "tectonic" {
locals {
public_zone_id = "${join("", data.aws_route53_zone.tectonic.*.zone_id)}"

zone_id = "${var.private_endpoints ?
var.private_zone_id :
local.public_zone_id}"
}

resource "aws_route53_record" "tectonic_api" {
count = "${var.elb_alias_enabled ? 0 : 1}"
zone_id = "${local.public_zone_id}"
name = "${var.cluster_name}-k8s"
type = "A"
ttl = "60"
records = ["${var.api_ip_addresses}"]
zone_id = "${var.private_endpoints ? var.private_zone_id : local.public_zone_id}"
}

resource "aws_route53_record" "tectonic_api_external" {
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"

zone_id = "${local.public_zone_id}"
name = "${var.cluster_name}-api.${var.base_domain}"
type = "A"

alias {
name = "${var.api_external_elb_dns_name}"
zone_id = "${var.api_external_elb_zone_id}"
name = "${var.api_external_lb_dns_name}"
zone_id = "${var.api_external_lb_zone_id}"
evaluate_target_health = true
}
}

resource "aws_route53_record" "tectonic_api_internal" {
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
zone_id = "${var.private_zone_id}"
name = "${var.cluster_name}-api.${var.base_domain}"
type = "A"

alias {
name = "${var.api_internal_elb_dns_name}"
zone_id = "${var.api_internal_elb_zone_id}"
evaluate_target_health = true
}
}

resource "aws_route53_record" "tectonic-console" {
count = "${var.elb_alias_enabled ? 0 : 1}"
zone_id = "${local.public_zone_id}"
name = "${var.cluster_name}"
type = "A"
ttl = "60"
records = ["${var.worker_ip_addresses}"]
}

resource "aws_route53_record" "tectonic_ingress_public" {
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
zone_id = "${local.public_zone_id}"
name = "${var.cluster_name}.${var.base_domain}"
type = "A"

alias {
name = "${var.console_elb_dns_name}"
zone_id = "${var.console_elb_zone_id}"
evaluate_target_health = true
}
}
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"

resource "aws_route53_record" "tectonic_ingress_private" {
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
zone_id = "${var.private_zone_id}"
name = "${var.cluster_name}.${var.base_domain}"
type = "A"

alias {
name = "${var.console_elb_dns_name}"
zone_id = "${var.console_elb_zone_id}"
evaluate_target_health = true
}
}

resource "aws_route53_record" "routes_ingress_public" {
count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
zone_id = "${local.public_zone_id}"
name = "*.${var.cluster_name}.${var.base_domain}"
type = "A"

alias {
name = "${var.console_elb_dns_name}"
zone_id = "${var.console_elb_zone_id}"
evaluate_target_health = true
}
}

resource "aws_route53_record" "routes_ingress_private" {
count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
zone_id = "${var.private_zone_id}"
name = "*.${var.cluster_name}.${var.base_domain}"
name = "${var.cluster_name}-api.${var.base_domain}"
type = "A"

alias {
name = "${var.console_elb_dns_name}"
zone_id = "${var.console_elb_zone_id}"
name = "${var.api_internal_lb_dns_name}"
zone_id = "${var.api_internal_lb_zone_id}"
evaluate_target_health = true
}
}
Loading

0 comments on commit 16dfbb3

Please sign in to comment.