Chore/update fx

terraform/ansible/k8s-deploy/ansible.tf

@@ -12,7 +12,7 @@ resource "local_sensitive_file" "ansible_inventory" {
       all_hosts_var_maps          = merge(var.all_hosts_var_maps, local.ssh_private_key_file_map, local.all_hosts_var_maps),
       agent_hosts_yaml_maps       = var.agent_hosts_yaml_maps,
       master_hosts_yaml_maps      = var.master_hosts_yaml_maps,
-      bastion_hosts_yaml_maps     = var.bastion_hosts_yaml_maps,
+      bastion_hosts_yaml_maps     = merge(var.bastion_hosts_yaml_maps, local.bastion_hosts_yaml_maps)
       test_harness_hosts          = var.test_harness_hosts,
       test_harness_hosts_var_maps = merge(var.test_harness_hosts_var_maps, local.jumphostmap)
     }
@@ -36,7 +36,7 @@ resource "null_resource" "run_ansible" {
     EOT
     working_dir = path.module
   }
- 
+
   depends_on = [
     local_sensitive_file.ansible_inventory,
     local_sensitive_file.ec2_ssh_key
@@ -53,12 +53,12 @@ resource "null_resource" "destroy_ansible_actions" {
           ansible-playbook "$destroy_ansible_playbook" -i "$destroy_ansible_inventory"
     EOT
     working_dir = path.module
-  } 
+  }
 
- depends_on = [
+  depends_on = [
     local_sensitive_file.ansible_inventory,
     local_sensitive_file.ec2_ssh_key,
- ]
+  ]
 
 }
 
@@ -68,6 +68,19 @@ resource "local_sensitive_file" "ec2_ssh_key" {
   file_permission = "0600"
 }
 
+data "gitlab_project_variable" "external_rds_stateful_resource_instance_address" {
+  for_each = local.managed_rds_stateful_resources
+  project  = var.current_gitlab_project_id
+  key      = each.value.external_resource_config.instance_address_key_name
+}
+
+data "gitlab_project_variable" "external_kafka_stateful_resource_instance_address" {
+  for_each = local.managed_kafka_stateful_resources
+  project  = var.current_gitlab_project_id
+  key      = each.value.external_resource_config.instance_address_key_name
+}
+
+
 locals {
   jumphostmap = {
     ansible_ssh_common_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ProxyCommand=\"ssh -W %h:%p -i ${local_sensitive_file.ec2_ssh_key.filename} -o StrictHostKeyChecking=no -q ${var.ansible_bastion_os_username}@${var.ansible_bastion_public_ip}\""
@@ -80,4 +93,41 @@ locals {
     kubeconfig_local_location = local.ansible_output_dir
   }
 
+  stateful_resources               = jsondecode(file(var.stateful_resources_config_file))
+  enabled_stateful_resources       = { for stateful_resource in local.stateful_resources : stateful_resource.resource_name => stateful_resource if stateful_resource.enabled }
+  managed_rds_stateful_resources   = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service && managed_resource.resource_type == "mysql" }
+  managed_kafka_stateful_resources = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service && managed_resource.resource_type == "kafka" }
+
+
+  external_rds_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_rds_stateful_resource_instance_address : address.key => address.value }
+  external_kafka_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_kafka_stateful_resource_instance_address : address.key => address.value }
+
+
+  managed_kafka_brokers_list                    = { for service in local.managed_kafka_stateful_resources : service.resource_name => split(",", local.external_kafka_stateful_resource_instance_addresses[service.external_resource_config.instance_address_key_name]) }
+
+
+  managed_rds_svc_port_maps = [for service in local.managed_rds_stateful_resources :
+    {
+      "local_listening_port" = service.logical_service_config.logical_service_port
+      "mode"                 = service.communication_mode
+      "name"                 = service.resource_name
+      "dest_fqdn"            = local.external_rds_stateful_resource_instance_addresses[service.external_resource_config.instance_address_key_name]
+      "dest_port"            = service.external_resource_config.port
+    }
+  ]
+
+  managed_kafka_svc_maps = [for service in local.managed_kafka_stateful_resources :
+    {
+      "local_listening_port"       = service.logical_service_config.logical_service_port
+      "managed_kafka_brokers_list" = local.managed_kafka_brokers_list[service.resource_name]
+      "mode"                       = service.communication_mode
+      "name"                       = service.resource_name
+      "dest_port"                  = service.external_resource_config.port
+    }
+
+  ]
+  bastion_hosts_yaml_maps = {
+    managed_rds_svc   = yamlencode(local.managed_rds_svc_port_maps)
+    managed_kafka_svc = yamlencode(local.managed_kafka_svc_maps)
+  }
 }

terraform/ansible/k8s-deploy/variables.tf

@@ -105,3 +105,12 @@ variable "ansible_debug" {
   type    = string
   default = ""
 }
+
+variable "stateful_resources_config_file" {
+  type = string
+}
+
+variable "current_gitlab_project_id" {
+  type        = string
+  description = "current_gitlab_project_id"
+}

terraform/aws/base-infra/data.tf

@@ -1,23 +1,28 @@
 data "aws_route53_zone" "public" {
   count = (var.create_public_zone || !var.configure_route_53) ? 0 : 1
-  name = "${local.cluster_domain}."
+  name  = "${local.cluster_domain}."
+}
+
+data "aws_route53_zone" "public_int" {
+  count = (var.create_public_zone || !var.configure_route_53) ? 0 : 1
+  name  = "${var.private_subdomain_string}.${local.cluster_domain}."
 }
 
 data "aws_route53_zone" "private" {
   count = (var.create_private_zone || !var.configure_route_53) ? 0 : 1
-  name = "${local.cluster_domain}.internal."
+  name  = "${local.cluster_domain}.internal."
 }
 
 data "aws_route53_zone" "cluster_parent" {
   count = (var.manage_parent_domain || !var.configure_route_53) ? 0 : 1
-  name = "${local.cluster_parent_domain}."
+  name  = "${local.cluster_parent_domain}."
 }
 
 data "aws_route53_zone" "cluster_parent_parent" {
   count = (var.manage_parent_domain && var.manage_parent_domain_ns && var.configure_route_53) ? 1 : 0
-  name = "${local.cluster_parent_parent_domain}."
+  name  = "${local.cluster_parent_parent_domain}."
 }
 
 data "aws_availability_zones" "available" {
   state = "available"
-}
+}

terraform/aws/base-infra/infra.tf

@@ -18,6 +18,9 @@ module "vpc" {
   enable_dns_hostnames          = true
   enable_dns_support            = true
   enable_nat_gateway            = true
+  single_nat_gateway            = true
+  one_nat_gateway_per_az        = false
+  reuse_nat_ips                 = false
   manage_default_security_group = false
   manage_default_network_acl    = false
   manage_default_route_table    = false
@@ -34,7 +37,7 @@ module "subnet_addrs" {
 
   base_cidr_block = var.vpc_cidr
   networks = [
-    for subnet in concat(local.private_subnets_list, local.public_subnets_list) : {
+    for subnet in local.subnet_list : {
       name     = subnet
       new_bits = var.block_size
     }

terraform/aws/base-infra/outputs.tf

@@ -22,6 +22,10 @@ output "private_zone" {
   value = local.private_zone
 }
 
+output "public_int_zone" {
+  value = local.public_int_zone
+}
+
 output "public_zone" {
   value = local.public_zone
 }
@@ -62,4 +66,4 @@ output "key_pair_name" {
 output "haproxy_server_fqdn" {
   description = "haproxy server Hostname"
   value       = var.create_haproxy_dns_record ? aws_route53_record.haproxy_server_private[0].fqdn : ""
-}
+}

terraform/aws/base-infra/route53.tf

@@ -1,7 +1,7 @@
 resource "aws_route53_zone" "private" {
   force_destroy = var.route53_zone_force_destroy
-  count = (var.configure_route_53 && var.create_private_zone) ? 1 : 0
-  name  = "${local.cluster_domain}.internal."
+  count         = (var.configure_route_53 && var.create_private_zone) ? 1 : 0
+  name          = "${local.cluster_domain}.internal."
 
   vpc {
     vpc_id = module.vpc.vpc_id
@@ -11,29 +11,45 @@ resource "aws_route53_zone" "private" {
 
 resource "aws_route53_zone" "public" {
   force_destroy = var.route53_zone_force_destroy
-  count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
-  name  = "${local.cluster_domain}."
-  tags = merge({ Name = "${local.cluster_domain}-public" }, local.common_tags)
+  count         = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  name          = "${local.cluster_domain}."
+  tags          = merge({ Name = "${local.cluster_domain}-public" }, local.common_tags)
+}
+
+resource "aws_route53_zone" "public_int" {
+  force_destroy = var.route53_zone_force_destroy
+  count         = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  name          = "${var.private_subdomain_string}.${local.cluster_domain}."
+  tags          = merge({ Name = "${local.cluster_domain}-public-int" }, local.common_tags)
 }
 
 resource "aws_route53_record" "public_ns" {
-  count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  count   = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
   zone_id = local.cluster_parent_zone_id
   name    = local.cluster_domain
   type    = "NS"
   ttl     = "30"
   records = aws_route53_zone.public[0].name_servers
 }
 
+resource "aws_route53_record" "public_int_ns" {
+  count   = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  zone_id = aws_route53_zone.public[0].zone_id
+  name    = "${var.private_subdomain_string}.${local.cluster_domain}"
+  type    = "NS"
+  ttl     = "30"
+  records = aws_route53_zone.public_int[0].name_servers
+}
+
 resource "aws_route53_zone" "cluster_parent" {
   force_destroy = var.route53_zone_force_destroy
-  count = (var.configure_route_53 && var.manage_parent_domain) ? 1 : 0
-  name  = "${local.cluster_parent_domain}."
-  tags = merge({ Name = "${local.cluster_domain}-cluster-parent" }, local.common_tags)
+  count         = (var.configure_route_53 && var.manage_parent_domain) ? 1 : 0
+  name          = "${local.cluster_parent_domain}."
+  tags          = merge({ Name = "${local.cluster_domain}-cluster-parent" }, local.common_tags)
 }
 
 resource "aws_route53_record" "cluster_ns" {
-  count = (var.configure_route_53 && var.manage_parent_domain && var.manage_parent_domain_ns) ? 1 : 0
+  count   = (var.configure_route_53 && var.manage_parent_domain && var.manage_parent_domain_ns) ? 1 : 0
   zone_id = data.aws_route53_zone.cluster_parent_parent[0].zone_id
   name    = local.cluster_parent_domain
   type    = "NS"
@@ -42,10 +58,10 @@ resource "aws_route53_record" "cluster_ns" {
 }
 
 resource "aws_route53_record" "haproxy_server_private" {
-  count = (var.configure_route_53 && var.create_haproxy_dns_record) ? 1 : 0
+  count   = (var.configure_route_53 && var.create_haproxy_dns_record) ? 1 : 0
   zone_id = local.public_zone.id
   name    = "haproxy"
   type    = "A"
   ttl     = "300"
   records = [aws_instance.bastion.private_ip]
-}
+}

terraform/aws/base-infra/variables.tf

@@ -9,7 +9,7 @@ variable "cluster_name" {
 variable "domain" {
   description = "Domain to attach the cluster to."
   type        = string
-  default = ""
+  default     = ""
 }
 
 variable "tags" {
@@ -25,33 +25,33 @@ variable "vpc_cidr" {
 }
 
 variable "configure_route_53" {
-  type = bool
-  default = true
+  type        = bool
+  default     = true
   description = "whether route53 is to be configured at all or not"
 }
 
 variable "create_public_zone" {
-  default = true
-  type = bool
+  default     = true
+  type        = bool
   description = "Whether to create public zone in route53. true or false, default true"
 }
 
 variable "create_private_zone" {
-  default = true
-  type = bool
+  default     = true
+  type        = bool
   description = "Whether to create private zone in route53. true or false, default true"
 }
- variable "manage_parent_domain" {
-  default = false
-  type = bool
+variable "manage_parent_domain" {
+  default     = false
+  type        = bool
   description = "whether parent domain should be created and managed here"
- }
+}
 
- variable "manage_parent_domain_ns" {
-  default = false
-  type = bool
+variable "manage_parent_domain_ns" {
+  default     = false
+  type        = bool
   description = "whether ns record should be created for parent domain in that parent's zone that should already exist"
- }
+}
 
 variable "az_count" {
   type        = number
@@ -70,48 +70,56 @@ variable "bastion_ami" {
 }
 variable "netmaker_ami" {
   description = "ami for netmaker"
-  default = "none for enable_netmaker false"
+  default     = "none for enable_netmaker false"
 }
 
 variable "block_size" {
-  type = number
+  type    = number
   default = 3
 }
 
 variable "enable_netmaker" {
-  type = bool
+  type    = bool
   default = false
 }
 
 variable "netmaker_vpc_cidr" {
-  type = string
+  type    = string
   default = "10.26.0.0/24"
 }
 
- variable "create_haproxy_dns_record" {
-  default = false
-  type = bool
+variable "create_haproxy_dns_record" {
+  default     = false
+  type        = bool
   description = "whether to create public dns record for private ip of bastion for haproxy"
- }
+}
+
+variable "private_subdomain_string" {
+  type    = string
+  default = "int"
+}
+
 
 ###
 # Local copies of variables to allow for parsing
 ###
 locals {
-  name = var.cluster_name
-  cluster_domain     = "${replace(var.cluster_name, "-", "")}.${var.domain}"
-  cluster_parent_domain   = join(".", [for idx, part in split(".", local.cluster_domain) : part if idx > 0])
-  cluster_parent_parent_domain = join(".", [for idx, part in split(".", local.cluster_parent_domain) : part if idx > 0])
-  identifying_tags = { Cluster = var.cluster_name, Domain = local.cluster_domain}
-  common_tags = merge(local.identifying_tags, var.tags)
-  azs = slice(data.aws_availability_zones.available.names, 0, var.az_count)
-  public_zone = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public[0] : data.aws_route53_zone.public[0]) : null
-  private_zone = var.configure_route_53 ? (var.create_private_zone ? aws_route53_zone.private[0] : data.aws_route53_zone.private[0]) : null
-  cluster_parent_zone_id = var.configure_route_53 ? (var.manage_parent_domain ? aws_route53_zone.cluster_parent[0].zone_id : data.aws_route53_zone.cluster_parent[0].zone_id) : null
+  name                          = var.cluster_name
+  cluster_domain                = "${replace(var.cluster_name, "-", "")}.${var.domain}"
+  cluster_parent_domain         = join(".", [for idx, part in split(".", local.cluster_domain) : part if idx > 0])
+  cluster_parent_parent_domain  = join(".", [for idx, part in split(".", local.cluster_parent_domain) : part if idx > 0])
+  identifying_tags              = { Cluster = var.cluster_name, Domain = local.cluster_domain }
+  common_tags                   = merge(local.identifying_tags, var.tags)
+  azs                           = slice(data.aws_availability_zones.available.names, 0, var.az_count)
+  public_zone                   = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public[0] : data.aws_route53_zone.public[0]) : null
+  private_zone                  = var.configure_route_53 ? (var.create_private_zone ? aws_route53_zone.private[0] : data.aws_route53_zone.private[0]) : null
+  public_int_zone               = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public_int[0] : data.aws_route53_zone.public_int[0]) : null
+  cluster_parent_zone_id        = var.configure_route_53 ? (var.manage_parent_domain ? aws_route53_zone.cluster_parent[0].zone_id : data.aws_route53_zone.cluster_parent[0].zone_id) : null
   cluster_parent_parent_zone_id = var.configure_route_53 ? ((var.manage_parent_domain && var.manage_parent_domain_ns) ? data.aws_route53_zone.cluster_parent_parent[0].zone_id : null) : null
-  ssh_keys = []
-  public_subnets_list  = [for az in local.azs : "public-${az}"]
-  private_subnets_list = [for az in local.azs : "private-${az}"]
-  public_subnet_cidrs  = [for subnet_name in local.public_subnets_list : module.subnet_addrs.network_cidr_blocks[subnet_name]]
-  private_subnet_cidrs = [for subnet_name in local.private_subnets_list : module.subnet_addrs.network_cidr_blocks[subnet_name]]
-}
+  ssh_keys                      = []
+  public_subnets_list           = [for az in local.azs : "public-${az}"]
+  private_subnets_list          = [for az in local.azs : "private-${az}"]
+  subnet_list                   = flatten([for az in local.azs : concat(["private-${az}", "public-${az}"])])
+  public_subnet_cidrs           = [for subnet_name in local.public_subnets_list : module.subnet_addrs.network_cidr_blocks[subnet_name]]
+  private_subnet_cidrs          = [for subnet_name in local.private_subnets_list : module.subnet_addrs.network_cidr_blocks[subnet_name]]
+}