FEATURE/ added new KOPS K8s layer (#600)

* FEATURE/ added new KOPS K8s layer

* FEATURE/ improved docs

* FEATURE/ fixed var name and value

apps-devstg/us-east-1/k8s-kops/.gitignore

@@ -0,0 +1 @@
+*k8s.local

apps-devstg/us-east-1/k8s-kops/1-prerequisites/config.tf

@@ -0,0 +1,53 @@
+# Providers
+provider "aws" {
+  region  = var.region_primary
+  profile = var.profile
+}
+
+#replica provider
+provider "aws" {
+  alias   = "region_secondary"
+  region  = var.region_secondary
+  profile = var.profile
+}
+
+provider "aws" {
+  alias   = "shared"
+  region  = var.region
+  profile = "${var.project}-shared-devops"
+}
+
+# Backend Config (partial)
+terraform {
+  required_version = "~> 1.3"
+
+  required_providers {
+    aws = "~> 4.10"
+  }
+
+  backend "s3" {
+    key = "apps-devstg/ca-central-1/k8s-kops/1-prerequisites/terraform.tfstate"
+  }
+}
+
+data "terraform_remote_state" "vpc" {
+  backend = "s3"
+
+  config = {
+    region  = var.region
+    profile = var.profile
+    bucket  = var.bucket
+    key     = "${var.environment}/ca-central-1/kops-network/terraform.tfstate"
+  }
+}
+
+data "terraform_remote_state" "vpc-shared" {
+  backend = "s3"
+
+  config = {
+    region  = var.region
+    profile = "${var.project}-shared-devops"
+    bucket  = "${var.project}-shared-terraform-backend"
+    key     = "shared/us-east-1/network/terraform.tfstate"
+  }
+}

apps-devstg/us-east-1/k8s-kops/1-prerequisites/data.tf

@@ -0,0 +1,3 @@
+data "aws_iam_roles" "devopsrole" {
+  name_regex = ".*AWSReservedSSO_DevOps.*"
+}

apps-devstg/us-east-1/k8s-kops/1-prerequisites/dns.tf

@@ -0,0 +1,57 @@
+#
+# Private Hosted Zone for this k8s cluster
+#
+resource "aws_route53_zone" "cluster_domain" {
+  count = local.gossip_cluster ? 0 : 1
+
+  name = local.k8s_cluster_name
+
+  vpc {
+    vpc_id     = data.terraform_remote_state.vpc.outputs.vpc_id
+    vpc_region = var.region
+  }
+
+  #
+  # This Remote Account VPCs are added as a post step after the local-exec assoc occurs.
+  # If you won't like to add them please consider the below workaround
+  # Had to add this ignore override because of the cross-vpc resolution
+  # between shared and vpc-dev
+  # between shared and vpc-dev-eks
+  #
+  #lifecycle {
+  #      ignore_changes = [
+  #          vpc,
+  #      ]
+  #  }
+
+  ## IMPORTANT!!! ##
+  # Needs to be uncommented after the -> resource "null_resource" "create_remote_zone_auth"
+  # is established
+  vpc {
+    vpc_id     = data.terraform_remote_state.vpc-shared.outputs.vpc_id
+    vpc_region = var.region
+  }
+}
+
+#
+# DNS/VPC association between Shared VPC and cluster-kops-1.k8s.devstg.binbash.aws
+#
+
+# Authorize association from the owner account of the Private Zone
+resource "aws_route53_vpc_association_authorization" "with_shared_vpc" {
+  count = local.gossip_cluster ? 0 : 1
+
+  vpc_id  = data.terraform_remote_state.vpc-shared.outputs.vpc_id
+  zone_id = aws_route53_zone.cluster_domain[0].zone_id
+}
+
+
+# Complete the association from the owner account of the VPC
+resource "aws_route53_zone_association" "with_shared_vpc" {
+  count = local.gossip_cluster ? 0 : 1
+
+  provider = aws.shared
+
+  vpc_id  = aws_route53_vpc_association_authorization.with_shared_vpc[0].vpc_id
+  zone_id = aws_route53_vpc_association_authorization.with_shared_vpc[0].zone_id
+}

apps-devstg/us-east-1/k8s-kops/1-prerequisites/locals.tf

@@ -0,0 +1,59 @@
+locals {
+  tags = {
+    Terraform   = "true"
+    Environment = var.environment
+  }
+
+  # We'll use a shorter environment name in order to keep things simple
+  short_environment = replace(var.environment, "apps-", "")
+
+  # The name of the cluster
+  # if gossip_cluster then base_domain_name must be k8s.local
+  gossip_cluster   = true
+  base_domain_name = "k8s.local"
+  k8s_cluster_name = "canada01-kops.${local.short_environment}.${local.base_domain_name}"
+
+  # The kubernetes version
+  k8s_cluster_version = "1.28.9"
+
+  # The etcd version
+  # Ref1: https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#etcdclusters-v3--tls
+  # Ref2: https://github.com/etcd-io/etcd/releases
+  etcd_clusters_version = "3.5.9"
+
+  # The Calico Network CNI version
+  # Ref1: https://github.com/kubernetes/kops/blob/master/docs/calico-v3.md
+  # Ref2: https://itnext.io/benchmark-results-of-kubernetes-network-plugins-cni-over-10gbit-s-network-36475925a560
+  networking_calico_major_version = "v3"
+
+  # Kops AMI Identifier
+  # check image in https://cloud-images.ubuntu.com/locator/ec2/ , look for your zone.
+  kops_ami_id = "ami-04fea581fe25e2675"
+
+  # Tags that will be applied to all K8s Kops cluster instances
+  cluster_tags = {
+    "kubernetes.io/cluster/${local.k8s_cluster_name}" = "owned"
+  }
+
+  # Tags that will be applied to all K8s Kops Worker nodes
+  node_cloud_labels = {}
+
+  # K8s Kops Master Nodes Machine (EC2) type and size + ASG Min-Max per AZ
+  # then min/max = 1 will create 1 Master Node x AZ => 3 x Masters
+  kops_master_machine_type     = "t3.medium"
+  kops_master_machine_max_size = 1
+  kops_master_machine_min_size = 1
+
+  # K8s Kops Worker Nodes Machine (EC2) type and size + ASG Min-Max
+  kops_worker_machine_type     = "t3.medium"
+  kops_worker_machine_max_size = 5
+  kops_worker_machine_min_size = 1
+  # If you use Karpenter set the list of types here
+  kops_worker_machine_types_karpenter = ["t2.medium", "t2.large", "t3.medium", "t3.large", "t3a.medium", "t3a.large", "m4.large"]
+
+  # master nodes AZs
+  # number_of_cluster_master_azs is 1, 2 or 3
+  # master nodes will be deployed in these AZs
+  number_of_cluster_master_azs = 1
+  cluster_master_azs           = slice(data.terraform_remote_state.vpc.outputs.availability_zones, 0, local.number_of_cluster_master_azs )
+}

apps-devstg/us-east-1/k8s-kops/1-prerequisites/outputs.tf

@@ -0,0 +1,160 @@
+#
+# Cluster Settings
+#
+output "project_short" {
+  description = "Project Short Name"
+  value       = var.project
+}
+output "profile" {
+  description = "AWS Profile"
+  value       = var.profile
+}
+output "region" {
+  description = "AWS Region"
+  value       = var.region
+}
+output "environment" {
+  description = "AWS Region"
+  value       = var.environment
+}
+output "cluster_name" {
+  description = "The name of this cluster"
+  value       = local.k8s_cluster_name
+}
+output "cluster_version" {
+  description = "Kubernetes version"
+  value       = local.k8s_cluster_version
+}
+output "etcd_clusters_version" {
+  description = "etcd version"
+  value       = local.etcd_clusters_version
+}
+output "networking_calico_major_version" {
+  description = "Calico network CNI major version"
+  value       = local.networking_calico_major_version
+}
+
+#
+# Cluster Master Instance Group (IG)
+#
+output "cluster_master_azs" {
+  description = "Availability Zones where masters will be deployed"
+  value = local.cluster_master_azs
+}
+output "cluster_api_elb_extra_security_group" {
+  value = ""
+}
+output "kops_master_machine_type" {
+  description = "K8s Kops Master Nodes Machine (EC2) type and size"
+  value       = local.kops_master_machine_type
+}
+output "kops_master_machine_max_size" {
+  description = "K8s Kops Master Nodes ASG max size"
+  value       = local.kops_master_machine_max_size
+}
+output "kops_master_machine_min_size" {
+  description = "K8s Kops Master Nodes ASG min size"
+  value       = local.kops_master_machine_min_size
+}
+
+#
+# Cluster Worker Nodes Instance Group (IG)
+#
+output "node_cloud_labels" {
+  description = "Cloud labels will become tags on the nodes"
+  value       = local.node_cloud_labels
+}
+output "kops_worker_machine_type" {
+  description = "K8s Kops Worker Nodes Machine (EC2) type and size"
+  value       = local.kops_worker_machine_type
+}
+output "kops_worker_machine_max_size" {
+  description = "K8s Kops Worker Nodes ASG max size"
+  value       = local.kops_worker_machine_max_size
+}
+output "kops_worker_machine_min_size" {
+  description = "K8s Kops Worker Nodes ASG min size"
+  value       = local.kops_worker_machine_min_size
+}
+
+#
+# Kops Resources
+#
+output "kops_s3_bucket" {
+  description = "Kops State S3 Bucket"
+  value       = aws_s3_bucket.kops_state.bucket
+}
+output "kops_ami_id" {
+  description = "Kops AMI ID"
+  value       = local.kops_ami_id
+}
+
+#
+# Network Resources
+#
+output "gossip_cluster" {
+  description = "Wheter is a gossip cluster"
+  value       = local.gossip_cluster
+}
+output "hosted_zone_id" {
+  description = "Hosted Zone ID (Kops requires a domain for the cluster)"
+  value       = local.gossip_cluster ? null : aws_route53_zone.cluster_domain[0].zone_id
+}
+output "vpc_id" {
+  value = data.terraform_remote_state.vpc.outputs.vpc_id
+}
+output "vpc_cidr_block" {
+  value = data.terraform_remote_state.vpc.outputs.vpc_cidr_block
+}
+output "availability_zones" {
+  value = data.terraform_remote_state.vpc.outputs.availability_zones
+}
+output "public_subnet_ids" {
+  value = zipmap(
+    data.terraform_remote_state.vpc.outputs.availability_zones,
+    data.terraform_remote_state.vpc.outputs.public_subnets
+  )
+}
+output "private_subnet_ids" {
+  value = zipmap(
+    data.terraform_remote_state.vpc.outputs.availability_zones,
+    data.terraform_remote_state.vpc.outputs.private_subnets
+  )
+}
+output "nat_gateway_ids" {
+  value = zipmap(
+    data.terraform_remote_state.vpc.outputs.availability_zones, [for az in data.terraform_remote_state.vpc.outputs.availability_zones: data.terraform_remote_state.vpc.outputs.nat_gateway_ids[0]]
+  )
+}
+output "shared_vpc_cidr_block" {
+  value = data.terraform_remote_state.vpc-shared.outputs.vpc_cidr_block
+}
+output "ssh_pub_key_path" {
+  value = var.ssh_pub_key_path
+}
+
+output "devops_role" {
+  value = tolist(data.aws_iam_roles.devopsrole.arns)[0]
+}
+
+#
+# IRSA
+#
+output "irsa_enabled" {
+  value = var.enable_irsa
+}
+output "irsa_bucket_name" {
+  value = var.enable_irsa ? aws_s3_bucket.kops_irsa[0].id : ""
+}
+
+#
+# KARPENTER
+#
+output "karpenter_enabled" {
+  value = var.enable_irsa && var.enable_karpenter
+}
+
+output "kops_worker_machine_types_karpenter" {
+  description = "List of K8s Kops Worker Nodes Machine (EC2) types and size for Karpenter"
+  value       = local.kops_worker_machine_types_karpenter
+}