From 6dd46d133edc317e36fe0eb1aecbebd3c36fe6f0 Mon Sep 17 00:00:00 2001 From: Naveen Mandadhi Date: Wed, 12 Jun 2024 15:56:28 -0500 Subject: [PATCH] chore: add gke island cluster anywhere in GCP design (#1967) --- .../README.md | 37 ++++ .../main.tf | 93 ++++++++ .../manifests/k8s.yaml | 88 ++++++++ .../network.tf | 207 ++++++++++++++++++ .../outputs.tf | 19 ++ .../router.tf | 54 +++++ .../terraform.tfvars | 46 ++++ .../variables.tf | 78 +++++++ .../versions.tf | 28 +++ 9 files changed, 650 insertions(+) create mode 100644 examples/island_cluster_anywhere_in_gcp_design/README.md create mode 100644 examples/island_cluster_anywhere_in_gcp_design/main.tf create mode 100644 examples/island_cluster_anywhere_in_gcp_design/manifests/k8s.yaml create mode 100644 examples/island_cluster_anywhere_in_gcp_design/network.tf create mode 100644 examples/island_cluster_anywhere_in_gcp_design/outputs.tf create mode 100644 examples/island_cluster_anywhere_in_gcp_design/router.tf create mode 100644 examples/island_cluster_anywhere_in_gcp_design/terraform.tfvars create mode 100644 examples/island_cluster_anywhere_in_gcp_design/variables.tf create mode 100644 examples/island_cluster_anywhere_in_gcp_design/versions.tf diff --git a/examples/island_cluster_anywhere_in_gcp_design/README.md b/examples/island_cluster_anywhere_in_gcp_design/README.md new file mode 100644 index 0000000000..73a18ff5ca --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/README.md @@ -0,0 +1,37 @@ +# GKE island cluster anywhere in GCP design + +This example provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters across different GCP organizations. + +## Deploy + +1. Create NCC hub. +2. Update `ncc_hub_project_id`, `ncc_hub_name`, `network_name` and gke spokes in `terraform.tfvars`. +3. Run `terraform apply`. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| gke\_spokes | n/a | `any` | n/a | yes | +| ingress\_ip\_addrs\_subnet\_cidr | Subnet to use for reserving internal ip addresses for the ILBs. | `string` | n/a | yes | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes | +| ncc\_hub\_name | n/a | `string` | n/a | yes | +| ncc\_hub\_project\_id | n/a | `string` | n/a | yes | +| net\_attachment\_subnet\_cidr | Subnet for the router PSC interface network attachment in island network. | `string` | n/a | yes | +| node\_locations | n/a | `list(string)` | n/a | yes | +| primary\_net\_name | Primary VPC network name. | `string` | n/a | yes | +| primary\_subnet | Subnet to use in primary network to deploy the router. | `string` | n/a | yes | +| proxy\_subnet\_cidr | CIDR for the regional managed proxy subnet. | `string` | n/a | yes | +| region | n/a | `string` | n/a | yes | +| router\_machine\_type | n/a | `string` | n/a | yes | +| secondary\_ranges | n/a | `map(string)` | n/a | yes | +| subnet\_cidr | Primary subnet CIDR used by the cluster. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_ids | n/a | + + diff --git a/examples/island_cluster_anywhere_in_gcp_design/main.tf b/examples/island_cluster_anywhere_in_gcp_design/main.tf new file mode 100644 index 0000000000..8dbbe9894b --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/main.tf @@ -0,0 +1,93 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_id" "rand" { + byte_length = 4 +} + +resource "google_service_account" "gke-sa" { + for_each = { for k, v in var.gke_spokes : k => v } + + account_id = "gke-sa-${random_id.rand.hex}" + project = each.value["project_id"] +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" + version = "~> 31.0" + + for_each = { for k, v in var.gke_spokes : k => v } + + name = each.value["cluster_name"] + project_id = each.value["project_id"] + region = var.region + release_channel = "RAPID" + zones = var.node_locations + network = module.net[each.key].network_name + subnetwork = "${each.value["cluster_name"]}-${var.region}-snet" + ip_range_pods = "${each.value["cluster_name"]}-${var.region}-snet-pods" + ip_range_services = "${each.value["cluster_name"]}-${var.region}-snet-services" + enable_private_endpoint = true + enable_private_nodes = true + datapath_provider = "ADVANCED_DATAPATH" + monitoring_enable_managed_prometheus = false + enable_shielded_nodes = true + master_global_access_enabled = false + master_ipv4_cidr_block = var.secondary_ranges["master_cidr"] + master_authorized_networks = var.master_authorized_networks + deletion_protection = false + remove_default_node_pool = true + disable_default_snat = true + gateway_api_channel = "CHANNEL_STANDARD" + + node_pools = [ + { + name = "default" + machine_type = "e2-highcpu-2" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = true + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = google_service_account.gke-sa[each.key].email + initial_node_count = 1 + enable_secure_boot = true + }, + ] + + node_pools_tags = { + all = ["gke-${random_id.rand.hex}"] + } + + node_pools_oauth_scopes = { + all = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + } + + timeouts = { + create = "15m" + update = "15m" + delete = "15m" + } +} diff --git a/examples/island_cluster_anywhere_in_gcp_design/manifests/k8s.yaml b/examples/island_cluster_anywhere_in_gcp_design/manifests/k8s.yaml new file mode 100644 index 0000000000..b728ce8a59 --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/manifests/k8s.yaml @@ -0,0 +1,88 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: whereami +spec: + replicas: 3 + selector: + matchLabels: + app: whereami + template: + metadata: + labels: + app: whereami + spec: + containers: + - name: whereami + image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.19 + ports: + - name: http + containerPort: 8080 + resources: + requests: + cpu: "50m" + memory: 128Mi + limits: + cpu: "100m" + memory: 256Mi + readinessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + timeoutSeconds: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: whereami +spec: + type: ClusterIP + selector: + app: whereami + ports: + - port: 80 + targetPort: 8080 + protocol: TCP +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: l7-ilb +spec: + gatewayClassName: gke-l7-rilb + listeners: + - name: http + protocol: HTTP + port: 80 + addresses: + - type: NamedAddress + value: gke-spoke-1-l7-rilb-ip +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: whereami +spec: + parentRefs: + - kind: Gateway + name: l7-ilb + rules: + - backendRefs: + - name: whereami + port: 80 diff --git a/examples/island_cluster_anywhere_in_gcp_design/network.tf b/examples/island_cluster_anywhere_in_gcp_design/network.tf new file mode 100644 index 0000000000..a0539c8252 --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/network.tf @@ -0,0 +1,207 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "net" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + + for_each = { for k, v in var.gke_spokes : k => v } + + network_name = "gke-net-${random_id.rand.hex}" + routing_mode = "GLOBAL" + project_id = each.value["project_id"] + + subnets = [ + { + subnet_name = "${each.value["cluster_name"]}-${var.region}-snet" + subnet_ip = var.subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + }, + { + subnet_name = "${each.value["cluster_name"]}-${var.region}-int-ip-addr-snet" + subnet_ip = var.ingress_ip_addrs_subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + }, + { + subnet_name = "${each.value["cluster_name"]}-${var.region}-net-attachment-snet" + subnet_ip = var.net_attachment_subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + }, + { + subnet_name = "${each.value["cluster_name"]}-${var.region}-proxy-snet" + subnet_ip = var.proxy_subnet_cidr + subnet_region = var.region + purpose = "REGIONAL_MANAGED_PROXY" + role = "ACTIVE" + }, + { + subnet_name = "${each.value["cluster_name"]}-${var.region}-private-nat-snet" + subnet_ip = each.value["private_nat_subnet_cidr"] + subnet_region = var.region + subnet_private_access = "true" + purpose = "PRIVATE_NAT" + }, + ] + + secondary_ranges = { + "${each.value["cluster_name"]}-${var.region}-snet" = [ + { + range_name = "${each.value["cluster_name"]}-${var.region}-snet-pods" + ip_cidr_range = var.secondary_ranges["pods"] + }, + { + range_name = "${each.value["cluster_name"]}-${var.region}-snet-services" + ip_cidr_range = var.secondary_ranges["services"] + }, + ] + } + + firewall_rules = [ + { + name = "${each.value["cluster_name"]}-iap" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + ports = ["22"] + } + ] + ranges = ["35.235.240.0/20"] + }, + { + name = "${each.value["cluster_name"]}-tcp-primary" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [ + var.net_attachment_subnet_cidr + ] + }, + { + name = "${each.value["cluster_name"]}-allow-proxy" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [var.proxy_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa[each.key].email] + }, + ] +} + +resource "google_compute_route" "primary_to_spoke" { + for_each = { for k, v in var.gke_spokes : k => v } + + name = "primary-to-spoke-for-${each.value["cluster_name"]}" + description = "primary to GKE spoke through router" + project = var.ncc_hub_project_id + network = var.primary_net_name + dest_range = each.value["spoke_netmap_subnet"] + next_hop_instance = google_compute_instance.vm[each.key].id +} + +resource "google_network_connectivity_spoke" "spoke" { + provider = google-beta + for_each = { for k, v in var.gke_spokes : k => v } + + name = "${each.value["cluster_name"]}-spoke-${random_id.rand.hex}" + project = each.value["project_id"] + location = "global" + description = "vpc spoke for inter vpc nat" + hub = "projects/${var.ncc_hub_project_id}/locations/global/hubs/${var.ncc_hub_name}" + linked_vpc_network { + exclude_export_ranges = [ + var.subnet_cidr, + var.ingress_ip_addrs_subnet_cidr, + var.net_attachment_subnet_cidr, + var.secondary_ranges["pods"], + var.secondary_ranges["services"], + var.secondary_ranges["master_cidr"], + var.proxy_subnet_cidr + ] + uri = module.net[each.key].network_self_link + } +} + +resource "google_compute_network_attachment" "router_net_attachment" { + provider = google-beta + for_each = { for k, v in var.gke_spokes : k => v } + + name = "net-attachment-${each.value["cluster_name"]}" + project = each.value["project_id"] + region = var.region + description = "router network attachment for cluster ${each.value["cluster_name"]}" + connection_preference = "ACCEPT_MANUAL" + + subnetworks = [ + module.net[each.key].subnets["${var.region}/${each.value["cluster_name"]}-${var.region}-net-attachment-snet"]["self_link"] + ] + + producer_accept_lists = [ + var.ncc_hub_project_id + ] +} + +module "cloud_router" { + source = "terraform-google-modules/cloud-router/google" + version = "~> 6.0" + for_each = { for k, v in var.gke_spokes : k => v } + + name = "router-${each.value["cluster_name"]}-${random_id.rand.hex}" + project = each.value["project_id"] + network = module.net[each.key].network_name + region = var.region +} + +resource "google_compute_router_nat" "nat_type" { + provider = google-beta + depends_on = [module.cloud_router] + + for_each = { for k, v in var.gke_spokes : k => v } + + name = "private-nat-${random_id.rand.hex}" + router = "router-${each.value["cluster_name"]}-${random_id.rand.hex}" + project = each.value["project_id"] + region = var.region + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + type = "PRIVATE" + rules { + rule_number = 100 + description = "rule for private nat" + match = "nexthop.hub == \"//networkconnectivity.googleapis.com/projects/${var.ncc_hub_project_id}/locations/global/hubs/${var.ncc_hub_name}\"" + action { + source_nat_active_ranges = [ + module.net[each.key].subnets["${var.region}/${each.value["cluster_name"]}-${var.region}-private-nat-snet"]["self_link"] + ] + } + } +} + +resource "google_compute_address" "gke-l7-rilb-ip" { + for_each = { for k, v in var.gke_spokes : k => v } + + name = "${each.value["cluster_name"]}-l7-rilb-ip" + address_type = "INTERNAL" + region = var.region + project = each.value["project_id"] + subnetwork = module.net[each.key].subnets["${var.region}/${each.value["cluster_name"]}-${var.region}-int-ip-addr-snet"]["self_link"] +} diff --git a/examples/island_cluster_anywhere_in_gcp_design/outputs.tf b/examples/island_cluster_anywhere_in_gcp_design/outputs.tf new file mode 100644 index 0000000000..438d3f7be0 --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_ids" { + value = [for c in module.gke : c.cluster_id] +} diff --git a/examples/island_cluster_anywhere_in_gcp_design/router.tf b/examples/island_cluster_anywhere_in_gcp_design/router.tf new file mode 100644 index 0000000000..3e91f328de --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/router.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_instance" "vm" { + provider = google-beta + for_each = { for k, v in var.gke_spokes : k => v } + + project = var.ncc_hub_project_id + zone = var.node_locations[0] + name = "${each.value["cluster_name"]}-router-${random_id.rand.hex}" + machine_type = var.router_machine_type + allow_stopping_for_update = true + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + } + } + can_ip_forward = true + shielded_instance_config { + enable_secure_boot = true + } + network_interface { + subnetwork = var.primary_subnet + } + network_interface { + network_attachment = google_compute_network_attachment.router_net_attachment[each.key].self_link + } + metadata_startup_script = <<-EOT + #!/bin/bash + set -ex + sudo apt-get update + echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p + sudo iptables -t nat -A PREROUTING -d ${each.value["spoke_netmap_subnet"]} -i ens4 -j NETMAP --to ${var.ingress_ip_addrs_subnet_cidr} + GWY_URL="http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/1/gateway" + GWY_IP=$(curl $${GWY_URL} -H "Metadata-Flavor: Google") + sudo ip route add ${var.ingress_ip_addrs_subnet_cidr} via $${GWY_IP} dev ens5 + sudo iptables -t nat -A POSTROUTING -o ens4 -j MASQUERADE + sudo iptables -t nat -A POSTROUTING -o ens5 -j MASQUERADE + EOT +} diff --git a/examples/island_cluster_anywhere_in_gcp_design/terraform.tfvars b/examples/island_cluster_anywhere_in_gcp_design/terraform.tfvars new file mode 100644 index 0000000000..ebdd1f0e13 --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/terraform.tfvars @@ -0,0 +1,46 @@ +ncc_hub_project_id = "" +ncc_hub_name = "" +region = "us-central1" +primary_net_name = "" +primary_subnet = "projects//regions/us-central1/subnetworks/" +gke_spokes = [ + { + project_id = "", + cluster_name = "gke-spoke-1", + private_nat_subnet_cidr = "100.65.1.0/24", + spoke_netmap_subnet = "10.244.0.0/28" + }, + { + project_id = "", + cluster_name = "gke-spoke-2", + private_nat_subnet_cidr = "100.65.2.0/24", + spoke_netmap_subnet = "10.244.0.16/28" + }, + { + project_id = "", + cluster_name = "gke-spoke-3", + private_nat_subnet_cidr = "100.65.3.0/24", + spoke_netmap_subnet = "10.244.0.32/28" + } +] +node_locations = [ + "us-central1-a", + "us-central1-b", + "us-central1-f" +] +subnet_cidr = "100.64.0.0/19" +net_attachment_subnet_cidr = "100.64.87.0/29" +router_machine_type = "n2-highcpu-4" +secondary_ranges = { + pods = "100.64.32.0/19" + services = "100.64.64.0/20" + master_cidr = "100.64.96.32/28" +} +proxy_subnet_cidr = "100.64.83.0/24" +ingress_ip_addrs_subnet_cidr = "100.64.84.0/28" +master_authorized_networks = [ + { + cidr_block = "100.64.0.0/10" + display_name = "cluster net" + } +] diff --git a/examples/island_cluster_anywhere_in_gcp_design/variables.tf b/examples/island_cluster_anywhere_in_gcp_design/variables.tf new file mode 100644 index 0000000000..293165c18f --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/variables.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "region" { + type = string +} + +variable "node_locations" { + type = list(string) +} + +variable "subnet_cidr" { + type = string + description = "Primary subnet CIDR used by the cluster." +} + +variable "net_attachment_subnet_cidr" { + type = string + description = "Subnet for the router PSC interface network attachment in island network." +} + +variable "ingress_ip_addrs_subnet_cidr" { + type = string + description = "Subnet to use for reserving internal ip addresses for the ILBs." +} + +variable "proxy_subnet_cidr" { + type = string + description = "CIDR for the regional managed proxy subnet." +} + +variable "secondary_ranges" { + type = map(string) +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +} + +variable "primary_net_name" { + type = string + description = "Primary VPC network name." +} + +variable "ncc_hub_project_id" { + type = string +} + +variable "ncc_hub_name" { + type = string +} + +variable "router_machine_type" { + type = string +} + +variable "primary_subnet" { + type = string + description = "Subnet to use in primary network to deploy the router." +} + +variable "gke_spokes" { + type = any +} diff --git a/examples/island_cluster_anywhere_in_gcp_design/versions.tf b/examples/island_cluster_anywhere_in_gcp_design/versions.tf new file mode 100644 index 0000000000..4818f24fa7 --- /dev/null +++ b/examples/island_cluster_anywhere_in_gcp_design/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + } + google-beta = { + source = "hashicorp/google-beta" + } + } +}