Add Terraform config for Windows clusters (#2046)

Co-authored-by: Mark Mandel <[email protected]>

build/README.md

@@ -51,13 +51,13 @@ Table of Contents
         * [make grafana-portforward](#make-grafana-portforward)
         * [make controller-portforward](#make-controller-portforward)
         * [make pprof-cpu-web](#make-pprof-cpu-web)
-        * [make pprof-heap-web](#make-pprof-heap-web)          
+        * [make pprof-heap-web](#make-pprof-heap-web)
         * [make shell](#make-shell)
         * [make godoc](#make-godoc)
         * [make build-controller-image](#make-build-controller-image)
         * [make build-agones-sdk-image](#make-build-agones-sdk-image)
         * [make gen-install](#make-gen-install)
-        * [make gen-embedded-openapi](#make-gen-embedded-openapi)  
+        * [make gen-embedded-openapi](#make-gen-embedded-openapi)
         * [make gen-crd-client](#make-gen-crd-client)
         * [make gen-sdk-grpc](#make-gen-sdk-grpc)
      * [Build Image Targets](#build-image-targets)
@@ -201,12 +201,14 @@ configuration found in the `build/terraform/gke` directory.
 You can customize GKE cluster via environment variables or by using a [`local-includes`](./local-includes) file.
 See the table below for available customizations :
 
-| Parameter                             | Description                                                                   | Default       |
-|---------------------------------------|-------------------------------------------------------------------------------|---------------|
-| `GCP_CLUSTER_NAME`                    | The name of the cluster                                                       | `test-cluster`  |
-| `GCP_CLUSTER_ZONE`                    | The name of the Google Compute Engine zone in which the cluster will resides. |  `us-west1-c`   |
-| `GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT`| The number of nodes to create in this cluster.                                |  `4`            |
-| `GCP_CLUSTER_NODEPOOL_MACHINETYPE`    | The name of a Google Compute Engine machine type.                             | `n1-standard-4` |
+| Parameter                                      | Description                                                                   | Default         |
+|------------------------------------------------|-------------------------------------------------------------------------------|-----------------|
+| `GCP_CLUSTER_NAME`                             | The name of the cluster                                                       | `test-cluster`  |
+| `GCP_CLUSTER_ZONE`                             | The name of the Google Compute Engine zone in which the cluster will resides. | `us-west1-c`    |
+| `GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT`        | The number of nodes to create in this cluster.                                | `4`             |
+| `GCP_CLUSTER_NODEPOOL_MACHINETYPE`             | The name of a Google Compute Engine machine type.                             | `n1-standard-4` |
+| `GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT` | The number of Windows nodes to create in this cluster.                        | `0`             |
+| `GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE`      | The name of a Google Compute Engine machine type for Windows nodes.           | `n1-standard-4` |
 
 If you would like to change more settings, feel free to edit the [`cluster.yml.jinja`](./gke-test-cluster/cluster.yml.jinja) file before running this command.
 
@@ -282,7 +284,7 @@ It's worth noting that Minikube does let you [reuse its Docker daemon](https://g
 and build directly on Minikube, but in this case this approach is far simpler,
 and makes cross-platform support for the build system much easier.
 
-To push your own images into the cluster, take a look at Minikube's 
+To push your own images into the cluster, take a look at Minikube's
 [Pushing Images](https://minikube.sigs.k8s.io/docs/handbook/pushing/) guide.
 
 Running end-to-end tests on Minikube is done via the `make minikube-test-e2e` target. This target use the same `make test-e2e` but also setup some prerequisites for use with a Minikube cluster.
@@ -670,7 +672,7 @@ Use this instead of `make install`, as it disables PullAlways on the install.yam
 
 #### `make minikube-push`
 
-Push the local Agones Docker images that have already been built 
+Push the local Agones Docker images that have already been built
 via `make build` or `make build-images` into the "agones" minikube instance with `minikube cache add`
 
 #### `make minikube-setup-prometheus`

build/includes/google-cloud.mk

@@ -26,9 +26,14 @@ gcloud-init: ensure-build-config
 # Creates and authenticates a small, 6 node GKE cluster to work against (2 nodes are used for agones-metrics and agones-system)
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4
+gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0
+gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= n1-standard-4
 gcloud-test-cluster: $(ensure-build-image)
 	$(MAKE) gcloud-terraform-cluster GCP_TF_CLUSTER_NAME="$(GCP_CLUSTER_NAME)" GCP_CLUSTER_ZONE="$(GCP_CLUSTER_ZONE)" \
-		GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT="$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)" GCP_CLUSTER_NODEPOOL_MACHINETYPE="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)"
+		GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT="$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)" \
+		GCP_CLUSTER_NODEPOOL_MACHINETYPE="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \
+		GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT="$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT)" \
+		GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE="$(GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE)"
 	$(MAKE) gcloud-auth-cluster
 
 clean-gcloud-test-cluster: $(ensure-build-image)

build/includes/terraform.mk

@@ -37,6 +37,8 @@ terraform-clean:
 # Alpha Feature gates are disabled
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4
+gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0
+gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= n1-standard-4
 gcloud-terraform-cluster: AGONES_VERSION ?= ''
 gcloud-terraform-cluster: GCP_TF_CLUSTER_NAME ?= agones-tf-cluster
 gcloud-terraform-cluster: LOG_LEVEL ?= debug
@@ -52,14 +54,18 @@ gcloud-terraform-cluster:
 		-var feature_gates=$(FEATURE_GATES) \
 		-var zone="$(GCP_CLUSTER_ZONE)" -var project="$(GCP_PROJECT)" \
 		-var log_level="$(LOG_LEVEL)" \
-		-var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)'
+		-var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT) \
+		-var windows_node_count=$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT) \
+		-var windows_machine_type=$(GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE)'
 	GCP_CLUSTER_NAME=$(GCP_TF_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster
 
 # Creates a cluster and install current version of Agones controller
 # Set all necessary variables as `make install` does
 # Unifies previous `make gcloud-test-cluster` and `make install` targets
 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4
+gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0
+gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= n1-standard-4
 gcloud-terraform-install: ALWAYS_PULL_SIDECAR := true
 gcloud-terraform-install: IMAGE_PULL_POLICY := "Always"
 gcloud-terraform-install: PING_SERVICE_TYPE := "LoadBalancer"
@@ -82,7 +88,9 @@ gcloud-terraform-install:
 		-var zone=$(GCP_CLUSTER_ZONE) -var project=$(GCP_PROJECT) \
 		-var log_level=$(LOG_LEVEL) \
 		-var feature_gates=$(FEATURE_GATES) \
-		-var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)'
+		-var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT) \
+		-var windows_node_count=$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT) \
+		-var windows_machine_type=$(GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE)'
 	GCP_CLUSTER_NAME=$(GCP_TF_CLUSTER_NAME) $(MAKE) gcloud-auth-cluster
 
 gcloud-terraform-destroy-cluster: GCP_PROJECT ?= $(current_project)

build/terraform/e2e/module.tf

@@ -28,11 +28,11 @@ module "gke_cluster" {
   source = "../../../install/terraform/modules/gke"
 
   cluster = {
-    "name"              = "e2e-test-cluster"
-    "zone"              = "us-west1-c"
-    "machineType"       = "n1-standard-4"
-    "initialNodeCount"  = 8
-    "project"           = var.project
+    "name"             = "e2e-test-cluster"
+    "zone"             = "us-west1-c"
+    "machineType"      = "n1-standard-4"
+    "initialNodeCount" = 8
+    "project"          = var.project
   }
 
   firewallName = "gke-game-server-firewall"
@@ -50,15 +50,15 @@ provider "helm" {
 
 resource "helm_release" "consul" {
   chart = "stable/consul"
-  name = "consul"
+  name  = "consul"
 
   set {
-    name = "Replicas"
+    name  = "Replicas"
     value = "1"
   }
 
   set {
-    name = "uiService.type"
+    name  = "uiService.type"
     value = "ClusterIP"
   }
 }

build/terraform/gke/module.tf

@@ -37,6 +37,10 @@ variable "machine_type" {
   default = "n1-standard-4"
 }
 
+variable "windows_machine_type" {
+  default = "n1-standard-4"
+}
+
 variable "name" {
   default = "agones-tf-cluster"
 }
@@ -87,6 +91,10 @@ variable "log_level" {
 variable "node_count" {
   default = "4"
 }
+// Note: This is the number of gameserver Windows nodes.
+variable "windows_node_count" {
+  default = "0"
+}
 
 variable "network" {
   default     = "default"
@@ -101,12 +109,14 @@ module "gke_cluster" {
   source = "../../../install/terraform/modules/gke"
 
   cluster = {
-    "name"              = var.name
-    "zone"              = var.zone
-    "machineType"       = var.machine_type
-    "initialNodeCount"  = var.node_count
-    "project"           = var.project
-    "network"           = var.network
+    "name"                    = var.name
+    "zone"                    = var.zone
+    "machineType"             = var.machine_type
+    "initialNodeCount"        = var.node_count
+    "windowsMachineType"      = var.windows_machine_type
+    "windowsInitialNodeCount" = var.windows_node_count
+    "project"                 = var.project
+    "network"                 = var.network
   }
 }
 

build/terraform/prow/module.tf

@@ -21,10 +21,10 @@ variable "project" {
 }
 
 resource "google_container_cluster" "prow-build-cluster" {
-  name = "prow-build-cluster"
-  project = var.project
-  location = "us-west1-c"
-  description = "Prow cluster to run tests for Agones"
+  name               = "prow-build-cluster"
+  project            = var.project
+  location           = "us-west1-c"
+  description        = "Prow cluster to run tests for Agones"
   min_master_version = "1.18"
   initial_node_count = 8
   node_config {

examples/terraform-submodules/gke/module.tf

@@ -69,6 +69,14 @@ variable "feature_gates" {
   default = ""
 }
 
+variable "windows_node_count" {
+  default = "0"
+}
+
+variable "windows_machine_type" {
+  default = "n1-standard-4"
+}
+
 module "gke_cluster" {
   // ***************************************************************************************************
   // Update ?ref= to the agones release you are installing. For example, ?ref=release-1.8.0 corresponds
@@ -77,13 +85,15 @@ module "gke_cluster" {
   source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=main"
 
   cluster = {
-    "name"             = var.name
-    "zone"             = var.zone
-    "machineType"      = var.machine_type
-    "initialNodeCount" = var.node_count
-    "project"          = var.project
-    "network"          = var.network
-    "subnetwork"       = var.subnetwork
+    "name"                    = var.name
+    "zone"                    = var.zone
+    "machineType"             = var.machine_type
+    "initialNodeCount"        = var.node_count
+    "project"                 = var.project
+    "network"                 = var.network
+    "subnetwork"              = var.subnetwork
+    "windowsInitialNodeCount" = var.windows_node_count
+    "windowsMachineType"      = var.windows_machine_type
   }
 }
 

install/terraform/modules/aks/aks.tf

@@ -89,8 +89,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "system" {
   os_disk_size_gb       = var.disk_size
   enable_auto_scaling   = false
   node_taints           = ["agones.dev/agones-system=true:NoExecute"]
-  node_labels           = {
-    "agones.dev/agones-system":"true"
+  node_labels = {
+    "agones.dev/agones-system" : "true"
   }
 }
 
@@ -102,8 +102,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "metrics" {
   os_disk_size_gb       = var.disk_size
   enable_auto_scaling   = false
   node_taints           = ["agones.dev/agones-metrics=true:NoExecute"]
-  node_labels           = {
-    "agones.dev/agones-metrics":"true"
+  node_labels = {
+    "agones.dev/agones-metrics" : "true"
   }
 }
 

install/terraform/modules/gke/cluster.tf

@@ -22,25 +22,33 @@ data "google_client_config" "default" {}
 # A list of all parameters used in interpolation var.cluster
 # Set values to default if not key was not set in original map
 locals {
-  project           = lookup(var.cluster, "project", "agones")
-  zone              = lookup(var.cluster, "zone", "us-west1-c")
-  name              = lookup(var.cluster, "name", "test-cluster")
-  machineType       = lookup(var.cluster, "machineType", "n1-standard-4")
-  initialNodeCount  = lookup(var.cluster, "initialNodeCount", "4")
-  network           = lookup(var.cluster, "network", "default")
-  subnetwork        = lookup(var.cluster, "subnetwork", "")
-  kubernetesVersion = lookup(var.cluster, "kubernetesVersion", "1.18")
+  project                 = lookup(var.cluster, "project", "agones")
+  zone                    = lookup(var.cluster, "zone", "us-west1-c")
+  name                    = lookup(var.cluster, "name", "test-cluster")
+  machineType             = lookup(var.cluster, "machineType", "n1-standard-4")
+  initialNodeCount        = lookup(var.cluster, "initialNodeCount", "4")
+  network                 = lookup(var.cluster, "network", "default")
+  subnetwork              = lookup(var.cluster, "subnetwork", "")
+  kubernetesVersion       = lookup(var.cluster, "kubernetesVersion", "1.18")
+  windowsInitialNodeCount = lookup(var.cluster, "windowsInitialNodeCount", "0")
+  windowsMachineType      = lookup(var.cluster, "windowsMachineType", "n1-standard-4")
 }
 
 # echo command used for debugging purpose
 # Run `terraform taint null_resource.test-setting-variables` before second execution
 resource "null_resource" "test-setting-variables" {
   provisioner "local-exec" {
     command = <<EOT
-    ${format("echo Current variables set as following - name: %s, project: %s, machineType: %s, initialNodeCount: %s, network: %s, zone: %s",
-    local.name, local.project,
-    local.machineType, local.initialNodeCount, local.network,
-local.zone)}
+    ${format("echo Current variables set as following - name: %s, project: %s, machineType: %s, initialNodeCount: %s, network: %s, zone: %s, windowsInitialNodeCount: %s, windowsMachineType: %s",
+    local.name,
+    local.project,
+    local.machineType,
+    local.initialNodeCount,
+    local.network,
+    local.zone,
+    local.windowsInitialNodeCount,
+    local.windowsMachineType,
+)}
     EOT
 }
 }
@@ -57,7 +65,7 @@ resource "google_container_cluster" "primary" {
   node_pool {
     name       = "default"
     node_count = local.initialNodeCount
-    version = local.kubernetesVersion
+    version    = local.kubernetesVersion
 
     management {
       auto_upgrade = false
@@ -81,7 +89,7 @@ resource "google_container_cluster" "primary" {
   node_pool {
     name       = "agones-system"
     node_count = 1
-    version = local.kubernetesVersion
+    version    = local.kubernetesVersion
 
     management {
       auto_upgrade = false
@@ -113,7 +121,7 @@ resource "google_container_cluster" "primary" {
   node_pool {
     name       = "agones-metrics"
     node_count = 1
-    version = local.kubernetesVersion
+    version    = local.kubernetesVersion
 
     management {
       auto_upgrade = false
@@ -142,6 +150,42 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+  dynamic "ip_allocation_policy" {
+    for_each = tonumber(local.windowsInitialNodeCount) > 0 ? [1] : []
+    content {
+      # Enable Alias IPs to allow Windows Server networking.
+      cluster_ipv4_cidr_block  = "/14"
+      services_ipv4_cidr_block = "/20"
+    }
+  }
+  dynamic "node_pool" {
+    for_each = tonumber(local.windowsInitialNodeCount) > 0 ? [1] : []
+    content {
+      name       = "windows"
+      node_count = local.windowsInitialNodeCount
+      version    = local.kubernetesVersion
+
+      management {
+        auto_upgrade = false
+      }
+
+      node_config {
+        image_type   = "WINDOWS_LTSC"
+        machine_type = local.windowsMachineType
+
+        oauth_scopes = [
+          "https://www.googleapis.com/auth/devstorage.read_only",
+          "https://www.googleapis.com/auth/logging.write",
+          "https://www.googleapis.com/auth/monitoring",
+          "https://www.googleapis.com/auth/service.management.readonly",
+          "https://www.googleapis.com/auth/servicecontrol",
+          "https://www.googleapis.com/auth/trace.append",
+        ]
+
+        tags = ["game-server"]
+      }
+    }
+  }
   timeouts {
     create = "30m"
     update = "40m"