From e3cb2a4ad80b4a9f3ff204ac09493f5546880aa3 Mon Sep 17 00:00:00 2001 From: Chia-Yi Liang Date: Fri, 18 Nov 2022 01:50:55 -0800 Subject: [PATCH] Enabled creation of autoscaling nodepools and regional clusters (#2813) Added options to the GKE terraform install for creating a regional cluster and enabling nodepool autoscaling. --- build/Makefile | 1 + build/README.md | 5 +++- build/includes/google-cloud.mk | 13 +++++++--- build/includes/terraform.mk | 18 +++++++++++-- build/terraform/gke/module.tf | 25 +++++++++++++++++-- examples/terraform-submodules/gke/module.tf | 25 +++++++++++++++++-- install/terraform/modules/gke/cluster.tf | 18 ++++++++++--- install/terraform/modules/gke/variables.tf | 5 +++- .../en/docs/Installation/Terraform/gke.md | 12 +++++++++ 9 files changed, 107 insertions(+), 15 deletions(-) diff --git a/build/Makefile b/build/Makefile index c0a20cd0ec..cb7038383a 100644 --- a/build/Makefile +++ b/build/Makefile @@ -43,6 +43,7 @@ KUBECONFIG ?= ~/.kube/config # The (gcloud) test cluster that is being worked against GCP_CLUSTER_NAME ?= test-cluster GCP_CLUSTER_ZONE ?= us-west1-c +GCP_CLUSTER_LOCATION ?= $(GCP_CLUSTER_ZONE) GCP_BUCKET_CHARTS ?= agones-chart # the profile to use when developing on minikube MINIKUBE_PROFILE ?= agones diff --git a/build/README.md b/build/README.md index b50e99dc0c..4bce5bc097 100644 --- a/build/README.md +++ b/build/README.md @@ -199,7 +199,10 @@ See the table below for available customizations : | Parameter | Description | Default | |------------------------------------------------|---------------------------------------------------------------------------------------|-----------------| | `GCP_CLUSTER_NAME` | The name of the cluster | `test-cluster` | -| `GCP_CLUSTER_ZONE` | The name of the Google Compute Engine zone in which the cluster will resides. | `us-west1-c` | +| `GCP_CLUSTER_ZONE` or `GCP_CLUSTER_LOCATION` | The name of the Google Compute Engine zone/location in which the cluster will resides | `us-west1-c` | +| `GCP_CLUSTER_NODEPOOL_AUTOSCALE` | Whether or not to enable autoscaling on game server nodepool | `false` | +| `GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT` | The number of minimum nodes if autoscale is enabled | `1` | +| `GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT` | The number of maximum nodes if autoscale is enabled | `5` | | `GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT` | The number of nodes to create in this cluster. | `4` | | `GCP_CLUSTER_NODEPOOL_MACHINETYPE` | The name of a Google Compute Engine machine type. | `e2-standard-4` | | `GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING` | Whether or not to enable image streaming for the `"default"` node pool in the cluster | `true` | diff --git a/build/includes/google-cloud.mk b/build/includes/google-cloud.mk index f1c3717f3a..df92fc58bb 100644 --- a/build/includes/google-cloud.mk +++ b/build/includes/google-cloud.mk @@ -27,10 +27,17 @@ gcloud-init: ensure-build-config gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= e2-standard-4 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING ?= true +gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_AUTOSCALE ?= false +gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT ?= 1 +gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT ?= 5 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= e2-standard-4 gcloud-test-cluster: $(ensure-build-image) - $(MAKE) gcloud-terraform-cluster GCP_TF_CLUSTER_NAME="$(GCP_CLUSTER_NAME)" GCP_CLUSTER_ZONE="$(GCP_CLUSTER_ZONE)" \ + $(MAKE) gcloud-terraform-cluster GCP_TF_CLUSTER_NAME="$(GCP_CLUSTER_NAME)" \ + GCP_CLUSTER_LOCATION="$(GCP_CLUSTER_LOCATION)" \ + GCP_CLUSTER_AUTOSCALE="$(GCP_CLUSTER_NODEPOOL_AUTOSCALE)" \ + GCP_CLUSTER_MIN_NODECOUNT="$(GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT)" \ + GCP_CLUSTER_MAX_NODECOUNT="$(GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT)" \ GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT="$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)" \ GCP_CLUSTER_NODEPOOL_MACHINETYPE="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING="$(GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING)" \ @@ -73,9 +80,7 @@ clean-gcloud-prow-build-cluster: $(ensure-build-image) # Pulls down authentication information for kubectl against a cluster, name can be specified through GCP_CLUSTER_NAME # (defaults to 'test-cluster') gcloud-auth-cluster: $(ensure-build-image) - docker run --rm $(common_mounts) $(build_tag) gcloud config set container/cluster $(GCP_CLUSTER_NAME) - docker run --rm $(common_mounts) $(build_tag) gcloud config set compute/zone $(GCP_CLUSTER_ZONE) - docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME) + docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME) --zone $(GCP_CLUSTER_LOCATION) # authenticate our docker configuration so that you can do a docker push directly # to the gcr.io repository diff --git a/build/includes/terraform.mk b/build/includes/terraform.mk index 530c80c7fb..abf5ccb01e 100644 --- a/build/includes/terraform.mk +++ b/build/includes/terraform.mk @@ -38,6 +38,9 @@ terraform-clean: gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= e2-standard-4 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING ?= true +gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_AUTOSCALE ?= false +gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT ?= 1 +gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT ?= 5 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= e2-standard-4 gcloud-terraform-cluster: AGONES_VERSION ?= '' @@ -53,8 +56,12 @@ gcloud-terraform-cluster: -var name=$(GCP_TF_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ -var values_file="" \ -var feature_gates=$(FEATURE_GATES) \ - -var zone="$(GCP_CLUSTER_ZONE)" -var project="$(GCP_PROJECT)" \ + -var project="$(GCP_PROJECT)" \ + -var location="$(GCP_CLUSTER_LOCATION)" \ -var log_level="$(LOG_LEVEL)" \ + -var autoscale=$(GCP_CLUSTER_NODEPOOL_AUTOSCALE) \ + -var min_node_count=$(GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT) \ + -var max_node_count=$(GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT) \ -var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT) \ -var enable_image_streaming=$(GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING) \ -var windows_node_count=$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT) \ @@ -66,6 +73,9 @@ gcloud-terraform-cluster: # Unifies previous `make gcloud-test-cluster` and `make install` targets gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= e2-standard-4 +gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_AUTOSCALE ?= false +gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT ?= 1 +gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT ?= 5 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= e2-standard-4 gcloud-terraform-install: ALWAYS_PULL_SIDECAR := true @@ -87,8 +97,12 @@ gcloud-terraform-install: -var crd_cleanup="$(CRD_CLEANUP)" \ -var chart="../../../install/helm/agones/" \ -var name=$(GCP_TF_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \ - -var zone=$(GCP_CLUSTER_ZONE) -var project=$(GCP_PROJECT) \ + -var project=$(GCP_PROJECT) \ + -var location=$(GCP_CLUSTER_LOCATION) \ -var log_level=$(LOG_LEVEL) \ + -var autoscale=$(GCP_CLUSTER_NODEPOOL_AUTOSCALE) \ + -var min_node_count=$(GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT) \ + -var max_node_count=$(GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT) \ -var feature_gates=$(FEATURE_GATES) \ -var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT) \ -var windows_node_count=$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT) \ diff --git a/build/terraform/gke/module.tf b/build/terraform/gke/module.tf index 91ec7de1d4..6704a6b364 100644 --- a/build/terraform/gke/module.tf +++ b/build/terraform/gke/module.tf @@ -63,9 +63,14 @@ variable "ping_service_type" { default = "LoadBalancer" } -variable "zone" { +variable "location" { default = "us-west1-c" - description = "The GCP zone to create the cluster in" + description = "The GCP location to create the cluster in" +} + +variable "zone" { + default = "" + description = "The GCP zone to create the cluster in (deprecated, use `location`)" } variable "pull_policy" { @@ -88,6 +93,18 @@ variable "log_level" { default = "info" } +variable "autoscale" { + default = "false" +} + +variable "min_node_count" { + default = "1" +} + +variable "max_node_count" { + default = "5" +} + // Note: This is the number of gameserver nodes. The Agones module will automatically create an additional // two node pools with 1 node each for "agones-system" and "agones-metrics". variable "node_count" { @@ -116,6 +133,7 @@ module "gke_cluster" { cluster = { "name" = var.name + "location" = var.location "zone" = var.zone "machineType" = var.machine_type "initialNodeCount" = var.node_count @@ -124,6 +142,9 @@ module "gke_cluster" { "windowsInitialNodeCount" = var.windows_node_count "project" = var.project "network" = var.network + "autoscale" = var.autoscale + "minNodeCount" = var.min_node_count + "maxNodeCount" = var.max_node_count } } diff --git a/examples/terraform-submodules/gke/module.tf b/examples/terraform-submodules/gke/module.tf index 111b5e0bda..4654a0bbc8 100644 --- a/examples/terraform-submodules/gke/module.tf +++ b/examples/terraform-submodules/gke/module.tf @@ -53,9 +53,14 @@ variable "enable_image_streaming" { default = "true" } -variable "zone" { +variable "location" { default = "us-west1-c" - description = "The GCP zone to create the cluster in" + description = "The GCP location to create the cluster in" +} + +variable "zone" { + default = "" + description = "The GCP zone to create the cluster in (deprecated, use `location`)" } variable "network" { @@ -84,6 +89,18 @@ variable "windows_machine_type" { default = "e2-standard-4" } +variable "autoscale" { + default = "false" +} + +variable "min_node_count" { + default = "1" +} + +variable "max_node_count" { + default = "5" +} + module "gke_cluster" { // *************************************************************************************************** // Update ?ref= to the agones release you are installing. For example, ?ref=release-1.17.0 corresponds @@ -93,6 +110,7 @@ module "gke_cluster" { cluster = { "name" = var.name + "location" = var.location "zone" = var.zone "machineType" = var.machine_type "initialNodeCount" = var.node_count @@ -102,6 +120,9 @@ module "gke_cluster" { "subnetwork" = var.subnetwork "windowsInitialNodeCount" = var.windows_node_count "windowsMachineType" = var.windows_machine_type + "autoscale" = var.autoscale + "mindNodeCount" = var.min_node_count + "maxNodeCount" = var.max_node_count } } diff --git a/install/terraform/modules/gke/cluster.tf b/install/terraform/modules/gke/cluster.tf index 95fad710f5..bca5541365 100644 --- a/install/terraform/modules/gke/cluster.tf +++ b/install/terraform/modules/gke/cluster.tf @@ -23,7 +23,8 @@ data "google_client_config" "default" {} # Set values to default if not key was not set in original map locals { project = lookup(var.cluster, "project", "agones") - zone = lookup(var.cluster, "zone", "us-west1-c") + location = lookup(var.cluster, "location", "us-west1-c") + zone = lookup(var.cluster, "zone", "") name = lookup(var.cluster, "name", "test-cluster") machineType = lookup(var.cluster, "machineType", "e2-standard-4") initialNodeCount = lookup(var.cluster, "initialNodeCount", "4") @@ -33,6 +34,9 @@ locals { kubernetesVersion = lookup(var.cluster, "kubernetesVersion", "1.23") windowsInitialNodeCount = lookup(var.cluster, "windowsInitialNodeCount", "0") windowsMachineType = lookup(var.cluster, "windowsMachineType", "e2-standard-4") + autoscale = lookup(var.cluster, "autoscale", false) + minNodeCount = lookup(var.cluster, "minNodeCount", "1") + maxNodeCount = lookup(var.cluster, "maxNodeCount", "5") } # echo command used for debugging purpose @@ -56,7 +60,7 @@ resource "null_resource" "test-setting-variables" { resource "google_container_cluster" "primary" { name = local.name - location = local.zone + location = local.zone != "" ? local.zone : local.location project = local.project network = local.network subnetwork = local.subnetwork @@ -65,9 +69,17 @@ resource "google_container_cluster" "primary" { node_pool { name = "default" - node_count = local.initialNodeCount + node_count = local.autoscale ? null : local.initialNodeCount version = local.kubernetesVersion + dynamic "autoscaling" { + for_each = local.autoscale ? [1] : [] + content { + min_node_count = local.minNodeCount + max_node_count = local.maxNodeCount + } + } + management { auto_upgrade = false } diff --git a/install/terraform/modules/gke/variables.tf b/install/terraform/modules/gke/variables.tf index 9f7331c3b5..2067ef1d01 100644 --- a/install/terraform/modules/gke/variables.tf +++ b/install/terraform/modules/gke/variables.tf @@ -30,7 +30,7 @@ variable "cluster" { type = map default = { - "zone" = "us-west1-c" + "location" = "us-west1-c" "name" = "test-cluster" "machineType" = "e2-standard-4" "initialNodeCount" = "4" @@ -40,6 +40,9 @@ variable "cluster" { "kubernetesVersion" = "1.23" "windowsInitialNodeCount" = "0" "windowsMachineType" = "e2-standard-4" + "autoscale" = false + "minNodeCount" = "1" + "maxNodeCount" = "5" } } diff --git a/site/content/en/docs/Installation/Terraform/gke.md b/site/content/en/docs/Installation/Terraform/gke.md index 2687880084..c4b3195663 100644 --- a/site/content/en/docs/Installation/Terraform/gke.md +++ b/site/content/en/docs/Installation/Terraform/gke.md @@ -86,8 +86,14 @@ Configurable parameters: - machine_type - machine type for hosting game servers (default is "e2-standard-4") - node_count - count of game server nodes for the default node pool (default is "4") - enable_image_streaming - whether or not to enable image streaming for the `"default"` node pool (default is true) +{{% feature expiryVersion="1.28" %}} - zone - the name of the [zone](https://cloud.google.com/compute/docs/regions-zones) you want your cluster to be created in (default is "us-west1-c") +{{% /feature %}} +{{% feature publishVersion="1.28" %}} +- zone - (Deprecated, use location) the name of the [zone](https://cloud.google.com/compute/docs/regions-zones) you want your cluster to be + created in (default is "us-west1-c") +{{% /feature %}} - network - the name of the VPC network you want your cluster and firewall rules to be connected to (default is "default") - subnetwork - the name of the subnetwork in which the cluster's instances are launched. (required when using non default network) - log_level - possible values: Fatal, Error, Warn, Info, Debug (default is "info") @@ -96,6 +102,12 @@ Configurable parameters: - gameserver_maxPort - the upper bound of the port range which gameservers will listen on (default is "8000") - gameserver_namespaces - a list of namespaces which will be used to run gameservers (default is `["default"]`). For example `["default", "xbox-gameservers", "mobile-gameservers"]` - force_update - whether or not to force the replacement/update of resource (default is true, false may be required to prevent immutability errors when updating the configuration) +{{% feature publishVersion="1.28" %}} +- location - the name of the [location](https://cloud.google.com/compute/docs/regions-zones) you want your cluster to eb created in (default is "us-west1-c") +- autoscale - whether you want to enable autoscale for the gameserver nodepool (default is false) +- min_node_count - the minimum number of nodes for a nodepool when autoscale is enabled (default is "1") +- max_node_count - the maximum number of nodes for a nodepool when autoscale is enabled (default is "5") +{{% /feature %}} {{% alert title="Warning" color="warning"%}} On the lines that read `source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=main"`