From f6444b3c4121b11629552c497329bbf1d0b6acef Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Thu, 21 Mar 2019 17:07:22 +0300 Subject: [PATCH] Add helm terraform config Deploying agones using two options: 1. From repository release charts by specifying version. 2. Local version using Chart.yaml file with values.yaml. --- build/README.md | 12 +- build/cluster.tf | 59 +++--- build/helm.tf | 196 ++++++++++++++++++ build/includes/google-cloud.mk | 38 +++- examples/terraform-submodule/module.tf | 47 +++++ .../content/en/docs/Installation/terraform.md | 85 ++++++++ 6 files changed, 404 insertions(+), 33 deletions(-) create mode 100644 build/helm.tf create mode 100644 examples/terraform-submodule/module.tf create mode 100644 site/content/en/docs/Installation/terraform.md diff --git a/build/README.md b/build/README.md index b895cf9ffb..fc9d4c0731 100644 --- a/build/README.md +++ b/build/README.md @@ -557,12 +557,22 @@ Utilities for deploying a Kubernetes Engine cluster on Google Cloud Platform usi Install `google` and `google-beta` terraform providers and authorize. #### `make gcloud-terraform-cluster` -Run next command with your project ID specified: +Create GKE cluster and install release version of agones. +Run next command to create GKE cluster with agones (version from helm repository): ``` [GKE_PASSWORD=""] make gcloud-terraform-cluster ``` Where `` should be at least 16 characters in length. You can omit GKE_PASSWORD and then basic auth would be disabled. Also you change `ports="7000-8000"` setting using tfvars file. Also you can define password `password=` string in `build/terraform.tfvars`. +Change AGONES_VERSION to a specific version you want to install. + +#### `make gcloud-terraform-install` +Create GKE cluster and install current version of agones. +The current version should be built and pushed to release_registry beforehand: +``` +make build-images +make push +``` #### `make gcloud-terraform-destroy-cluster` Run `terraform destroy` on your cluster. diff --git a/build/cluster.tf b/build/cluster.tf index c2975c5de1..849bc5671c 100644 --- a/build/cluster.tf +++ b/build/cluster.tf @@ -13,10 +13,14 @@ # limitations under the License. provider "google-beta" { + version = "~> 2.4" zone = "${lookup(var.cluster, "zone")}" } +provider "google" { + version = "~> 2.4" +} # Password for the Kubernetes API. # Could be defined using GKE_PASSWORD env variable # or by setting `password="somepass"` string in build/terraform.tfvars @@ -81,8 +85,9 @@ resource "google_container_cluster" "primary" { enable_legacy_abac = "${lookup(var.cluster, "legacyAbac")}" node_pool = [ { + name = "default" node_count = "${lookup(var.cluster, "initialNodeCount")}" - node_config { + node_config = { machine_type = "${lookup(var.cluster, "machineType")}" oauth_scopes = [ "https://www.googleapis.com/auth/devstorage.read_only", @@ -94,42 +99,42 @@ resource "google_container_cluster" "primary" { ] tags = ["game-server"] - timeouts { + timeouts = { create = "30m" update = "40m" } } }, { - name = "agones-system" - node_count = 1 - node_config { - preemptible = true - machine_type = "n1-standard-4" - - oauth_scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/trace.append", - ] - labels = { - "stable.agones.dev/agones-system" = "true" - } - taint = { - key = "stable.agones.dev/agones-system" - value = "true" - effect = "NO_EXECUTE" + name = "agones-system" + node_count = 1 + node_config = { + preemptible = true + machine_type = "n1-standard-4" + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + labels = { + "stable.agones.dev/agones-system" = "true" + } + taint = { + key = "stable.agones.dev/agones-system" + value = "true" + effect = "NO_EXECUTE" + } } - } }, { name = "agones-metrics" node_count = 1 - node_config { + node_config = { preemptible = true machine_type = "n1-standard-4" @@ -155,7 +160,7 @@ resource "google_container_cluster" "primary" { } resource "google_compute_firewall" "default" { - name = "game-server-firewall-firewall" + name = "game-server-firewall-firewall-${lookup(var.cluster, "name")}" project = "${lookup(var.cluster, "project")}" network = "${google_compute_network.default.name}" @@ -169,7 +174,7 @@ resource "google_compute_firewall" "default" { resource "google_compute_network" "default" { project = "${lookup(var.cluster, "project")}" - name = "agones-network" + name = "agones-network-${lookup(var.cluster, "name")}" } diff --git a/build/helm.tf b/build/helm.tf new file mode 100644 index 0000000000..8285d2a391 --- /dev/null +++ b/build/helm.tf @@ -0,0 +1,196 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "kubernetes_service_account" "tiller" { + metadata { + name = "tiller" + namespace = "kube-system" + } + depends_on = ["google_container_cluster.primary"] + + automount_service_account_token = true +} + +resource "kubernetes_cluster_role_binding" "tiller" { + depends_on = ["kubernetes_service_account.tiller"] + metadata { + name = "tiller" + } + + role_ref { + kind = "ClusterRole" + name = "cluster-admin" + api_group = "rbac.authorization.k8s.io" + } + + subject { + kind = "ServiceAccount" + name = "tiller" + + api_group = "" + namespace = "kube-system" + } +} +variable "chart" { + default = "../install/helm/agones/" +} + +variable "agones_version" { + default = "" +} + +variable "crd_cleanup" { + default = "true" +} +variable "image_registry" { + default = "gcr.io/agones-images" +} +variable "pull_policy" { + default = "Always" +} +variable "always_pull_sidecar" { + default = "true" +} +variable "image_pull_secret" { + default = "" +} +variable "ping_service_type" { + default = "LoadBalancer" +} + +variable "values_file" { + default = "../install/helm/agones/values.yaml" +} +data "google_client_config" "default" {} + +provider "kubernetes" { + version = "~> 1.5" + load_config_file = false + host = "https://${google_container_cluster.primary.endpoint}" + token = "${data.google_client_config.default.access_token}" + cluster_ca_certificate = "${base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)}" +} + +provider "helm" { + version = "~> 0.7" + + debug = true + install_tiller = true + service_account = "${kubernetes_service_account.tiller.metadata.0.name}" + tiller_image = "gcr.io/kubernetes-helm/tiller:v2.12.3" + + kubernetes { + load_config_file = false + host = "https://${google_container_cluster.primary.endpoint}" + token = "${data.google_client_config.default.access_token}" + cluster_ca_certificate = "${base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)}" + } +} + +# In Terraform version 0.12 Interpolation would only evaluate one branch of a condition +# https://github.com/hashicorp/terraform/issues/15605 +# so we can remove this and change values in helm_release to: +# +# values = [ +# "${length(var.values_file) == 0 ? "" : file("${var.values_file}"))}" +# ] +data "null_data_source" "values_file" { + count = "${length(var.values_file) == 0 ? 0 : 1}" + inputs = { + "values" = "${file("${var.values_file}")}" + } +} + +data "google_client_config" "current" {} + +data "helm_repository" "agones" { + depends_on = ["kubernetes_cluster_role_binding.tiller"] + name = "agones" + url = "https://agones.dev/chart/stable" +} + + +# TODO: remove - not needed in Terraform 0.12 +locals { + values = { + params = "${join("", data.null_data_source.values_file.*.outputs.values)}" + } + # Skip image tag if it is not needed + # for installing latest image it would use chart value + tag_name = "${var.agones_version != "" ? "agones.image.tag" : "skip"}" +} + +resource "helm_release" "agones" { + depends_on = ["null_resource.helm_init", "kubernetes_cluster_role_binding.tiller"] + name = "agones" + force_update = "true" + repository = "${data.helm_repository.agones.metadata.0.name}" + chart = "${var.chart}" + timeout = 180 + + values = [ + # Switch to in terraform 0.12 + # "${length(var.values_file) == 0 ? "" : file("${var.values_file}"))}" + "${length(var.values_file) == 0 ? "" : local.values["params"]}" + ] + + set { + name = "crds.CleanupOnDelete" + value = "${var.crd_cleanup}" + } + set { + name = "${local.tag_name}" + value = "${var.agones_version}" + } + set { + name = "agones.image.registry" + value = "${var.image_registry}" + } + set { + name = "agones.image.controller.pullPolicy" + value = "${var.pull_policy}" + } + set { + name = "agones.image.sdk.alwaysPull" + value = "${var.always_pull_sidecar}" + } + set { + name = "agones.image.controller.pullSecret" + value = "${var.image_pull_secret}" + } + set { + name = " agones.ping.http.serviceType" + value = "${var.ping_service_type}" + } + set { + name = "agones.ping.udp.serviceType" + value = "${var.ping_service_type}" + } + version = "${var.agones_version}" + namespace = "agones-system" +} + +provider "null" { + version = "~> 2.1" +} + +# Creates folder with repositories so that helm provider would not fail +resource "null_resource" "helm_init" { + triggers = { + always_run = "${timestamp()}" + } + provisioner "local-exec" { + command = "helm init --client-only" + } +} diff --git a/build/includes/google-cloud.mk b/build/includes/google-cloud.mk index e740c7857b..19204e5af7 100644 --- a/build/includes/google-cloud.mk +++ b/build/includes/google-cloud.mk @@ -33,7 +33,6 @@ gcloud-test-cluster: $(ensure-build-image) --properties cluster.zone:$(GCP_CLUSTER_ZONE),cluster.name:$(GCP_CLUSTER_NAME),cluster.nodePool.initialNodeCount:$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT),cluster.nodePool.machineType:$(GCP_CLUSTER_NODEPOOL_MACHINETYPE),cluster.legacyAbac:$(GCP_CLUSTER_LEGACYABAC)\ --template=$(mount_path)/build/gke-test-cluster/cluster.yml.jinja $(MAKE) gcloud-auth-cluster - $(MAKE) setup-test-cluster clean-gcloud-test-cluster: $(ensure-build-image) docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \ @@ -48,23 +47,52 @@ terraform-clean: rm -r ./.terraform rm ./terraform.tfstate* - +# Creates a cluster and install release version of Agones controller +# Version could be specified by AGONES_VERSION gcloud-terraform-cluster: GCP_CLUSTER_LEGACYABAC ?= false gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 +gcloud-terraform-cluster: AGONES_VERSION ?= '' gcloud-terraform-cluster: $(ensure-build-image) gcloud-terraform-cluster: ifndef GCP_PROJECT $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) endif - $(DOCKER_RUN) bash -c 'export TF_VAR_password=$(GKE_PASSWORD) && \ - cd $(mount_path)/build && terraform apply -auto-approve \ + $(DOCKER_RUN) bash -c 'export TF_VAR_agones_version=$(AGONES_VERSION) && \ + export TF_VAR_password=$(GKE_PASSWORD) && \ + cd $(mount_path)/build && terraform apply -auto-approve -var values_file="" \ + -var chart="agones" \ -var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \ zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \ initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\", \ legacyABAC=\"$(GCP_CLUSTER_LEGACYABAC)\"}"' $(MAKE) gcloud-auth-cluster - $(MAKE) setup-test-cluster + +# Creates a cluster and install current version of Agones controller +# Set all necessary variables as `make install` does +gcloud-terraform-install: GCP_CLUSTER_LEGACYABAC ?= false +gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 +gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 +gcloud-terraform-install: ALWAYS_PULL_SIDECAR := true +gcloud-terraform-install: IMAGE_PULL_POLICY := "Always" +gcloud-terraform-install: PING_SERVICE_TYPE := "LoadBalancer" +gcloud-terraform-install: CRD_CLEANUP := true +gcloud-terraform-install: +ifndef GCP_PROJECT + $(eval GCP_PROJECT=$(shell sh -c "gcloud config get-value project 2> /dev/null")) +endif + $(DOCKER_RUN) bash -c 'export TF_VAR_password=$(GKE_PASSWORD) && \ + cd $(mount_path)/build && terraform apply -auto-approve -var agones_version="$(VERSION)" -var image_registry="$(REGISTRY)" \ + -var pull_policy="$(IMAGE_PULL_POLICY)" \ + -var always_pull_sidecar="$(ALWAYS_PULL_SIDECAR)" \ + -var image_pull_secret="$(IMAGE_PULL_SECRET)" \ + -var ping_service_type="$(PING_SERVICE_TYPE)" \ + -var crd_cleanup="$(CRD_CLEANUP)" \ + -var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \ + zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \ + initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\", \ + legacyABAC=\"$(GCP_CLUSTER_LEGACYABAC)\"}"' + $(MAKE) gcloud-auth-cluster gcloud-terraform-destroy-cluster: $(DOCKER_RUN) bash -c 'cd $(mount_path)/build && \ diff --git a/examples/terraform-submodule/module.tf b/examples/terraform-submodule/module.tf new file mode 100644 index 0000000000..dc72579975 --- /dev/null +++ b/examples/terraform-submodule/module.tf @@ -0,0 +1,47 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Run: +// terraform apply -var project="" [-var agones_version="0.9.0"] +variable "password" { + default= "" +} + +// Install latest version of agones +variable "agones_version" { + default="" +} +variable "project" {default = "agones"} + +variable "machine_type" {default = "n1-standard-4"} + +//Additional 2 node pools would be created with 1 node each for "agones-system" and "agones-metrics" +variable "node_count" {default = "4"} +module "agones" { + source = "git::https://github.com/GoogleCloudPlatform/agones.git//build/?ref=master" + + password = "${var.password}" + cluster = { + "zone" = "us-west1-c" + "name" = "test-cluster" + "machineType" = "${var.machine_type}" + "initialNodeCount" = "${var.node_count}" + "legacyAbac" = false + "project" = "${var.project}" + } + agones_version = "${var.agones_version}" + values_file="" + chart="agones" +} \ No newline at end of file diff --git a/site/content/en/docs/Installation/terraform.md b/site/content/en/docs/Installation/terraform.md new file mode 100644 index 0000000000..3cec872316 --- /dev/null +++ b/site/content/en/docs/Installation/terraform.md @@ -0,0 +1,85 @@ +--- +title: "Deploy GKE cluster and install Agones using Terraform" +linkTitle: "Install with Terraform" +weight: 4 +description: > + This chart install the Agones application and defines deployment on a [Kubernetes](http://kubernetes.io) cluster using the Terraform. + +--- + +## Prerequisites + +- Terraform v0.11.13 +- [Helm](https://docs.helm.sh/helm/) package manager 2.10.0+ +- Access to Google Cloud Kubernetes Engine +- `gcloud` utility installed +- Git + +## Installing the Agones as Terraform submodule + +You can use Terraform to provision your GKE cluster and install agones on it using Helm Terraform provider. + +GKE cluster would contain 3 Node Pools: +- Primary Node Pool with `"game-server"` tag, containing 4 nodes. +- `"agones-system"` node pool for Agones Controller. +- `"agones-metrics"` for monitoring and metrics collecting purpose. + +Additionally `"tiller"` service account would be created with ClusterRole. + +By default you will receive the latest version from [Helm repository](https://agones.dev/chart/stable), but you can configure version using `-var agones-version=`. + +## Example and parameters which is configurable + +The example of submodule configuration could be found here: + {{< ghlink href="examples/terraform-submodule/module.tf" >}}Terraform configuration with Agones submodule{{< /ghlink >}} + +Configurable parameters and their meaning: +- password - if not specified basic Auth would be disabled in GKE cluster +- agones_version - which version of agones to install +- project - your Google Cloud Project ID +- machine_type - primary cluster machine type ( default is "n1-standard-4") +- node_count - count of nodes in primary Node Pool. Defaults to "4". + +## Applying Agones terraform configuration + +First you should run: +``` +terraform init +``` + +It would use git to clone the current master of Agones, and use `./build` folder as starting point of Agones submodule, which contains all necessary Terraform configuration files. + +Next step you should make sure that you authenticate using gcloud: +``` +gcloud auth application-default login +``` + +Now you are able to deploy properly configured GKE cluster and specify release version of Agones you want to use: +``` +terraform apply -var project="" -var agones_version="0.9.0" +``` + +Run next command to setup your kubectl: +``` +gcloud container clusters get-credentials --zone us-west1-c test-cluster +``` + +You would see: +``` +Fetching cluster endpoint and auth data. +kubeconfig entry generated for test-cluster. +``` + +Check that your has access to kubernetes cluster: +``` +kubectl get nodes +``` + +Make sure you have 6 nodes in `Ready` state. + +## Uninstall the Agones and delete GKE cluster + +Run next command to delete all Terraform provisioned resources: +``` +terraform destroy +```