From 27d1857fee7d809a78cd945edfdcd2fa9a69b839 Mon Sep 17 00:00:00 2001 From: Alexander Apalikov Date: Thu, 21 Mar 2019 17:07:22 +0300 Subject: [PATCH] Include terraform config to generate test cluster Creating GKE cluster with three node pools: system, metrics, gameservers. Added tfvars parameters and new make targets. E2E tests pass on a terraform deployed cluster. --- .gitignore | 2 + build/README.md | 23 +++++ build/build-image/Dockerfile | 3 + build/cluster.tf | 182 +++++++++++++++++++++++++++++++++ build/includes/google-cloud.mk | 29 ++++++ 5 files changed, 239 insertions(+) create mode 100644 build/cluster.tf diff --git a/.gitignore b/.gitignore index e47b4e9855..f3e894463e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ bin *.o tmp +terraform.tfvars +terraform.tfstate* build/local-includes/* !build/local-includes/README.md /release diff --git a/build/README.md b/build/README.md index d74eacbb8e..f4467b6ad8 100644 --- a/build/README.md +++ b/build/README.md @@ -379,6 +379,9 @@ The Kubernetes config file used to access the cluster. Defaults to `~/.kube/conf ### CLUSTER_NAME The (gcloud) test cluster that is being worked against. Defaults to `test-cluster` +### GCP_PROJECT +Your GCP project for deploying GKE cluster. + ### IMAGE_PULL_SECRET The name of the secret required to pull the Agones images, if needed. If unset, no pull secret will be used. @@ -542,6 +545,26 @@ Pulls down authentication information for kubectl against a cluster, name can be Creates a short lived access to Google Cloud container repositories, so that you are able to call `docker push` directly. Useful when used in combination with `make push` command. +### Terraform + +Utilities for deploying a Kubernetes Engine cluster on Google Cloud Platform using `google` Terraform provider. + +#### `make terraform-init` +Install `google` and `google-beta` terraform providers and authorize. + +#### `make gcloud-terraform-cluster` +Run next command with your project ID specified: +``` +GCP_PROJECT= GKE_PASSWORD="" make gcloud-terraform-cluster +``` +Where `` should be 16 characters in length. You can omit GKE_PASSWORD and define `password=` string in `build/terraform.tfvars`. Also you change `ports="7000-8000"` setting using tfvars file. + +#### `make gcloud-terraform-destroy-cluster` +Run `terraform destroy` on your cluster. + +#### `make terraform-clean` +Remove .terraform directory with configs as well as tfstate files. + ### Minikube A set of utilities for setting up and running a [Minikube](https://github.com/kubernetes/minikube) instance, diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile index 6e180218ec..9e0c9c29ac 100644 --- a/build/build-image/Dockerfile +++ b/build/build-image/Dockerfile @@ -140,6 +140,9 @@ RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc # make nano the editor RUN echo "export EDITOR=nano" >> /root/.bashrc +# install terraform +RUN wget -nv https://releases.hashicorp.com/terraform/0.11.13/terraform_0.11.13_linux_386.zip && unzip ./terraform_0.11.13_linux_386.zip && mv terraform /usr/local/bin/ + # code generation scripts COPY *.sh /root/ RUN chmod +x /root/*.sh diff --git a/build/cluster.tf b/build/cluster.tf new file mode 100644 index 0000000000..6283f0abbe --- /dev/null +++ b/build/cluster.tf @@ -0,0 +1,182 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "google-beta" { + zone = "${lookup(var.cluster, "zone")}" +} + + +# Password for the Kubernetes API. +# Could be defined using GKE_PASSWORD env variable +# or by setting `password="somepass"` string in build/terraform.tfvars +variable "password" {default = ""} + +# Ports can be overriden using tfvars file +variable "ports" {default="7000-8000"} + +# Set of GKE cluster parameters which defines its name, zone +# and primary node pool configuration. +# It is crucial to set valid ProjectID for "project". +variable "cluster" { + description = "Set of GKE cluster parameters." + type = "map" + default = { + "zone" = "us-west1-c" + "name" = "test-cluster" + "machineType" = "n1-standard-4" + "initialNodeCount" = "4" + "legacyAbac" = false + "project" = "agones" + } +} + + +# echo command used for debugging purpose +# Run `terraform taint null_resource.test-setting-variables` before second execution +resource "null_resource" "test-setting-variables" { + provisioner "local-exec" { + command = "${"${format("echo Current variables set as following - name: %s, project: %s, machineType: %s, initialNodeCount: %s, zone: %s, legacyAbac: %s", + "${lookup(var.cluster, "name")}", "${lookup(var.cluster, "project")}", + "${lookup(var.cluster, "machineType")}", "${lookup(var.cluster, "initialNodeCount")}", + "${lookup(var.cluster, "zone")}", "${lookup(var.cluster, "legacyAbac")}")}"}" + } +} + +# assert that password has correct length +# before creating the cluster to avoid +# unfinished configurations +resource "null_resource" "check-password-length" { + count = "${length(var.password) >= 16 ? 0 : 1}" + "Password must be more than 16 chars in length" = true +} + +resource "google_container_cluster" "primary" { + name = "${lookup(var.cluster, "name")}" + location = "${lookup(var.cluster, "zone")}" + project = "${lookup(var.cluster, "project")}" + provider = "google-beta" + + master_auth { + username = "admin" + password = "${var.password}" + } + enable_legacy_abac = "${lookup(var.cluster, "legacyAbac")}" + node_pool = [ + { + node_count = "${lookup(var.cluster, "initialNodeCount")}" + node_config { + machine_type = "${lookup(var.cluster, "machineType")}" + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + + tags = ["game-server"] + timeouts { + create = "30m" + update = "40m" + } + } + }, + { + name = "agones-system" + node_count = 1 + node_config { + preemptible = true + machine_type = "n1-standard-4" + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + labels = { + "stable.agones.dev/agones-system" = "true" + } + taint = { + key = "stable.agones.dev/agones-system" + value = "true" + effect = "NO_EXECUTE" + } + } + }, + { + name = "agones-metrics" + node_count = 1 + + node_config { + preemptible = true + machine_type = "n1-standard-4" + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + labels = { + "stable.agones.dev/agones-metrics" = "true" + } + taint = { + key = "stable.agones.dev/agones-metrics" + value = "true" + effect = "NO_EXECUTE" + } + } + } + ] +} + +resource "google_compute_firewall" "default" { + name = "game-server-firewall-firewall" + project = "${lookup(var.cluster, "project")}" + network = "${google_compute_network.default.name}" + + allow { + protocol = "udp" + ports = ["${var.ports}"] + } + + source_tags = ["game-server"] +} + +resource "google_compute_network" "default" { + project = "${lookup(var.cluster, "project")}" + name = "agones-network" +} + + + +# The following outputs allow authentication and connectivity to the GKE Cluster +# by using certificate-based authentication. +output "client_certificate" { + value = "${google_container_cluster.primary.master_auth.0.client_certificate}" +} + +output "client_key" { + value = "${google_container_cluster.primary.master_auth.0.client_key}" +} + +output "cluster_ca_certificate" { + value = "${google_container_cluster.primary.master_auth.0.cluster_ca_certificate}" +} diff --git a/build/includes/google-cloud.mk b/build/includes/google-cloud.mk index 3e249720f9..c6b9888530 100644 --- a/build/includes/google-cloud.mk +++ b/build/includes/google-cloud.mk @@ -35,6 +35,35 @@ gcloud-test-cluster: $(ensure-build-image) $(MAKE) gcloud-auth-cluster $(MAKE) setup-test-cluster +terraform-init: + docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c '\ + cd $(mount_path)/build && terraform init && gcloud auth application-default login' + +terraform-clean: + rm -r ./.terraform + rm ./terraform.tfstate* + + +gcloud-terraform-cluster: GCP_PROJECT ?= "" +gcloud-terraform-cluster: GCP_CLUSTER_LEGACYABAC ?= false +gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4 +gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4 +gcloud-terraform-cluster: $(ensure-build-image) +gcloud-terraform-cluster: + test -n "$(GCP_PROJECT)" # $$GCP_PROJECT is undefined + $(DOCKER_RUN) bash -c 'export TF_VAR_password=$(GKE_PASSWORD) && \ + cd $(mount_path)/build && terraform apply -auto-approve \ + -var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \ + zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \ + initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\", \ + legacyABAC=\"$(GCP_CLUSTER_LEGACYABAC)\"}"' + $(MAKE) gcloud-auth-cluster + $(MAKE) setup-test-cluster + +gcloud-terraform-destroy-cluster: + $(DOCKER_RUN) bash -c 'export TF_VAR_password="1234567890123456" && \ + cd $(mount_path)/build && terraform destroy -auto-approve' + clean-gcloud-test-cluster: $(ensure-build-image) docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \ deployment-manager deployments delete $(GCP_CLUSTER_NAME)