Skip to content

Commit

Permalink
Include terraform config to generate test cluster
Browse files Browse the repository at this point in the history
Creating GKE cluster with three node pools: system, metrics,
gameservers. Added tfvars parameters and new make targets.
E2E tests pass on a terraform deployed cluster.
  • Loading branch information
aLekSer committed Apr 12, 2019
1 parent e6d9190 commit fa9b949
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 0 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
bin
*.o
tmp
terraform.tfvars
terraform.tfstate*
build/local-includes/*
!build/local-includes/README.md
/release
Expand Down
2 changes: 2 additions & 0 deletions build/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ KUBECONFIG ?= ~/.kube/config
GCP_CLUSTER_NAME ?= test-cluster
GCP_CLUSTER_ZONE ?= us-west1-c
GCP_BUCKET_CHARTS ?= agones-chart
# Parameter which is used by terraform targets
GCP_PROJECT ?= agones
# the profile to use when developing on minikube
MINIKUBE_PROFILE ?= agones
GO_BUILD_TAGS ?= none
Expand Down
28 changes: 28 additions & 0 deletions build/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,13 @@ The Kubernetes config file used to access the cluster. Defaults to `~/.kube/conf
### CLUSTER_NAME
The (gcloud) test cluster that is being worked against. Defaults to `test-cluster`

### GCP_PROJECT
Your GCP project for deploying GKE cluster.

### GKE_PASSWORD
If specified basic authentication would be enabled for your cluster with username "admin".
Empty string `""` would disable basic authentication.

### IMAGE_PULL_SECRET
The name of the secret required to pull the Agones images, if needed.
If unset, no pull secret will be used.
Expand Down Expand Up @@ -542,6 +549,27 @@ Pulls down authentication information for kubectl against a cluster, name can be
Creates a short lived access to Google Cloud container repositories, so that you are able to call
`docker push` directly. Useful when used in combination with `make push` command.

### Terraform

Utilities for deploying a Kubernetes Engine cluster on Google Cloud Platform using `google` Terraform provider.

#### `make terraform-init`
Install `google` and `google-beta` terraform providers and authorize.

#### `make gcloud-terraform-cluster`
Run next command with your project ID specified:
```
GCP_PROJECT=<YOUR_PROJECT_ID> [GKE_PASSWORD="<YOUR_PASSWORD>"] make gcloud-terraform-cluster
```
Where `<YOUR_PASSWORD>` should be at least 16 characters in length. You can omit GKE_PASSWORD and then basic auth would be disabled. Also you change `ports="7000-8000"` setting using tfvars file.
Also you can define password `password=<YOUR_PASSWORD>` string in `build/terraform.tfvars`.

#### `make gcloud-terraform-destroy-cluster`
Run `terraform destroy` on your cluster.

#### `make terraform-clean`
Remove .terraform directory with configs as well as tfstate files.

### Minikube

A set of utilities for setting up and running a [Minikube](https://github.com/kubernetes/minikube) instance,
Expand Down
3 changes: 3 additions & 0 deletions build/build-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ RUN echo "export PATH=/usr/local/go/bin:/go/bin/:\$PATH" >> /root/.bashrc
# make nano the editor
RUN echo "export EDITOR=nano" >> /root/.bashrc

# install terraform
RUN wget -nv https://releases.hashicorp.com/terraform/0.11.13/terraform_0.11.13_linux_386.zip && unzip ./terraform_0.11.13_linux_386.zip && mv terraform /usr/local/bin/

# code generation scripts
COPY *.sh /root/
RUN chmod +x /root/*.sh
Expand Down
189 changes: 189 additions & 0 deletions build/cluster.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
# Copyright 2019 Google LLC All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

provider "google-beta" {
zone = "${lookup(var.cluster, "zone")}"
}


# Password for the Kubernetes API.
# Could be defined using GKE_PASSWORD env variable
# or by setting `password="somepass"` string in build/terraform.tfvars
variable "password" {default = ""}
variable "username" {default = "admin"}

# Ports can be overriden using tfvars file
variable "ports" {default="7000-8000"}

# Set of GKE cluster parameters which defines its name, zone
# and primary node pool configuration.
# It is crucial to set valid ProjectID for "project".
variable "cluster" {
description = "Set of GKE cluster parameters."
type = "map"
default = {
"zone" = "us-west1-c"
"name" = "test-cluster"
"machineType" = "n1-standard-4"
"initialNodeCount" = "4"
"legacyAbac" = false
"project" = "agones"
}
}


# echo command used for debugging purpose
# Run `terraform taint null_resource.test-setting-variables` before second execution
resource "null_resource" "test-setting-variables" {
provisioner "local-exec" {
command = "${"${format("echo Current variables set as following - name: %s, project: %s, machineType: %s, initialNodeCount: %s, zone: %s, legacyAbac: %s",
"${lookup(var.cluster, "name")}", "${lookup(var.cluster, "project")}",
"${lookup(var.cluster, "machineType")}", "${lookup(var.cluster, "initialNodeCount")}",
"${lookup(var.cluster, "zone")}", "${lookup(var.cluster, "legacyAbac")}")}"}"
}
}


locals {
username = "${var.password != "" ? var.username : ""}"
}

# assert that password has correct length
# before creating the cluster to avoid
# unfinished configurations
resource "null_resource" "check-password-length" {
count = "${length(var.password) >= 16 || length(var.password) == 0 ? 0 : 1}"
"Password must be more than 16 chars in length" = true
}

resource "google_container_cluster" "primary" {
name = "${lookup(var.cluster, "name")}"
location = "${lookup(var.cluster, "zone")}"
project = "${lookup(var.cluster, "project")}"
provider = "google-beta"

# Setting an empty username and password explicitly disables basic auth
master_auth {
username = "${local.username}"
password = "${var.password}"
}
enable_legacy_abac = "${lookup(var.cluster, "legacyAbac")}"
node_pool = [
{
node_count = "${lookup(var.cluster, "initialNodeCount")}"
node_config {
machine_type = "${lookup(var.cluster, "machineType")}"
oauth_scopes = [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
]

tags = ["game-server"]
timeouts {
create = "30m"
update = "40m"
}
}
},
{
name = "agones-system"
node_count = 1
node_config {
preemptible = true
machine_type = "n1-standard-4"

oauth_scopes = [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
]
labels = {
"stable.agones.dev/agones-system" = "true"
}
taint = {
key = "stable.agones.dev/agones-system"
value = "true"
effect = "NO_EXECUTE"
}
}
},
{
name = "agones-metrics"
node_count = 1

node_config {
preemptible = true
machine_type = "n1-standard-4"

oauth_scopes = [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
]
labels = {
"stable.agones.dev/agones-metrics" = "true"
}
taint = {
key = "stable.agones.dev/agones-metrics"
value = "true"
effect = "NO_EXECUTE"
}
}
}
]
}

resource "google_compute_firewall" "default" {
name = "game-server-firewall-firewall"
project = "${lookup(var.cluster, "project")}"
network = "${google_compute_network.default.name}"

allow {
protocol = "udp"
ports = ["${var.ports}"]
}

source_tags = ["game-server"]
}

resource "google_compute_network" "default" {
project = "${lookup(var.cluster, "project")}"
name = "agones-network"
}



# The following outputs allow authentication and connectivity to the GKE Cluster
# by using certificate-based authentication.
output "client_certificate" {
value = "${google_container_cluster.primary.master_auth.0.client_certificate}"
}

output "client_key" {
value = "${google_container_cluster.primary.master_auth.0.client_key}"
}

output "cluster_ca_certificate" {
value = "${google_container_cluster.primary.master_auth.0.cluster_ca_certificate}"
}
30 changes: 30 additions & 0 deletions build/includes/google-cloud.mk
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,36 @@ clean-gcloud-test-cluster: $(ensure-build-image)
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) gcloud \
deployment-manager deployments delete $(GCP_CLUSTER_NAME)

### Deploy cluster with Terraform
terraform-init:
docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) bash -c '\
cd $(mount_path)/build && terraform init && gcloud auth application-default login'

terraform-clean:
rm -r ./.terraform
rm ./terraform.tfstate*


gcloud-terraform-cluster: GCP_PROJECT ?= ""
gcloud-terraform-cluster: GCP_CLUSTER_LEGACYABAC ?= false
gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= n1-standard-4
gcloud-terraform-cluster: $(ensure-build-image)
gcloud-terraform-cluster:
test -n "$(GCP_PROJECT)" # $$GCP_PROJECT is undefined
$(DOCKER_RUN) bash -c 'export TF_VAR_password=$(GKE_PASSWORD) && \
cd $(mount_path)/build && terraform apply -auto-approve \
-var "cluster={name=\"$(GCP_CLUSTER_NAME)\", machineType=\"$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)\", \
zone=\"$(GCP_CLUSTER_ZONE)\", project=\"$(GCP_PROJECT)\", \
initialNodeCount=\"$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)\", \
legacyABAC=\"$(GCP_CLUSTER_LEGACYABAC)\"}"'
$(MAKE) gcloud-auth-cluster
$(MAKE) setup-test-cluster

gcloud-terraform-destroy-cluster:
$(DOCKER_RUN) bash -c 'cd $(mount_path)/build && \
terraform destroy -auto-approve'

# Creates a gcloud cluster for end-to-end
# it installs also a consul cluster to handle build system concurrency using a distributed lock
gcloud-e2e-test-cluster: $(ensure-build-image)
Expand Down

0 comments on commit fa9b949

Please sign in to comment.