From 718f9d02a9aae732c6dcb89ec1f4cb1744a7c3a8 Mon Sep 17 00:00:00 2001 From: Alexey Date: Tue, 24 Mar 2020 02:31:23 +0300 Subject: [PATCH] Applied allocation test (#1417) * Applied allocation test Co-authored-by: Mark Mandel --- build/Makefile | 2 +- examples/crd-client/create-gs.yaml | 2 +- examples/crd-client/main.go | 2 +- examples/fleet.yaml | 2 +- examples/gameserver.yaml | 2 +- examples/simple-udp/Makefile | 2 +- examples/simple-udp/dev-gameserver.yaml | 2 +- examples/simple-udp/fleet-distributed.yaml | 2 +- examples/simple-udp/fleet.yaml | 2 +- .../simple-udp/gameserver-passthrough.yaml | 2 +- examples/simple-udp/gameserver.yaml | 2 +- examples/simple-udp/gameserverset.yaml | 2 +- examples/simple-udp/main.go | 28 ++++++ pkg/util/webhooks/webhooks_test.go | 2 +- site/config.toml | 2 +- .../en/docs/Guides/local-game-server.md | 2 +- .../content/en/docs/Guides/troubleshooting.md | 8 +- site/content/en/docs/Reference/fleet.md | 2 +- site/content/en/docs/Reference/gameserver.md | 2 +- test/e2e/framework/framework.go | 23 ++++- test/load/allocation/README.md | 8 ++ test/load/allocation/allocationload.go | 96 +++++++++++++++++++ test/load/allocation/fleet.yaml | 39 ++++++++ test/load/allocation/runAllocation.sh | 35 +++++++ 24 files changed, 248 insertions(+), 23 deletions(-) create mode 100644 test/load/allocation/README.md create mode 100644 test/load/allocation/allocationload.go create mode 100644 test/load/allocation/fleet.yaml create mode 100755 test/load/allocation/runAllocation.sh diff --git a/build/Makefile b/build/Makefile index 0acf58e166..6f156371d3 100644 --- a/build/Makefile +++ b/build/Makefile @@ -57,7 +57,7 @@ KIND_PROFILE ?= agones KIND_CONTAINER_NAME=$(KIND_PROFILE)-control-plane # Game Server image to use while doing end-to-end tests -GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.18 +GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.19 # Directory that this Makefile is in. mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) diff --git a/examples/crd-client/create-gs.yaml b/examples/crd-client/create-gs.yaml index d1acfd6731..3633a7cd4a 100644 --- a/examples/crd-client/create-gs.yaml +++ b/examples/crd-client/create-gs.yaml @@ -28,5 +28,5 @@ spec: imagePullPolicy: Always env: - name: GAMESERVER_IMAGE - value: "gcr.io/agones-images/udp-server:0.18" + value: "gcr.io/agones-images/udp-server:0.19" restartPolicy: Never diff --git a/examples/crd-client/main.go b/examples/crd-client/main.go index 83410091dd..2a73d97090 100644 --- a/examples/crd-client/main.go +++ b/examples/crd-client/main.go @@ -32,7 +32,7 @@ import ( const ( gameServerImage = "GAMESERVER_IMAGE" - defaultImage = "gcr.io/agones-images/udp-server:0.18" + defaultImage = "gcr.io/agones-images/udp-server:0.19" ) func main() { diff --git a/examples/fleet.yaml b/examples/fleet.yaml index d3d8627a58..f682d8d874 100644 --- a/examples/fleet.yaml +++ b/examples/fleet.yaml @@ -74,4 +74,4 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 \ No newline at end of file + image: gcr.io/agones-images/udp-server:0.19 \ No newline at end of file diff --git a/examples/gameserver.yaml b/examples/gameserver.yaml index 31fb709887..d970c30c90 100644 --- a/examples/gameserver.yaml +++ b/examples/gameserver.yaml @@ -87,5 +87,5 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 imagePullPolicy: Always diff --git a/examples/simple-udp/Makefile b/examples/simple-udp/Makefile index 1c07533802..6de84e7dac 100644 --- a/examples/simple-udp/Makefile +++ b/examples/simple-udp/Makefile @@ -27,7 +27,7 @@ REPOSITORY = gcr.io/agones-scale-test-1 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) project_path := $(dir $(mkfile_path)) -server_tag = $(REPOSITORY)/udp-server:0.18 +server_tag = $(REPOSITORY)/udp-server:0.19 root_path = $(realpath $(project_path)/../..) # _____ _ diff --git a/examples/simple-udp/dev-gameserver.yaml b/examples/simple-udp/dev-gameserver.yaml index 323bd5d7bb..bb3c01f482 100644 --- a/examples/simple-udp/dev-gameserver.yaml +++ b/examples/simple-udp/dev-gameserver.yaml @@ -31,4 +31,4 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 diff --git a/examples/simple-udp/fleet-distributed.yaml b/examples/simple-udp/fleet-distributed.yaml index 42939ee964..6a73f6029a 100644 --- a/examples/simple-udp/fleet-distributed.yaml +++ b/examples/simple-udp/fleet-distributed.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 resources: requests: memory: "32Mi" diff --git a/examples/simple-udp/fleet.yaml b/examples/simple-udp/fleet.yaml index cc8d5f1e28..c5489c4120 100644 --- a/examples/simple-udp/fleet.yaml +++ b/examples/simple-udp/fleet.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 resources: requests: memory: "64Mi" diff --git a/examples/simple-udp/gameserver-passthrough.yaml b/examples/simple-udp/gameserver-passthrough.yaml index 57c98d8352..3377b12c60 100644 --- a/examples/simple-udp/gameserver-passthrough.yaml +++ b/examples/simple-udp/gameserver-passthrough.yaml @@ -24,7 +24,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 env: - name: "PASSTHROUGH" value: "TRUE" diff --git a/examples/simple-udp/gameserver.yaml b/examples/simple-udp/gameserver.yaml index 06f14cb2d7..a947f3364a 100644 --- a/examples/simple-udp/gameserver.yaml +++ b/examples/simple-udp/gameserver.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 resources: requests: memory: "32Mi" diff --git a/examples/simple-udp/gameserverset.yaml b/examples/simple-udp/gameserverset.yaml index e588a9e2d1..ed9cc3077c 100644 --- a/examples/simple-udp/gameserverset.yaml +++ b/examples/simple-udp/gameserverset.yaml @@ -31,4 +31,4 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 \ No newline at end of file + image: gcr.io/agones-images/udp-server:0.19 \ No newline at end of file diff --git a/examples/simple-udp/main.go b/examples/simple-udp/main.go index 4a608ebb07..ec44abfcba 100644 --- a/examples/simple-udp/main.go +++ b/examples/simple-udp/main.go @@ -45,7 +45,11 @@ func main() { ======= passthrough := flag.Bool("passthrough", false, "Get listening port from the SDK, rather than use the 'port' value") readyOnStart := flag.Bool("ready", true, "Mark this GameServer as Ready on startup") +<<<<<<< HEAD >>>>>>> 35493671e40293e27aa5cab9263e4772ce175e9f +======= + shutdownDelay := flag.Int("automaticShutdownDelayMin", 0, "If greater than zero, automatically shut down the server this many minutes after the server becomes allocated") +>>>>>>> 6176ee0d (Applied allocation test (#1417)) flag.Parse() if ep := os.Getenv("PORT"); ep != "" { port = &ep @@ -92,7 +96,13 @@ func main() { ready(s) } +<<<<<<< HEAD watchGameServerEvents(s) +======= + if *shutdownDelay > 0 { + shutdownAfterAllocation(s, *shutdownDelay) + } +>>>>>>> 6176ee0d (Applied allocation test (#1417)) readWriteLoop(conn, stop, s) } @@ -104,6 +114,24 @@ func doSignal() { os.Exit(0) } +// shutdownAfterAllocation creates a callback to automatically shut down +// the server a specified number of minutes after the server becomes +// allocated. +func shutdownAfterAllocation(s *sdk.SDK, shutdownDelay int) { + err := s.WatchGameServer(func(gs *coresdk.GameServer) { + if gs.Status.State == "Allocated" { + time.Sleep(time.Duration(shutdownDelay) * time.Minute) + shutdownErr := s.Shutdown() + if shutdownErr != nil { + log.Fatalf("Could not shutdown: %v", shutdownErr) + } + } + }) + if err != nil { + log.Fatalf("Could not watch Game Server events, %v", err) + } +} + func readWriteLoop(conn net.PacketConn, stop chan struct{}, s *sdk.SDK) { b := make([]byte, 1024) for { diff --git a/pkg/util/webhooks/webhooks_test.go b/pkg/util/webhooks/webhooks_test.go index a8165e17e3..e397dcc3fb 100644 --- a/pkg/util/webhooks/webhooks_test.go +++ b/pkg/util/webhooks/webhooks_test.go @@ -163,7 +163,7 @@ func TestWebHookFleetValidationHandler(t *testing.T) { "template": { "spec": { "containers": [{ - "image": "gcr.io/agones-images/udp-server:0.18", + "image": "gcr.io/agones-images/udp-server:0.19", "name": false }] } diff --git a/site/config.toml b/site/config.toml index 2407c9d4ae..ddf06ad7e2 100644 --- a/site/config.toml +++ b/site/config.toml @@ -91,7 +91,7 @@ release_branch = "release-1.4.0" release_version = "1.4.0" # example tag -example_image_tag = "gcr.io/agones-images/udp-server:0.18" +example_image_tag = "gcr.io/agones-images/udp-server:0.19" # User interface configuration [params.ui] diff --git a/site/content/en/docs/Guides/local-game-server.md b/site/content/en/docs/Guides/local-game-server.md index 5f2dd73bf5..5c5362261f 100644 --- a/site/content/en/docs/Guides/local-game-server.md +++ b/site/content/en/docs/Guides/local-game-server.md @@ -32,7 +32,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 ``` Once you save this to a file make sure you have `kubectl` configured to point to your Agones cluster and then run `kubectl apply -f dev-gameserver.yaml`. This will register your server with Agones. diff --git a/site/content/en/docs/Guides/troubleshooting.md b/site/content/en/docs/Guides/troubleshooting.md index 86dcc4b5ad..e22609dab0 100644 --- a/site/content/en/docs/Guides/troubleshooting.md +++ b/site/content/en/docs/Guides/troubleshooting.md @@ -51,7 +51,7 @@ Spec: Creation Timestamp: Spec: Containers: - Image: gcr.io/agones-images/udp-server:0.18 + Image: gcr.io/agones-images/udp-server:0.19 Name: simple-udp Resources: Limits: @@ -82,7 +82,7 @@ Events: The backing Pod has the same name as the `GameServer` - so it's also worth looking at the details and events for the Pod to see if there are any issues there, such as restarts due to binary crashes etc. -For example, you can see the restart count on the gcr.io/agones-images/udp-server:0.18 container +For example, you can see the restart count on the gcr.io/agones-images/udp-server:0.19 container is set to `1`, due to the game server binary crash ``` @@ -104,7 +104,7 @@ Controlled By: GameServer/simple-udp-zqppv Containers: simple-udp: Container ID: docker://69eacd03cc89b0636b78abe47926b02183ba84d18fa20649ca443f5232511661 - Image: gcr.io/agones-images/udp-server:0.18 + Image: gcr.io/agones-images/udp-server:0.19 Image ID: docker-pullable://gcr.io/agones-images/udp-server@sha256:6a60eff5e68b88b5ce75ae98082d79cff36cda411a090f3495760e5c3b6c3575 Port: 7654/UDP Host Port: 7058/UDP @@ -173,7 +173,7 @@ Events: Normal Created 2m28s kubelet, gke-test-cluster-default-590db5e4-4s6r Created container Normal Created 114s (x2 over 2m31s) kubelet, gke-test-cluster-default-590db5e4-4s6r Created container Normal Started 114s (x2 over 2m31s) kubelet, gke-test-cluster-default-590db5e4-4s6r Started container - Normal Pulled 114s (x2 over 2m31s) kubelet, gke-test-cluster-default-590db5e4-4s6r Container image "gcr.io/agones-images/udp-server:0.18" already present on machine + Normal Pulled 114s (x2 over 2m31s) kubelet, gke-test-cluster-default-590db5e4-4s6r Container image "gcr.io/agones-images/udp-server:0.19" already present on machine ``` Finally, you can also get the logs of your `GameServer` `Pod` as well via `kubectl logs -c `, for example: diff --git a/site/content/en/docs/Reference/fleet.md b/site/content/en/docs/Reference/fleet.md index 7607a21c5a..7b97ec6633 100644 --- a/site/content/en/docs/Reference/fleet.md +++ b/site/content/en/docs/Reference/fleet.md @@ -67,7 +67,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 ``` Since Agones defines a new diff --git a/site/content/en/docs/Reference/gameserver.md b/site/content/en/docs/Reference/gameserver.md index 854f018478..e6d5c22bef 100644 --- a/site/content/en/docs/Reference/gameserver.md +++ b/site/content/en/docs/Reference/gameserver.md @@ -77,7 +77,7 @@ spec: spec: containers: - name: simple-udp - image: gcr.io/agones-images/udp-server:0.18 + image: gcr.io/agones-images/udp-server:0.19 imagePullPolicy: Always ``` {{% /feature %}} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 771cdc6970..c8be5746b0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -66,13 +66,32 @@ type Framework struct { // New setups a testing framework using a kubeconfig path and the game server image to use for testing. func New(kubeconfig string) (*Framework, error) { + return newFramework(kubeconfig, 0, 0) +} + +// NewWithRates setups a testing framework using a kubeconfig path and the game server image +// to use for load testing with QPS and Burst overwrites. +func NewWithRates(kubeconfig string, qps float32, burst int) (*Framework, error) { + return newFramework(kubeconfig, qps, burst) +} + +func newFramework(kubeconfig string, qps float32, burst int) (*Framework, error) { config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { return nil, errors.Wrap(err, "build config from flags failed") } +<<<<<<< HEAD config.QPS = 1200 config.Burst = 1200 +======= + if qps > 0 { + config.QPS = qps + } + if burst > 0 { + config.Burst = burst + } +>>>>>>> 6176ee0d (Applied allocation test (#1417)) kubeClient, err := kubernetes.NewForConfig(config) if err != nil { @@ -95,8 +114,8 @@ func NewFromFlags() (*Framework, error) { usr, _ := user.Current() kubeconfig := flag.String("kubeconfig", filepath.Join(usr.HomeDir, "/.kube/config"), "kube config path, e.g. $HOME/.kube/config") - gsimage := flag.String("gameserver-image", "gcr.io/agones-images/udp-server:0.18", - "gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.18") + gsimage := flag.String("gameserver-image", "gcr.io/agones-images/udp-server:0.19", + "gameserver image to use for those tests, gcr.io/agones-images/udp-server:0.19") pullSecret := flag.String("pullsecret", "", "optional secret to be used for pulling the gameserver and/or Agones SDK sidecar images") stressTestLevel := flag.Int("stress", 0, "enable stress test at given level 0-100") diff --git a/test/load/allocation/README.md b/test/load/allocation/README.md new file mode 100644 index 0000000000..3d80e9f75d --- /dev/null +++ b/test/load/allocation/README.md @@ -0,0 +1,8 @@ +This is a load test to determine Allocation QPS over time against a set of GameServers that are constantly being shutdown after a period. + +This test creates a configured amount of GameServers at the initial step, switches them to Allocated state and finally shuts them down (`automaticShutdownDelayMin` flag in a simple-udp). + +1) Run kubectl apply -f ./fleet.yaml +2) Run `runAllocation.sh` script to perform this test. You can provide a number of runs as a parameter (3 is a default value). There is a 500 seconds pause after each run. + +To run this test under normal conditions the number of replicas in the yaml file should be >= numberOfClients * reqPerClient diff --git a/test/load/allocation/allocationload.go b/test/load/allocation/allocationload.go new file mode 100644 index 0000000000..b974d5a2ae --- /dev/null +++ b/test/load/allocation/allocationload.go @@ -0,0 +1,96 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os/user" + "path/filepath" + "sync" + + agonesv1 "agones.dev/agones/pkg/apis/agones/v1" + allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" + e2eframework "agones.dev/agones/test/e2e/framework" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const defaultNs = "default" +const reqPerClient = 10 + +func main() { + usr, err := user.Current() + if err != nil { + logrus.Fatalf("Unable to determine the current user: %v", err) + } + kubeconfig := flag.String("kubeconfig", filepath.Join(usr.HomeDir, "/.kube/config"), + "kube config path, e.g. $HOME/.kube/config") + fleetName := flag.String("fleet_name", "simple-udp", "The fleet name that the tests will run against") + qps := flag.Int("qps", 1000, "The QPS value that will overwrite the default value") + burst := flag.Int("burst", 1000, "The Burst value that will overwrite the default value") + clientCnt := flag.Int("clients", 10, "The number of concurrent clients") + + flag.Parse() + + logrus.SetFormatter(&logrus.TextFormatter{ + EnvironmentOverrideColors: true, + FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05.000", + }) + + framework, err := e2eframework.NewWithRates(*kubeconfig, float32(*qps), *burst) + if err != nil { + logrus.Fatalf("Failed to setup framework: %v", err) + } + + logrus.Info("Starting Allocation") + allocate(framework, *clientCnt, *fleetName) + logrus.Info("Finished Allocation.") + logrus.Info("=======================================================================") + logrus.Info("=======================================================================") + logrus.Info("=======================================================================") +} + +func allocate(framework *e2eframework.Framework, numOfClients int, fleetName string) { + gsa := &allocationv1.GameServerAllocation{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "allocation-"}, + Spec: allocationv1.GameServerAllocationSpec{ + Required: metav1.LabelSelector{MatchLabels: map[string]string{agonesv1.FleetNameLabel: fleetName}}, + Preferred: []metav1.LabelSelector{ + {MatchLabels: map[string]string{agonesv1.FleetNameLabel: fleetName}}, + }, + }, + } + var wg sync.WaitGroup + wg.Add(numOfClients) + + // Allocate GS by numOfClients in parallel + for i := 0; i < numOfClients; i++ { + go func() { + defer wg.Done() + for j := 0; j < reqPerClient; j++ { + gsa1, err := framework.AgonesClient.AllocationV1().GameServerAllocations(defaultNs).Create(gsa.DeepCopy()) + if err != nil { + logrus.Errorf("could not completed gsa1 allocation : %v", err) + } else if gsa1.Status.State == "Contention" { + logrus.Errorf("could not allocate : %v", gsa1.Status.State) + } + logrus.Infof("%+v", gsa1) + } + }() + } + + wg.Wait() +} diff --git a/test/load/allocation/fleet.yaml b/test/load/allocation/fleet.yaml new file mode 100644 index 0000000000..7cc52dcef2 --- /dev/null +++ b/test/load/allocation/fleet.yaml @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "agones.dev/v1" +kind: Fleet +metadata: + name: simple-udp +spec: + replicas: 100 + template: + spec: + ports: + - name: default + containerPort: 7654 + template: + spec: + containers: + - name: simple-udp + image: gcr.io/agones-alexander/udp-server:0.19 + resources: + requests: + memory: "64Mi" + cpu: "20m" + limits: + memory: "64Mi" + cpu: "20m" + args: + - "-automaticShutdownDelayMin=1" diff --git a/test/load/allocation/runAllocation.sh b/test/load/allocation/runAllocation.sh new file mode 100755 index 0000000000..64d0502368 --- /dev/null +++ b/test/load/allocation/runAllocation.sh @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +# Provide the number of times you want allocation test to be run +testRunsCount=3 +if [ -z "$1" ] + then + echo "No test run count provided, using default which is 3" + else + testRunsCount=$1 + if ! [[ $testRunsCount =~ ^[0-9]+$ ]] ; then + echo "error: Not a positive number provided" >&2; exit 1 + fi +fi + +counter=1 +while [ $counter -le $testRunsCount ] +do + echo "Run number: " $counter + go run allocationload.go 2>>./allocation_test_results.txt + sleep 500 + ((counter++)) +done