From 40158015decb0c0ddabad246b2a8fb600c682b3e Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Mon, 11 Oct 2021 15:37:51 -0700 Subject: [PATCH] Flaky TestGameServerUnhealthyAfterReadyCrash (#2302) I noticed in a few flaky end-to-end tests, I kept seeing this in the logs: ``` time="2021-10-08 19:02:01.419" level=info msg="sent UDP packet" address="35.247.94.25:7682" test=TestGameServerUnhealthyAfterReadyCrash ``` Over and over again, and also noting that it was happening _after_ the e2e test had completed. See: https://console.cloud.google.com/cloud-build/builds/9ca5715a-443c-4693-bbd5-2879e61f2aaa;step=21?project=agones-images https://console.cloud.google.com/cloud-build/builds/84cb8db2-1a11-4db3-a9e1-d9d51b9baf14;step=21?project=agones-images My theory: The go routine had nothing in it that forced it to stop once the test was complete - so depending on order of tests, it might keep continuing for a while, while other tests ran. If it did that, and a GameServer spun up on the same node and port as the originally crashed GameServer, it would crash it - likely breaking whatever test it ran into! Work on #2296 Co-authored-by: Robert Bailey --- test/e2e/gameserver_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/gameserver_test.go b/test/e2e/gameserver_test.go index 63ad565be1..6f449f5c89 100644 --- a/test/e2e/gameserver_test.go +++ b/test/e2e/gameserver_test.go @@ -22,6 +22,7 @@ import ( "net" "os/exec" "strings" + "sync/atomic" "testing" "time" @@ -385,8 +386,16 @@ func TestGameServerUnhealthyAfterReadyCrash(t *testing.T) { // keep crashing, until we move to Unhealthy. Solves potential issues with controller Informer cache // race conditions in which it has yet to see a GameServer is Ready before the crash. + var stop int32 = 0 + defer func() { + atomic.StoreInt32(&stop, 1) + }() go func() { for { + if atomic.LoadInt32(&stop) > 0 { + l.Info("UDP Crash stop signal received. Stopping.") + return + } conn, err := net.Dial("udp", address) assert.NoError(t, err) defer conn.Close() // nolint: errcheck