Merge pull request #164 from markmandel/refactor/health-tests

Refactor of workerqueue health testing

pkg/gameservers/controller.go

@@ -109,7 +109,7 @@ func NewController(
 	c.recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "gameserver-controller"})
 
 	c.workerqueue = workerqueue.NewWorkerQueue(c.syncGameServer, c.logger, stable.GroupName+".GameServerController")
-	health.AddLivenessCheck("game-server-worker-queue", healthcheck.Check(c.workerqueue.Healthy))
+	health.AddLivenessCheck("gameserver-workerqueue", healthcheck.Check(c.workerqueue.Healthy))
 
 	wh.AddHandler("/mutate", stablev1alpha1.Kind("GameServer"), admv1beta1.Create, c.creationMutationHandler)
 	wh.AddHandler("/validate", stablev1alpha1.Kind("GameServer"), admv1beta1.Create, c.creationValidationHandler)

pkg/gameservers/controller_test.go

@@ -17,7 +17,6 @@ package gameservers
 import (
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"testing"
 	"time"
 
@@ -185,35 +184,6 @@ func TestControllerWatchGameServers(t *testing.T) {
 	assert.Equal(t, "default/test", <-received)
 }
 
-func TestControllerHealthCheck(t *testing.T) {
-	m := newMocks()
-	m.extClient.AddReactor("get", "customresourcedefinitions", func(action k8stesting.Action) (bool, runtime.Object, error) {
-		return true, newEstablishedCRD(), nil
-	})
-	health := healthcheck.NewHandler()
-	c := NewController(webhooks.NewWebHook("", ""), health, 10, 20, "sidecar:dev", false,
-		m.kubeClient, m.kubeInformationFactory, m.extClient, m.agonesClient, m.agonesInformerFactory)
-
-	c.workerqueue.SyncHandler = func(name string) error {
-		return nil
-	}
-
-	stop, cancel := startInformers(m, c.gameServerSynced)
-
-	go http.ListenAndServe("localhost:9090", health)
-
-	go func() {
-		err := c.Run(1, stop)
-		assert.Nil(t, err, "Run should not error")
-	}()
-
-	testHTTPHealth(t, "http://localhost:9090/live", "{}\n", http.StatusOK)
-
-	cancel()
-
-	testHTTPHealth(t, "http://localhost:9090/live", "{}\n", http.StatusServiceUnavailable)
-}
-
 func TestControllerCreationMutationHandler(t *testing.T) {
 	t.Parallel()
 

pkg/gameservers/helper_test.go

@@ -16,12 +16,11 @@ package gameservers
 
 import (
 	"context"
-	"time"
-
 	"io"
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"agones.dev/agones/pkg/apis/stable/v1alpha1"
 	agonesfake "agones.dev/agones/pkg/client/clientset/versioned/fake"

pkg/util/workerqueue/workerqueue_test.go

@@ -18,12 +18,20 @@ import (
 	"testing"
 	"time"
 
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/heptiolabs/healthcheck"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
 )
 
 func TestWorkerQueueRun(t *testing.T) {
+	t.Parallel()
+
 	received := make(chan string)
 	defer close(received)
 
@@ -56,6 +64,8 @@ func TestWorkerQueueRun(t *testing.T) {
 }
 
 func TestWorkerQueueHealthy(t *testing.T) {
+	t.Parallel()
+
 	done := make(chan struct{})
 	handler := func(string) error {
 		<-done
@@ -81,3 +91,48 @@ func TestWorkerQueueHealthy(t *testing.T) {
 	assert.Equal(t, 0, wq.RunCount())
 	assert.EqualError(t, wq.Healthy(), "want 1 worker goroutine(s), got 0")
 }
+
+func TestWorkQueueHealthCheck(t *testing.T) {
+	t.Parallel()
+
+	health := healthcheck.NewHandler()
+	handler := func(string) error {
+		return nil
+	}
+	wq := NewWorkerQueue(handler, logrus.WithField("source", "test"), "test")
+	health.AddLivenessCheck("test", wq.Healthy)
+
+	server := httptest.NewServer(health)
+	defer server.Close()
+
+	stop := make(chan struct{})
+	go wq.Run(1, stop)
+
+	url := server.URL + "/live"
+
+	f := func(t *testing.T, url string, status int) {
+		resp, err := http.Get(url)
+		assert.Nil(t, err)
+		defer resp.Body.Close()
+
+		body, err := ioutil.ReadAll(resp.Body)
+		assert.Nil(t, err)
+		assert.Equal(t, status, resp.StatusCode)
+		assert.Equal(t, []byte("{}\n"), body)
+	}
+
+	f(t, url, http.StatusOK)
+
+	close(stop)
+	// closing can take a short while
+	err := wait.PollImmediate(time.Second, 5*time.Second, func() (bool, error) {
+		rc := wq.RunCount()
+		logrus.WithField("runcount", rc).Info("Checking run count")
+		return rc == 0, nil
+	})
+	assert.Nil(t, err)
+
+	// gate
+	assert.Error(t, wq.Healthy())
+	f(t, url, http.StatusServiceUnavailable)
+}