Skip to content

Commit

Permalink
Prototype of using env variables to dynamically set the grpc port (#1086
Browse files Browse the repository at this point in the history 
)

in the golang sdk.
  • Loading branch information
@roberthbailey @markmandel
roberthbailey authored and markmandel committed Oct 1, 2019
1 parent 72a3ddd commit c64de9f
Show file tree
Hide file tree
Showing 6 changed files with 358 additions and 11 deletions.
59 changes: 55 additions & 4 deletions pkg/gameservers/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"encoding/json"
"fmt"
"net"
"strconv"
"sync"
"time"

Expand Down Expand Up @@ -55,6 +56,12 @@ import (
"k8s.io/client-go/util/workqueue"
)

const (
sdkserverSidecarName = "agones-gameserver-sidecar"
grpcPortEnvVar = "AGONES_SDK_GRPC_PORT"
httpPortEnvVar = "AGONES_SDK_HTTP_PORT"
)

// Controller is a the main GameServer crd controller
type Controller struct {
baseLogger *logrus.Entry
Expand Down Expand Up @@ -520,11 +527,9 @@ func (c *Controller) syncDevelopmentGameServer(gs *agonesv1.GameServer) (*agones
// createGameServerPod creates the backing Pod for a given GameServer
func (c *Controller) createGameServerPod(gs *agonesv1.GameServer) (*agonesv1.GameServer, error) {
sidecar := c.sidecar(gs)
var pod *corev1.Pod
pod, err := gs.Pod(sidecar)

// this shouldn't happen, but if it does.
if err != nil {
// this shouldn't happen, but if it does.
c.loggerForGameServer(gs).WithError(err).Error("error creating pod from Game Server")
gs, err = c.moveToErrorState(gs, err.Error())
return gs, err
Expand All @@ -539,6 +544,7 @@ func (c *Controller) createGameServerPod(gs *agonesv1.GameServer) (*agonesv1.Gam
}

c.addGameServerHealthCheck(gs, pod)
c.addSDKServerEnvVars(gs, pod)

c.loggerForGameServer(gs).WithField("pod", pod).Info("creating Pod for GameServer")
pod, err = c.podGetter.Pods(gs.ObjectMeta.Namespace).Create(pod)
Expand All @@ -563,7 +569,7 @@ func (c *Controller) createGameServerPod(gs *agonesv1.GameServer) (*agonesv1.Gam
// sidecar creates the sidecar container for a given GameServer
func (c *Controller) sidecar(gs *agonesv1.GameServer) corev1.Container {
sidecar := corev1.Container{
Name: "agones-gameserver-sidecar",
Name: sdkserverSidecarName,
Image: c.sidecarImage,
Env: []corev1.EnvVar{
{
Expand Down Expand Up @@ -639,6 +645,51 @@ func (c *Controller) addGameServerHealthCheck(gs *agonesv1.GameServer, pod *core
})
}

func (c *Controller) addSDKServerEnvVars(gs *agonesv1.GameServer, pod *corev1.Pod) {
for i, c := range pod.Spec.Containers {
if c.Name != sdkserverSidecarName {
sdkEnvVars := sdkEnvironmentVariables(gs)
if sdkEnvVars == nil {
// If a gameserver was created before 1.1 when we started defaulting the grpc and http ports,
// don't change the container spec.
continue
}

// Filter out environment variables that have reserved names.
// From https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
env := c.Env[:0]
for _, e := range c.Env {
if !reservedEnvironmentVariableName(e.Name) {
env = append(env, e)
}
}
c.Env = append(env, sdkEnvVars...)
pod.Spec.Containers[i] = c
}
}
}

func reservedEnvironmentVariableName(name string) bool {
return name == grpcPortEnvVar || name == httpPortEnvVar
}

func sdkEnvironmentVariables(gs *agonesv1.GameServer) []corev1.EnvVar {
var env []corev1.EnvVar
if gs.Spec.SdkServer.GRPCPort != 0 {
env = append(env, corev1.EnvVar{
Name: grpcPortEnvVar,
Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort)),
})
}
if gs.Spec.SdkServer.HTTPPort != 0 {
env = append(env, corev1.EnvVar{
Name: httpPortEnvVar,
Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort)),
})
}
return env
}

// syncGameServerStartingState looks for a pod that has been scheduled for this GameServer
// and then sets the Status > Address and Ports values.
func (c *Controller) syncGameServerStartingState(gs *agonesv1.GameServer) (*agonesv1.GameServer, error) {
Expand Down
182 changes: 179 additions & 3 deletions pkg/gameservers/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"testing"

"agones.dev/agones/pkg/apis/agones"
Expand Down Expand Up @@ -48,9 +49,7 @@ const (
nodeFixtureName = "node1"
)

var (
GameServerKind = metav1.GroupVersionKind(agonesv1.SchemeGroupVersion.WithKind("GameServer"))
)
var GameServerKind = metav1.GroupVersionKind(agonesv1.SchemeGroupVersion.WithKind("GameServer"))

func TestControllerSyncGameServer(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -159,6 +158,7 @@ func runReconcileDeleteGameServer(t *testing.T, fixture *agonesv1.GameServer) {
assert.Nil(t, err, fmt.Sprintf("Shouldn't be an error from syncGameServer: %+v", err))
assert.False(t, podAction, "Nothing should happen to a Pod")
}

func TestControllerSyncGameServerWithDevIP(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -1208,6 +1208,182 @@ func TestControllerAddGameServerHealthCheck(t *testing.T) {
assert.Equal(t, fixture.Spec.Health.PeriodSeconds, probe.PeriodSeconds)
}

func TestControllerAddSDKServerEnvVars(t *testing.T) {

t.Run("legacy game server without ports set", func(t *testing.T) {
// For backwards compatibility, verify that no variables are set if the ports
// are not set on the game server.
c, _ := newFakeController()
gs := &agonesv1.GameServer{
ObjectMeta: metav1.ObjectMeta{Name: "gameserver", UID: "1234"},
Spec: newSingleContainerSpec(),
}
gs.ApplyDefaults()
gs.Spec.SdkServer = agonesv1.SdkServer{}
pod, err := gs.Pod()
assert.Nil(t, err, "Error: %v", err)
before := pod.DeepCopy()
c.addSDKServerEnvVars(gs, pod)
assert.Equal(t, before, pod, "Error: pod unexpectedly modified. before = %v, after = %v", before, pod)
})

t.Run("game server without any environment", func(t *testing.T) {
c, _ := newFakeController()
gs := &agonesv1.GameServer{
ObjectMeta: metav1.ObjectMeta{Name: "gameserver", UID: "2345"},
Spec: newSingleContainerSpec(),
}
gs.ApplyDefaults()
pod, err := gs.Pod()
assert.Nil(t, err, "Error: %v", err)
c.addSDKServerEnvVars(gs, pod)
assert.Len(t, pod.Spec.Containers, 1, "Expected 1 container, found %d", len(pod.Spec.Containers))
assert.Contains(t, pod.Spec.Containers[0].Env, corev1.EnvVar{Name: grpcPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort))})
assert.Contains(t, pod.Spec.Containers[0].Env, corev1.EnvVar{Name: httpPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort))})
})

t.Run("game server without any conflicting env vars", func(t *testing.T) {
c, _ := newFakeController()
gs := &agonesv1.GameServer{
ObjectMeta: metav1.ObjectMeta{Name: "gameserver", UID: "3456"},
Spec: agonesv1.GameServerSpec{
Ports: []agonesv1.GameServerPort{{ContainerPort: 7777}},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container",
Image: "container/image",
Env: []corev1.EnvVar{{Name: "one", Value: "value"}, {Name: "two", Value: "value"}},
},
},
},
},
},
}
gs.ApplyDefaults()
pod, err := gs.Pod()
assert.Nil(t, err, "Error: %v", err)
c.addSDKServerEnvVars(gs, pod)
assert.Len(t, pod.Spec.Containers, 1, "Expected 1 container, found %d", len(pod.Spec.Containers))
assert.Contains(t, pod.Spec.Containers[0].Env, corev1.EnvVar{Name: grpcPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort))})
assert.Contains(t, pod.Spec.Containers[0].Env, corev1.EnvVar{Name: httpPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort))})
})

t.Run("game server with conflicting env vars", func(t *testing.T) {
c, _ := newFakeController()
gs := &agonesv1.GameServer{
ObjectMeta: metav1.ObjectMeta{Name: "gameserver", UID: "4567"},
Spec: agonesv1.GameServerSpec{
Ports: []agonesv1.GameServerPort{{ContainerPort: 7777}},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container",
Image: "container/image",
Env: []corev1.EnvVar{{Name: grpcPortEnvVar, Value: "value"}, {Name: httpPortEnvVar, Value: "value"}},
},
},
},
},
},
}
gs.ApplyDefaults()
pod, err := gs.Pod()
assert.Nil(t, err, "Error: %v", err)
c.addSDKServerEnvVars(gs, pod)
assert.Len(t, pod.Spec.Containers, 1, "Expected 1 container, found %d", len(pod.Spec.Containers))
assert.Contains(t, pod.Spec.Containers[0].Env, corev1.EnvVar{Name: grpcPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort))})
assert.Contains(t, pod.Spec.Containers[0].Env, corev1.EnvVar{Name: httpPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort))})
})

t.Run("game server with multiple containers", func(t *testing.T) {
c, _ := newFakeController()
gs := &agonesv1.GameServer{
ObjectMeta: metav1.ObjectMeta{Name: "gameserver", UID: "5678"},
Spec: agonesv1.GameServerSpec{
Container: "container1",
Ports: []agonesv1.GameServerPort{{ContainerPort: 7777}},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container1",
Image: "container/gameserver",
},
{
Name: "container2",
Image: "container/image2",
Env: []corev1.EnvVar{{Name: "one", Value: "value"}, {Name: "two", Value: "value"}},
},
{
Name: "container3",
Image: "container/image2",
Env: []corev1.EnvVar{{Name: grpcPortEnvVar, Value: "value"}, {Name: httpPortEnvVar, Value: "value"}},
},
},
},
},
},
}
gs.ApplyDefaults()
pod, err := gs.Pod()
assert.Nil(t, err, "Error: %v", err)
c.addSDKServerEnvVars(gs, pod)
for _, c := range pod.Spec.Containers {
assert.Contains(t, c.Env, corev1.EnvVar{Name: grpcPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort))})
assert.Contains(t, c.Env, corev1.EnvVar{Name: httpPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort))})
}
})

t.Run("environment variables not applied to the sdkserver container", func(t *testing.T) {
c, _ := newFakeController()
gs := &agonesv1.GameServer{
ObjectMeta: metav1.ObjectMeta{Name: "gameserver", UID: "5678"},
Spec: agonesv1.GameServerSpec{
Container: "container1",
Ports: []agonesv1.GameServerPort{{ContainerPort: 7777}},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container1",
Image: "container/gameserver",
},
{
Name: "container2",
Image: "container/image2",
Env: []corev1.EnvVar{{Name: "one", Value: "value"}, {Name: "two", Value: "value"}},
},
{
Name: "container3",
Image: "container/image2",
Env: []corev1.EnvVar{{Name: grpcPortEnvVar, Value: "value"}, {Name: httpPortEnvVar, Value: "value"}},
},
},
},
},
},
}
gs.ApplyDefaults()
sidecar := c.sidecar(gs)
pod, err := gs.Pod(sidecar)
assert.Nil(t, err, "Error: %v", err)
c.addSDKServerEnvVars(gs, pod)
for _, c := range pod.Spec.Containers {
if c.Name == sdkserverSidecarName {
assert.NotContains(t, c.Env, corev1.EnvVar{Name: grpcPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort))})
assert.NotContains(t, c.Env, corev1.EnvVar{Name: httpPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort))})
} else {
assert.Contains(t, c.Env, corev1.EnvVar{Name: grpcPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.GRPCPort))})
assert.Contains(t, c.Env, corev1.EnvVar{Name: httpPortEnvVar, Value: strconv.Itoa(int(gs.Spec.SdkServer.HTTPPort))})
}
}
})

}

func TestIsGameServerPod(t *testing.T) {

t.Run("it is a game server pod", func(t *testing.T) {
Expand Down
31 changes: 27 additions & 4 deletions sdks/go/sdk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package sdk
import (
"fmt"
"io"
"os"
"strconv"
"time"

"agones.dev/agones/pkg/sdk"
Expand All @@ -26,7 +28,7 @@ import (
"google.golang.org/grpc"
)

const port = 59357
const defaultPort = 59357

// GameServerCallback is a function definition to be called
// when a GameServer CRD has been changed
Expand All @@ -39,11 +41,32 @@ type SDK struct {
health sdk.SDK_HealthClient
}

func port() int {
portStr := os.Getenv("AGONES_SDK_GRPC_PORT")
if portStr == "" {
// Environment variable is not set; use the default port.
_, _ = fmt.Fprintf(os.Stderr, "Environment variable AGONES_SDK_GRPC_PORT not defined, using default port %d\n", defaultPort)
return defaultPort
}
p, err := strconv.Atoi(portStr)
if err != nil {
// Environment variable cannot be parsed; use the default port.
_, _ = fmt.Fprintf(os.Stderr, "Unable to parse %q defined in AGONES_SDK_GRPC_PORT into an integer\n", portStr)
return defaultPort
}
if p < 1 || p > 65535 {
// Environment variable is not a valid port; use the default port.
_, _ = fmt.Fprintf(os.Stderr, "Invalid port %d defined in AGONES_SDK_GRPC_PORT. It must be between 1 and 65535\n", p)
return defaultPort
}
return p
}

// NewSDK starts a new SDK instance, and connects to
// localhost on port 59357. Blocks until connection and handshake are made.
// Times out after 30 seconds.
func NewSDK() (*SDK, error) {
addr := fmt.Sprintf("localhost:%d", port)
addr := fmt.Sprintf("localhost:%d", port())
s := &SDK{
ctx: context.Background(),
}
Expand Down Expand Up @@ -130,10 +153,10 @@ func (s *SDK) WatchGameServer(f GameServerCallback) error {
gs, err = stream.Recv()
if err != nil {
if err == io.EOF {
fmt.Println("gameserver event stream EOF received")
_, _ = fmt.Fprintln(os.Stderr, "gameserver event stream EOF received")
return
}
fmt.Printf("error watching GameServer: %s\n", err.Error())
_, _ = fmt.Fprintf(os.Stderr, "error watching GameServer: %s\n", err.Error())
// This is to wait for the reconnection, and not peg the CPU at 100%
time.Sleep(time.Second)
continue
Expand Down
Loading

0 comments on commit c64de9f

Please sign in to comment.