diff --git a/pkg/operator/controllers/config.go b/pkg/operator/controllers/config.go new file mode 100644 index 00000000..2431e3eb --- /dev/null +++ b/pkg/operator/controllers/config.go @@ -0,0 +1,20 @@ +package controllers + +// Config is a struct containing configuration from environment variables +// source: https://github.com/caarlos0/env +type Config struct { + ApiServiceSecurity string `env:"API_SERVICE_SECURITY" envDefault:"none"` + TlsSecretName string `env:"TLS_SECRET_NAME" envDefault:"tls-secret"` + TlsSecretNamespace string `env:"TLS_SECRET_NAMESPACE" envDefault:"thundernetes-system"` + TlsCertificateName string `env:"TLS_CERTIFICATE_FILENAME" envDefault:"tls.crt"` + TlsPrivateKeyFilename string `env:"TLS_PRIVATE_KEY_FILENAME" envDefault:"tls.key"` + PortRegistryExclusivelyGameServerNodes bool `env:"PORT_REGISTRY_EXCLUSIVELY_GAME_SERVER_NODES" envDefault:"false"` + LogLevel string `env:"LOG_LEVEL" envDefault:"info"` + MinPort int32 `env:"MIN_PORT" envDefault:"10000"` + MaxPort int32 `env:"MAX_PORT" envDefault:"12000"` + AllocationApiSvcPort int32 `env:"ALLOC_API_SVC_PORT" envDefault:"5000"` + InitContainerImageLinux string `env:"THUNDERNETES_INIT_CONTAINER_IMAGE,notEmpty" envDefault:"ghcr.io/playfab/thundernetes-initcontainer:0.6.0"` + InitContainerImageWin string `env:"THUNDERNETES_INIT_CONTAINER_IMAGE_WIN,notEmpty" envDefault:"ghcr.io/playfab/thundernetes-initcontainer-win:0.6.0"` + MaxNumberOfGameServersToAdd int `env:"MAX_NUM_GS_TO_ADD" envDefault:"20"` + MaxNumberOfGameServersToDelete int `env:"MAX_NUM_GS_TO_DEL" envDefault:"20"` +} diff --git a/pkg/operator/controllers/gameserverbuild_controller.go b/pkg/operator/controllers/gameserverbuild_controller.go index f976d330..14f66768 100644 --- a/pkg/operator/controllers/gameserverbuild_controller.go +++ b/pkg/operator/controllers/gameserverbuild_controller.go @@ -43,14 +43,6 @@ import ( // value is the number of crashes var crashesPerBuild = sync.Map{} -const ( - // maximum number of GameServers to create per reconcile loop - // we have this in place since each create call is synchronous and we want to minimize the time for each reconcile loop - maxNumberOfGameServersToAdd = 20 - // maximum number of GameServers to delete per reconcile loop - maxNumberOfGameServersToDelete = 20 -) - // Simple async map implementation using a mutex // used to manage the expected GameServer creations and deletions type MutexMap struct { @@ -65,10 +57,11 @@ type GameServerBuildReconciler struct { PortRegistry *PortRegistry Recorder record.EventRecorder expectations *GameServerExpectations + Config *Config } // NewGameServerBuildReconciler returns a pointer to a new GameServerBuildReconciler -func NewGameServerBuildReconciler(mgr manager.Manager, portRegistry *PortRegistry) *GameServerBuildReconciler { +func NewGameServerBuildReconciler(mgr manager.Manager, portRegistry *PortRegistry, cfg *Config) *GameServerBuildReconciler { cl := mgr.GetClient() return &GameServerBuildReconciler{ Client: cl, @@ -76,6 +69,7 @@ func NewGameServerBuildReconciler(mgr manager.Manager, portRegistry *PortRegistr PortRegistry: portRegistry, Recorder: mgr.GetEventRecorderFor("GameServerBuild"), expectations: NewGameServerExpectations(cl), + Config: cfg, } } @@ -187,12 +181,12 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ var totalNumberOfGameServersToDelete int = 0 // user has decreased standingBy numbers if nonActiveGameServersCount > gsb.Spec.StandingBy { - totalNumberOfGameServersToDelete += int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), maxNumberOfGameServersToDelete)) + totalNumberOfGameServersToDelete += int(math.Min(float64(nonActiveGameServersCount-gsb.Spec.StandingBy), float64(r.Config.MaxNumberOfGameServersToDelete))) } // we also need to check if we are above the max // this can happen if the user modifies the spec.Max during the GameServerBuild's lifetime if nonActiveGameServersCount+activeCount > gsb.Spec.Max { - totalNumberOfGameServersToDelete += int(math.Min(float64(totalNumberOfGameServersToDelete+(nonActiveGameServersCount+activeCount-gsb.Spec.Max)), maxNumberOfGameServersToDelete)) + totalNumberOfGameServersToDelete += int(math.Min(float64(totalNumberOfGameServersToDelete+(nonActiveGameServersCount+activeCount-gsb.Spec.Max)), float64(r.Config.MaxNumberOfGameServersToDelete))) } if totalNumberOfGameServersToDelete > 0 { err := r.deleteNonActiveGameServers(ctx, &gsb, &gameServers, totalNumberOfGameServersToDelete) @@ -205,12 +199,12 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ // we're also limiting the number of game servers that are created to avoid issues like this https://github.com/kubernetes-sigs/controller-runtime/issues/1782 // we attempt to create the missing number of game servers, but we don't want to create more than the max // an error channel for the go routines to write errors - errCh := make(chan error, maxNumberOfGameServersToAdd) + errCh := make(chan error, r.Config.MaxNumberOfGameServersToAdd) // a waitgroup for async create calls var wg sync.WaitGroup for i := 0; i < gsb.Spec.StandingBy-nonActiveGameServersCount && i+nonActiveGameServersCount+activeCount < gsb.Spec.Max && - i < maxNumberOfGameServersToAdd; i++ { + i < r.Config.MaxNumberOfGameServersToAdd; i++ { wg.Add(1) go func() { defer wg.Done() diff --git a/pkg/operator/controllers/suite_test.go b/pkg/operator/controllers/suite_test.go index f75b269e..3bfefc3d 100644 --- a/pkg/operator/controllers/suite_test.go +++ b/pkg/operator/controllers/suite_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/caarlos0/env/v6" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" @@ -72,6 +73,14 @@ var _ = BeforeSuite(func() { ErrorIfCRDPathMissing: true, } + //If config is passed to a constructor, whatever fields constructor uses need to be defined explicitly + //This does not pull values from operator.yaml like it does in main.go + //For suite_test the env defaults should be used, defined in const above + config := &Config{} + err := env.Parse(config) + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + cfg, err := testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) @@ -98,7 +107,7 @@ var _ = BeforeSuite(func() { err = portRegistry.SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) - err = (NewGameServerBuildReconciler(k8sManager, portRegistry)).SetupWithManager(k8sManager) + err = (NewGameServerBuildReconciler(k8sManager, portRegistry, config)).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) initContainerImageLinux, initContainerImageWin := "testImageLinux", "testImageWin" diff --git a/pkg/operator/main.go b/pkg/operator/main.go index 41adbbc5..7718d83b 100644 --- a/pkg/operator/main.go +++ b/pkg/operator/main.go @@ -48,23 +48,6 @@ import ( corev1 "k8s.io/api/core/v1" ) -// Config is a struct containing configuration from environment variables -// source: https://github.com/caarlos0/env -type Config struct { - ApiServiceSecurity string `env:"API_SERVICE_SECURITY"` - TlsSecretName string `env:"TLS_SECRET_NAME" envDefault:"tls-secret"` - TlsSecretNamespace string `env:"TLS_SECRET_NAMESPACE" envDefault:"thundernetes-system"` - TlsCertificateName string `env:"TLS_CERTIFICATE_FILENAME" envDefault:"tls.crt"` - TlsPrivateKeyFilename string `env:"TLS_PRIVATE_KEY_FILENAME" envDefault:"tls.key"` - PortRegistryExclusivelyGameServerNodes bool `env:"PORT_REGISTRY_EXCLUSIVELY_GAME_SERVER_NODES" envDefault:"false"` - LogLevel string `env:"LOG_LEVEL" envDefault:"info"` - MinPort int32 `env:"MIN_PORT" envDefault:"10000"` - MaxPort int32 `env:"MAX_PORT" envDefault:"12000"` - AllocationApiSvcPort int32 `env:"ALLOC_API_SVC_PORT" envDefault:"5000"` - InitContainerImageLinux string `env:"THUNDERNETES_INIT_CONTAINER_IMAGE,notEmpty"` - InitContainerImageWin string `env:"THUNDERNETES_INIT_CONTAINER_IMAGE_WIN,notEmpty"` -} - var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") @@ -79,7 +62,7 @@ func init() { func main() { // load configuration from env variables - cfg := &Config{} + cfg := &controllers.Config{} if err := env.Parse(cfg); err != nil { log.Fatal(err, "Cannot load configuration from environment variables") } @@ -151,7 +134,7 @@ func main() { } // initialize the GameServerBuild controller - if err = controllers.NewGameServerBuildReconciler(mgr, portRegistry).SetupWithManager(mgr); err != nil { + if err = controllers.NewGameServerBuildReconciler(mgr, portRegistry, cfg).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GameServerBuild") os.Exit(1) } @@ -187,7 +170,7 @@ func main() { // initializePortRegistry performs some initialization and creates a new PortRegistry struct // the k8sClient is a live API client and is used to get the existing gameservers and the "Ready" Nodes // the crClient is the cached controller-runtime client, used to watch for changes to the nodes from inside the PortRegistry -func initializePortRegistry(k8sClient client.Reader, crClient client.Client, setupLog logr.Logger, cfg *Config) (*controllers.PortRegistry, error) { +func initializePortRegistry(k8sClient client.Reader, crClient client.Client, setupLog logr.Logger, cfg *controllers.Config) (*controllers.PortRegistry, error) { var gameServers mpsv1alpha1.GameServerList if err := k8sClient.List(context.Background(), &gameServers); err != nil { return nil, err @@ -228,7 +211,7 @@ func initializePortRegistry(k8sClient client.Reader, crClient client.Client, set // getTlsSecret returns the TLS secret from the given namespace // used in the allocation API service -func getTlsSecret(k8sClient client.Reader, cfg *Config) ([]byte, []byte, error) { +func getTlsSecret(k8sClient client.Reader, cfg *controllers.Config) ([]byte, []byte, error) { var secret corev1.Secret err := k8sClient.Get(context.Background(), types.NamespacedName{ Name: cfg.TlsSecretName, @@ -241,7 +224,7 @@ func getTlsSecret(k8sClient client.Reader, cfg *Config) ([]byte, []byte, error) } // validateMinMaxPort validates minimum and maximum ports -func validateMinMaxPort(cfg *Config) (int32, int32, error) { +func validateMinMaxPort(cfg *controllers.Config) (int32, int32, error) { if cfg.MinPort >= cfg.MaxPort { return 0, 0, errors.New("MIN_PORT cannot be greater or equal than MAX_PORT") } @@ -273,7 +256,7 @@ func getLogLevel(logLevel string) zapcore.LevelEnabler { // for this to happen, user has to set "API_SERVICE_SECURITY" env as "usetls" and set the env "TLS_SECRET_NAMESPACE" with the namespace // that contains the Kubernetes Secret with the cert // if any of the mentioned conditions are not set, method returns nil -func getCrtKeyIfTlsEnabled(c client.Reader, cfg *Config) ([]byte, []byte) { +func getCrtKeyIfTlsEnabled(c client.Reader, cfg *controllers.Config) ([]byte, []byte) { if cfg.ApiServiceSecurity == "usetls" { crt, key, err := getTlsSecret(c, cfg) if err != nil {