Add QPS settings to Allocation endpoints

Allocation endpoints where throttled to the default ~4qps for a Kubernetes client. Matching the controller settings on standard QPS and Burst to allow higher throughput. Closes #1852
googleforgames · Oct 22, 2020 · e3140c3 · e3140c3
1 parent 76fde07
commit e3140c3
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 2 deletions.
diff --git a/cmd/allocator/main.go b/cmd/allocator/main.go
@@ -76,7 +76,7 @@ func main() {
 	// http.DefaultServerMux is used for http connection, not for https
 	http.Handle("/", health)
 
-	kubeClient, agonesClient, err := getClients()
+	kubeClient, agonesClient, err := getClients(conf)
 	if err != nil {
 		logger.WithError(err).Fatal("could not create clients")
 	}
@@ -312,13 +312,16 @@ func (h *serviceHandler) verifyClientCertificate(rawCerts [][]byte, verifiedChai
 }
 
 // Set up our client which we will use to call the API
-func getClients() (*kubernetes.Clientset, *versioned.Clientset, error) {
+func getClients(ctlConfig config) (*kubernetes.Clientset, *versioned.Clientset, error) {
 	// Create the in-cluster config
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		return nil, nil, errors.New("Could not create in cluster config")
 	}
 
+	config.QPS = float32(ctlConfig.APIServerSustainedQPS)
+	config.Burst = ctlConfig.APIServerBurstQPS
+
 	// Access to the Agones resources through the Agones Clientset
 	kubeClient, err := kubernetes.NewForConfig(config)
 	if err != nil {

diff --git a/cmd/allocator/metrics.go b/cmd/allocator/metrics.go
@@ -37,13 +37,17 @@ const (
 	tlsDisabledFlag                  = "disable-tls"
 	remoteAllocationTimeoutFlag      = "remote-allocation-timeout"
 	totalRemoteAllocationTimeoutFlag = "total-remote-allocation-timeout"
+	apiServerSustainedQPSFlag        = "api-server-qps"
+	apiServerBurstQPSFlag            = "api-server-qps-burst"
 )
 
 func init() {
 	registerMetricViews()
 }
 
 type config struct {
+	APIServerSustainedQPS        int
+	APIServerBurstQPS            int
 	TLSDisabled                  bool
 	MTLSDisabled                 bool
 	PrometheusMetrics            bool
@@ -56,6 +60,8 @@ type config struct {
 
 func parseEnvFlags() config {
 
+	viper.SetDefault(apiServerSustainedQPSFlag, 100)
+	viper.SetDefault(apiServerBurstQPSFlag, 200)
 	viper.SetDefault(enablePrometheusMetricsFlag, true)
 	viper.SetDefault(enableStackdriverMetricsFlag, false)
 	viper.SetDefault(projectIDFlag, "")
@@ -65,6 +71,8 @@ func parseEnvFlags() config {
 	viper.SetDefault(remoteAllocationTimeoutFlag, 10*time.Second)
 	viper.SetDefault(totalRemoteAllocationTimeoutFlag, 30*time.Second)
 
+	pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server")
+	pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server")
 	pflag.Bool(enablePrometheusMetricsFlag, viper.GetBool(enablePrometheusMetricsFlag), "Flag to activate metrics of Agones. Can also use PROMETHEUS_EXPORTER env variable.")
 	pflag.Bool(enableStackdriverMetricsFlag, viper.GetBool(enableStackdriverMetricsFlag), "Flag to activate stackdriver monitoring metrics for Agones. Can also use STACKDRIVER_EXPORTER env variable.")
 	pflag.String(projectIDFlag, viper.GetString(projectIDFlag), "GCP ProjectID used for Stackdriver, if not specified ProjectID from Application Default Credentials would be used. Can also use GCP_PROJECT_ID env variable.")
@@ -77,6 +85,8 @@ func parseEnvFlags() config {
 	pflag.Parse()
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	runtime.Must(viper.BindEnv(apiServerSustainedQPSFlag))
+	runtime.Must(viper.BindEnv(apiServerBurstQPSFlag))
 	runtime.Must(viper.BindEnv(enablePrometheusMetricsFlag))
 	runtime.Must(viper.BindEnv(enableStackdriverMetricsFlag))
 	runtime.Must(viper.BindEnv(projectIDFlag))
@@ -89,6 +99,8 @@ func parseEnvFlags() config {
 	runtime.Must(runtime.ParseFeaturesFromEnv())
 
 	return config{
+		APIServerSustainedQPS:        int(viper.GetInt32(apiServerSustainedQPSFlag)),
+		APIServerBurstQPS:            int(viper.GetInt32(apiServerBurstQPSFlag)),
 		PrometheusMetrics:            viper.GetBool(enablePrometheusMetricsFlag),
 		Stackdriver:                  viper.GetBool(enableStackdriverMetricsFlag),
 		GCPProjectID:                 viper.GetString(projectIDFlag),

diff --git a/install/helm/agones/templates/service/allocation.yaml b/install/helm/agones/templates/service/allocation.yaml
@@ -123,6 +123,10 @@ spec:
             path: /ready
             port: 8080
         env:
+        - name: API_SERVER_QPS
+          value: {{ .Values.agones.allocator.apiServerQPS | quote }}
+        - name: API_SERVER_QPS_BURST
+          value: {{ .Values.agones.allocator.apiServerQPSBurst | quote }}
         - name: PROMETHEUS_EXPORTER
           value: {{ .Values.agones.metrics.prometheusEnabled | quote }}
         - name: STACKDRIVER_EXPORTER

diff --git a/install/helm/agones/values.yaml b/install/helm/agones/values.yaml
@@ -108,6 +108,8 @@ agones:
       timeoutSeconds: 1
   allocator:
     install: true
+    apiServerQPS: 400
+    apiServerQPSBurst: 500
     annotations: {}
     healthCheck:
       initialDelaySeconds: 3

diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml
@@ -1569,6 +1569,10 @@ spec:
             path: /ready
             port: 8080
         env:
+        - name: API_SERVER_QPS
+          value: "400"
+        - name: API_SERVER_QPS_BURST
+          value: "500"
         - name: PROMETHEUS_EXPORTER
           value: "true"
         - name: STACKDRIVER_EXPORTER

diff --git a/site/content/en/docs/Installation/Install Agones/helm.md b/site/content/en/docs/Installation/Install Agones/helm.md
@@ -248,6 +248,8 @@ The following tables lists the configurable parameters of the Agones chart and t
 
 | Parameter                                           | Description                                                                                     | Default                |
 | --------------------------------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------- |
+| `agones.allocator.apiServerQPS`                     | Maximum sustained queries per second that an allocator should be making against API Server      | `400`                  |
+| `agones.allocator.apiServerQPSBurst`                | Maximum burst queries per second that an allocator should be making against API Server          | `500`                  |
 | `agones.allocator.allocationTimeout`                | Remote allocation call timeout.                                      | `10s`                  |
 | `agones.allocator.remoteAllocationTimeout`          | Total remote allocation timeout including retries.       | `30s`                |
 | `agones.controller.annotations`                     | [Annotations][annotations] added to the Agones controller pods                                  | `{}`                   |