-
Notifications
You must be signed in to change notification settings - Fork 820
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Sync Pod host ports back to GameServer in GCP (#2782)
This is the start of the implementation for #2777: * Most of this is mechanical and implements a thin cloud product abstraction layer in pkg/cloud, instantiated with New(product). The product abstraction provides a single function so far: SyncPodPortsToGameServer. * SyncPodPortsToGameServer is inserted as a hook while syncing IP/ports, to let different cloud providers handle port allocation slightly differently (in this case, GKE Autopilot) * In GKE Autopilot, we look for a JSON string like `{"min":7000,"max":8000,"portsAssigned":{"7001":7737,"7002":7738}}` as an indication that the host ports were reassigned (per policy). As a side note to anyone watching, this is currently an unreleased feature. If we see this, we use the provided mapping to map the host ports in the GameServer.Spec. With this change, it's possible to launch a GameServer and get a healthy GameServer Pod by adding the following annotation: ``` annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: "true" autopilot.gke.io/host-port-assignment: '{"min": 7000, "max": 8000}' ``` If this PR causes any issues, the cloud product auto detection can be disabled by setting `agones.cloudProduct=generic`, or forced to GKE Autopilot using `agones.cloudProduct=gke-autopilot`. In a future PR, I will add the host-port-assignment annotation automatically on Autopilot Co-authored-by: Mark Mandel <[email protected]>
- Loading branch information
1 parent
352617b
commit 3c38876
Showing
18 changed files
with
448 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| // Copyright 2022 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package cloudproduct | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| agonesv1 "agones.dev/agones/pkg/apis/agones/v1" | ||
| "agones.dev/agones/pkg/cloudproduct/generic" | ||
| "agones.dev/agones/pkg/cloudproduct/gke" | ||
| "agones.dev/agones/pkg/util/runtime" | ||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| // CloudProduct provides a generic interface that abstracts cloud product | ||
| // specific functionality. Users should call New() to instantiate a | ||
| // specific cloud product interface. | ||
| type CloudProduct interface { | ||
| // SyncPodPortsToGameServer runs after a Pod has been assigned to a Node and before we sync | ||
| // Pod host ports to the GameServer status. | ||
| SyncPodPortsToGameServer(*agonesv1.GameServer, *corev1.Pod) error | ||
| } | ||
|
|
||
| const ( | ||
| // If --cloud-product=auto, auto-detect | ||
| AutoDetect = "auto" | ||
|
|
||
| genericProduct = "generic" | ||
| ) | ||
|
|
||
| var ( | ||
| logger = runtime.NewLoggerWithSource("cloudproduct") | ||
| productDetectors = []func(context.Context, *kubernetes.Clientset) string{gke.Detect} | ||
| ) | ||
|
|
||
| // New instantiates a new CloudProduct interface by product name. | ||
| func New(ctx context.Context, product string, kc *kubernetes.Clientset) (CloudProduct, error) { | ||
| product = autoDetect(ctx, product, kc) | ||
|
|
||
| switch product { | ||
| case "gke-autopilot": | ||
| return gke.Autopilot() | ||
| case genericProduct: | ||
| return generic.New() | ||
| } | ||
| return nil, fmt.Errorf("unknown cloud product: %q", product) | ||
| } | ||
|
|
||
| func autoDetect(ctx context.Context, product string, kc *kubernetes.Clientset) string { | ||
| if product != AutoDetect { | ||
| logger.Infof("Cloud product forced to %q, skipping auto-detection", product) | ||
| return product | ||
| } | ||
| for _, detect := range productDetectors { | ||
| product = detect(ctx, kc) | ||
| if product != "" { | ||
| logger.Infof("Cloud product detected as %q", product) | ||
| return product | ||
| } | ||
| } | ||
| logger.Infof("Cloud product defaulted to %q", genericProduct) | ||
| return genericProduct | ||
| } | ||
|
|
||
| // MustNewGeneric returns the "generic" cloud product, panicking if an error is encountered. | ||
| func MustNewGeneric(ctx context.Context) CloudProduct { | ||
| c, err := New(ctx, genericProduct, nil) | ||
| runtime.Must(err) | ||
| return c | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| // Copyright 2022 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package cloudproduct provides an abstraction layer to product specific functionality | ||
| package cloudproduct |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Copyright 2022 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package generic provides the generic cloud provider. The generic cloud provider | ||
| // should be usable on all clouds, but provides no special functionality. | ||
| package generic |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| // Copyright 2022 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| package generic | ||
|
|
||
| import ( | ||
| agonesv1 "agones.dev/agones/pkg/apis/agones/v1" | ||
| corev1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| func New() (*generic, error) { return &generic{}, nil } | ||
|
|
||
| type generic struct{} | ||
|
|
||
| func (*generic) SyncPodPortsToGameServer(*agonesv1.GameServer, *corev1.Pod) error { return nil } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| // Copyright 2022 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package gke provides the GKE cloud product interfaces for GKE Standard and Autopilot | ||
| package gke |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| // Copyright 2022 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| package gke | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
|
|
||
| agonesv1 "agones.dev/agones/pkg/apis/agones/v1" | ||
| "agones.dev/agones/pkg/util/runtime" | ||
| "cloud.google.com/go/compute/metadata" | ||
| "github.com/pkg/errors" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| const ( | ||
| workloadDefaulterWebhook = "workload-defaulter.config.common-webhooks.networking.gke.io" | ||
| noWorkloadDefaulter = "failed to get MutatingWebhookConfigurations/workload-defaulter.config.common-webhooks.networking.gke.io (error expected if not on GKE Autopilot)" | ||
| hostPortAssignmentAnnotation = "autopilot.gke.io/host-port-assignment" | ||
| ) | ||
|
|
||
| var logger = runtime.NewLoggerWithSource("gke") | ||
|
|
||
| type gkeAutopilot struct{} | ||
|
|
||
| // hostPortAssignment is the JSON structure of the `host-port-assignment` annotation | ||
| type hostPortAssignment struct { | ||
| Min int32 `json:"min,omitempty"` | ||
| Max int32 `json:"max,omitempty"` | ||
| PortsAssigned map[int32]int32 `json:"portsAssigned,omitempty"` // old -> new | ||
| } | ||
|
|
||
| func Detect(ctx context.Context, kc *kubernetes.Clientset) string { | ||
| if !metadata.OnGCE() { | ||
| return "" | ||
| } | ||
| // Look for the workload defaulter - this is the current best method to detect Autopilot | ||
| if _, err := kc.AdmissionregistrationV1().MutatingWebhookConfigurations().Get( | ||
| ctx, workloadDefaulterWebhook, metav1.GetOptions{}); err != nil { | ||
| logger.WithError(err).WithField("reason", noWorkloadDefaulter).Info( | ||
| "Assuming GKE Standard and defaulting to generic provider") | ||
| return "" // GKE standard, but we don't need an interface for it just yet. | ||
| } | ||
| logger.Info("Running on GKE Autopilot (skip detection with --cloud-product=gke-autopilot)") | ||
| return "gke-autopilot" | ||
| } | ||
|
|
||
| func Autopilot() (*gkeAutopilot, error) { return &gkeAutopilot{}, nil } | ||
|
|
||
| func (*gkeAutopilot) SyncPodPortsToGameServer(gs *agonesv1.GameServer, pod *corev1.Pod) error { | ||
| // If applyGameServerAddressAndPort has already filled in Status, SyncPodPortsToGameServer | ||
| // has already run. Skip syncing from the Pod again - this avoids having to reason | ||
| // about whether we're re-applying the old->new mapping. | ||
| if len(gs.Status.Ports) == len(gs.Spec.Ports) { | ||
| return nil | ||
| } | ||
| annotation, ok := pod.ObjectMeta.Annotations[hostPortAssignmentAnnotation] | ||
| if !ok { | ||
| return nil | ||
| } | ||
| var hpa hostPortAssignment | ||
| if err := json.Unmarshal([]byte(annotation), &hpa); err != nil { | ||
| return errors.Wrapf(err, "could not unmarshal annotation %s (value %q)", hostPortAssignmentAnnotation, annotation) | ||
| } | ||
| for i, p := range gs.Spec.Ports { | ||
| if newPort, ok := hpa.PortsAssigned[p.HostPort]; ok { | ||
| gs.Spec.Ports[i].HostPort = newPort | ||
| } | ||
| } | ||
| return nil | ||
| } |
Oops, something went wrong.