Add GKE Autopilot to e2e (googleforgames#2913)

Run e2e tests against Autopilot 1.24 cluster. Mark as `allowFailure`
for now just to get a sense for whether it works - I will manually
scan CI after this goes in.

We probably should script additions/mutations to this in the
future...

Along the way: Skip the TestGameServerEvicted on Autopilot, filed googleforgames#2890

build/Makefile

@@ -319,8 +319,10 @@ test-e2e:
 # - When E2E_USE_GOTESTSUM is enabled, use GOTESTSUM_ARGS to pass args to gotestsum
 # If GAMESERVERS_NAMESPACE is empty - random namespace will be created
 test-e2e-integration: FEATURE_GATES ?= $(ALPHA_FEATURE_GATES)
+test-e2e-integration: CLOUD_PRODUCT ?= generic
 test-e2e-integration: GAMESERVERS_NAMESPACE ?= ""
 test-e2e-integration: GO_E2E_TEST_INTEGRATION_ARGS ?=\
+  --cloud-product=$(CLOUD_PRODUCT) \
   --gameserver-image=$(GS_TEST_IMAGE) \
   --pullsecret=$(IMAGE_PULL_SECRET) \
   --feature-gates=$(FEATURE_GATES) \
@@ -336,7 +338,9 @@ else
 endif
 	echo "Finishing e2e integration test!"
 
+test-e2e-failure: CLOUD_PRODUCT ?= generic
 test-e2e-failure: GO_E2E_TEST_INTEGRATION_ARGS ?=\
+	--cloud-product=$(CLOUD_PRODUCT) \
 	--gameserver-image=$(GS_TEST_IMAGE) \
 	--pullsecret=$(IMAGE_PULL_SECRET)
 test-e2e-failure: $(ensure-build-image)
@@ -387,6 +391,7 @@ install: LOG_LEVEL := "debug"
 install: EXTERNAL_IP ?= $(shell $(DOCKER_RUN) kubectl get services agones-allocator -n agones-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
 install: FEATURE_GATES ?= $(ALPHA_FEATURE_GATES)
 install: SERVICE_MONITOR := false
+install: CONTROLLER_SAFE_TO_EVICT := false # TODO(#2797): Remove when `SplitControllerAndExtensions` => safe-to-evict=true
 install: HELM_ARGS ?=
 install: $(ensure-build-image) install-custom-pull-secret
 	-$(DOCKER_RUN) bash -c '[[ $$(helm status agones -n agones-system --output json | jq -r ".info.status") =~ (failed|pending-.*) ]] && helm uninstall agones --namespace=agones-system || true'
@@ -403,6 +408,7 @@ install: $(ensure-build-image) install-custom-pull-secret
 		--set agones.featureGates=$(FEATURE_GATES) \
 		--set agones.allocator.service.loadBalancerIP=$(EXTERNAL_IP) \
 		--set agones.metrics.serviceMonitor.enabled=$(SERVICE_MONITOR) \
+		--set agones.controller.safeToEvict=$(CONTROLLER_SAFE_TO_EVICT) \
 		$(HELM_ARGS) \
 		agones $(mount_path)/install/helm/agones/
 

build/e2e-image/e2e.sh

@@ -15,13 +15,15 @@
 # limitations under the License.
 
 FEATURES=$1
-REGISTRY=$2
+CLOUD_PRODUCT=$2
+REGISTRY=$3
 
 echo $FEATURES
 echo $REGISTRY
 set -e
 echo "installing current release"
-DOCKER_RUN= make install FEATURE_GATES='"'$FEATURES'"' REGISTRY='"'$REGISTRY'"'
+# TODO(#2797): Remove when `SplitControllerAndExtensions` => safe-to-evict=true
+DOCKER_RUN= make install FEATURE_GATES='"'$FEATURES'"' REGISTRY='"'$REGISTRY'"' CONTROLLER_SAFE_TO_EVICT=true
 echo "starting e2e test"
-DOCKER_RUN= make test-e2e ARGS=-parallel=16 E2E_USE_GOTESTSUM=true GOTESTSUM_VERBOSE=true FEATURE_GATES='"'$FEATURES'"'
+DOCKER_RUN= make test-e2e ARGS=-parallel=16 E2E_USE_GOTESTSUM=true GOTESTSUM_VERBOSE=true FEATURE_GATES='"'$FEATURES'"' CLOUD_PRODUCT='"'$CLOUD_PRODUCT'"'
 echo "completed e2e test"

build/e2e-image/entrypoint.sh

@@ -15,8 +15,10 @@
 # limitations under the License.
 set -e
 FEATURES=$1
-TEST_CLUSTER_NAME=$2
-REGISTRY=$3
+CLOUD_PRODUCT=$2
+TEST_CLUSTER_NAME=$3
+TEST_CLUSTER_LOCATION=$4
+REGISTRY=$5
 
 echo $FEATURES
 export SHELL="/bin/bash"
@@ -29,12 +31,12 @@ then
         gcloud auth login
 fi
 gcloud container clusters get-credentials $TEST_CLUSTER_NAME \
-        --zone=us-west1-c --project=agones-images
+        --zone=${TEST_CLUSTER_LOCATION} --project=agones-images
 kubectl port-forward statefulset/consul-consul-server 8500:8500 &
 echo "Waiting consul port-forward to launch on 8500..."
 timeout 60 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' 127.0.0.1 8500
 echo "consul port-forward launched. Starting e2e tests..."
-echo "consul lock -child-exit-code=true -timeout 1h -verbose LockE2E '/root/e2e.sh "$FEATURES" "$REGISTRY"'"
-consul lock -child-exit-code=true -timeout 1h -verbose LockE2E '/root/e2e.sh "'$FEATURES'" "'$REGISTRY'"'
+echo "consul lock -child-exit-code=true -timeout 1h -verbose LockE2E '/root/e2e.sh "$FEATURES" "$CLOUD_PRODUCT" "$REGISTRY"'"
+consul lock -child-exit-code=true -timeout 1h -verbose LockE2E '/root/e2e.sh "'$FEATURES'" "'$CLOUD_PRODUCT'" "'$REGISTRY'"'
 killall -q kubectl || true
 echo "successfully killed kubectl proxy"

cloudbuild.yaml

@@ -254,7 +254,9 @@ steps:
 - name: 'e2e-runner'
   args:
     - 'CustomFasSyncInterval=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&Example=true'
+    - 'generic'
     - 'e2e-test-cluster'
+    - 'us-west1-c'
     - "${_REGISTRY}"
   id: e2e-feature-gates
   waitFor:
@@ -268,13 +270,50 @@ steps:
 - name: 'e2e-runner'
   args:
     - ''
+    - 'generic'
     - 'e2e-test-cluster'
+    - 'us-west1-c'
     - "${_REGISTRY}"
   id: e2e-stable
   waitFor:
     - push-images
     - build-e2e
 
+#
+# Run the e2e tests on GKE Autopilot with FeatureGates inverted compared to Stable
+#
+# Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
+
+- name: 'e2e-runner'
+  args:
+    - 'CustomFasSyncInterval=false&SDKGracefulTermination=false&StateAllocationFilter=false&PlayerAllocationFilter=true&PlayerTracking=true&ResetMetricsOnDelete=true&SafeToEvict=true&PodHostname=true&Example=true'
+    - 'gke-autopilot'
+    - 'gke-autopilot-e2e-test-cluster-1-24'
+    - 'us-west1'
+    - "${_REGISTRY}"
+  id: e2e-feature-gates-gke-autopilot
+  allowFailure: true # for now, run the step but don't block on it
+  waitFor:
+    - push-images
+    - build-e2e
+
+#
+# Run the e2e tests on GKE Autopilot without FeatureGates, except SafeToEvict, which we need on Autopilot
+#
+
+- name: 'e2e-runner'
+  args:
+    - 'SafeToEvict=true'
+    - 'gke-autopilot'
+    - 'gke-autopilot-e2e-test-cluster-1-24'
+    - 'us-west1'
+    - "${_REGISTRY}"
+  id: e2e-stable-gke-autopilot
+  allowFailure: true # for now, run the step but don't block on it
+  waitFor:
+    - push-images
+    - build-e2e
+
 #
 # SDK conformance tests
 #

test/e2e/gameserver_test.go

@@ -779,6 +779,7 @@ func TestGameServerShutdown(t *testing.T) {
 // TestGameServerEvicted test that if Gameserver would be evicted than it becomes Unhealthy
 // Ephemeral Storage limit set to 0Mi
 func TestGameServerEvicted(t *testing.T) {
+	framework.SkipOnCloudProduct(t, "gke-autopilot", "Autopilot adjusts ephmeral storage to a minimum of 10Mi, see https://github.com/googleforgames/agones/issues/2890")
 	t.Parallel()
 	ctx := context.Background()
 	gs := framework.DefaultGameServer(framework.Namespace)