Automate Zero Downtime Scale tests

tests/Makefile

@@ -97,7 +97,7 @@ run-tests-on-vm: ## Run the functional tests on a GCP VM
 
 .PHONY: nfr-test
 nfr-test: ## Run the NFR tests on a GCP VM
-	NFR=true bash scripts/run-tests-gcp-vm.sh
+	NFR=true CI=$(CI) bash scripts/run-tests-gcp-vm.sh
 
 .PHONY: start-longevity-test
 start-longevity-test: export START_LONGEVITY=true
@@ -110,7 +110,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
 	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \
-	    --label-filter "nfr" $(GINKGO_FLAGS) ./suite --  --gateway-api-version=$(GW_API_VERSION) \
+	    --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \

tests/README.md

@@ -258,8 +258,6 @@ Directory structure is as follows:
 - `scripts`: contain scripts used to set up the environment and run the tests
 - `suite`: contains the test files
 
-> Note: Existing NFR tests will be migrated into this testing `suite` and results stored in the `results` directory.
-
 ### Logging in tests
 
 To log in the tests, use the `GinkgoWriter` interface described here: https://onsi.github.io/ginkgo/#logging-output.

tests/framework/resourcemanager.go

@@ -744,3 +744,32 @@ func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, na
 		},
 	)
 }
+
+// WaitForGatewayObservedGeneration waits for the provided Gateway's ObservedGeneration to equal the expected value.
+func (rm *ResourceManager) WaitForGatewayObservedGeneration(
+	ctx context.Context,
+	namespace,
+	name string,
+	generation int,
+) error {
+	return wait.PollUntilContextCancel(
+		ctx,
+		500*time.Millisecond,
+		true, /* poll immediately */
+		func(ctx context.Context) (bool, error) {
+			var gw v1.Gateway
+			key := types.NamespacedName{Namespace: namespace, Name: name}
+			if err := rm.K8sClient.Get(ctx, key, &gw); err != nil {
+				return false, err
+			}
+
+			for _, cond := range gw.Status.Conditions {
+				if cond.ObservedGeneration == int64(generation) {
+					return true, nil
+				}
+			}
+
+			return false, nil
+		},
+	)
+}

tests/scripts/run-tests-gcp-vm.sh

@@ -23,7 +23,13 @@ if [ $retcode -ne 0 ]; then
 fi
 
 if [ "${NFR}" = "true" ]; then
-    gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
+    ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config
+    if [ "${CI}" = "false" ]; then
+      gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh > /dev/null
+      rsync -ave 'ssh -F ngf-gcp.ssh' username@${RESOURCE_NAME}.${GKE_CLUSTER_ZONE}.${GKE_PROJECT}:~/nginx-gateway-fabric/tests/results .
+    else
+      gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
+    fi
 fi
 
 ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results

tests/suite/manifests/ngf-upgrade/values.yaml

@@ -25,7 +25,3 @@ affinity:
       labelSelector:
         matchLabels:
           app.kubernetes.io/name: nginx-gateway
-
-service:
-  annotations:
-    networking.gke.io/load-balancer-type: "Internal"

tests/zero-downtime-scaling/values-25-node.yaml → tests/suite/manifests/scale/zero-downtime/values-affinity.yaml

@@ -1,18 +1,11 @@
 nginxGateway:
-  image:
-    repository: ghcr.io/nginxinc/nginx-gateway-fabric
-    tag: edge # change this tag if you are testing a different version
-    pullPolicy: IfNotPresent
   lifecycle:
     preStop:
       exec:
         command:
         - /usr/bin/gateway
         - sleep
         - --duration=40s
-  config:
-    logging:
-      level: debug
 
 nginx:
   lifecycle:
@@ -31,7 +24,3 @@ affinity:
       labelSelector:
         matchLabels:
           app.kubernetes.io/name: nginx-gateway
-
-service:
-  annotations:
-    networking.gke.io/load-balancer-type: "Internal"

tests/suite/manifests/scale/zero-downtime/values.yaml

@@ -0,0 +1,18 @@
+nginxGateway:
+  lifecycle:
+    preStop:
+      exec:
+        command:
+        - /usr/bin/gateway
+        - sleep
+        - --duration=40s
+
+nginx:
+  lifecycle:
+    preStop:
+      exec:
+        command:
+        - /bin/sleep
+        - "40"
+
+terminationGracePeriodSeconds: 50