Run conformance tests in pod on cluster (#787)

* Run conformance tests in pod on cluster
nginx · Jun 28, 2023 · bae93c1 · bae93c1
1 parent e04cc27
commit bae93c1
Show file tree

Hide file tree

Showing 5 changed files with 153 additions and 44 deletions.
diff --git a/conformance/Makefile b/conformance/Makefile
@@ -6,6 +6,8 @@ EXEMPT_FEATURES = ReferenceGrant
 KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind
 TAG = latest
 PREFIX = conformance-test-runner
+NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml
+NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(NKG_DEPLOYMENT_MANIFEST))
 .DEFAULT_GOAL := help
 
 .PHONY: help
@@ -21,13 +23,19 @@ create-kind-cluster: ## Create a kind cluster
 	kind create cluster --image kindest/node:v1.27.1
 	kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
 
-.PHONY: prepare-nkg
-prepare-nkg: ## Build and load NKG container on configured kind cluster
+.PHONY: preload-nginx-container
+preload-nginx-container: ## Preload NGINX container on configured kind cluster
+	docker pull $(NGINX_IMAGE)
+	kind load docker-image $(NGINX_IMAGE)
+
+.PHONY: build-and-load-images
+build-and-load-images: preload-nginx-container ## Build NKG container and load it and NGINX container on configured kind cluster
+	yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST)
 	cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container
 	kind load docker-image $(NKG_PREFIX):$(NKG_TAG)
 
-.PHONY: install-nkg
-install-nkg: ## Install NKG with provisioner on configured kind cluster
+.PHONY: prepare-nkg-dependencies
+prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster
 	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
 	kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system 
 	kubectl apply -f ../deploy/manifests/namespace.yaml
@@ -36,16 +44,30 @@ install-nkg: ## Install NKG with provisioner on configured kind cluster
 	kubectl apply -f ../deploy/manifests/rbac.yaml
 	kubectl apply -f ../deploy/manifests/gatewayclass.yaml
 	kubectl apply -f ../deploy/manifests/service/nodeport.yaml
-	kubectl apply -f provisioner/provisioner.yaml
 
-.PHONY: update-test-kind-config 
-update-test-kind-config: ## Update kind config
-	sed -ir "s|server:.*|server: https://kind-control-plane:6443|" $(KIND_KUBE_CONFIG_FOLDER)/config
+.PHONY: install-nkg-local-build
+install-nkg-local-build: build-and-load-images prepare-nkg-dependencies ## Install NKG from local build with provisioner on configured kind cluster
+	yq '(select(di != 3))' provisioner/provisioner.yaml | kubectl apply -f -
+	yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' provisioner/provisioner.yaml | kubectl apply -f -
+
+.PHONY: install-nkg-edge
+install-nkg-edge: preload-nginx-container prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster
+	kubectl apply -f provisioner/provisioner.yaml
 
 .PHONY: run-conformance-tests
-run-conformance-tests: update-test-kind-config ## Run conformance tests
-	docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) \
-		go test -timeout 25m -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug --supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES)
+run-conformance-tests: ## Run conformance tests
+	kind load docker-image $(PREFIX):$(TAG)
+	kubectl apply -f tests/conformance-rbac.yaml
+	kubectl run -i conformance \
+		--image=$(PREFIX):$(TAG) --image-pull-policy=Never \
+		--overrides='{ "spec": { "serviceAccountName": "conformance" }  }' \
+		--restart=Never -- go test -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug \
+						        --supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES)
+
+.PHONY: cleanup-conformance-tests
+cleanup-conformance-tests: ## Clean up conformance tests fixtures
+	kubectl delete pod conformance
+	kubectl delete -f tests/conformance-rbac.yaml
 
 .PHONY: uninstall-nkg
 uninstall-nkg: ## Uninstall NKG on configured kind cluster
@@ -54,7 +76,11 @@ uninstall-nkg: ## Uninstall NKG on configured kind cluster
 	kubectl delete -f ../deploy/manifests/namespace.yaml
 	kubectl delete clusterrole nginx-gateway-provisioner
 	kubectl delete clusterrolebinding nginx-gateway-provisioner
-
+
+.PHONY: undo-image-update
+undo-image-update: ## Undo the NKG image name and tag in deployment manifest
+	git checkout -- $(NKG_DEPLOYMENT_MANIFEST)
+
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
 	kind delete cluster
diff --git a/conformance/README.md b/conformance/README.md
@@ -5,6 +5,7 @@
 * [kind](https://kind.sigs.k8s.io/).
 * Docker.
 * Golang.
+* [yq](https://github.com/mikefarah/yq/#install)
 
 **Note**: all commands in steps below are executed from the ```conformance``` directory
 
@@ -13,66 +14,80 @@ List available commands:
 ```bash
 $ make
 
+build-and-load-images          Build NKG container and load it and NGINX container on configured kind cluster
 build-test-runner-image        Build conformance test runner image
+cleanup-conformance-tests      Clean up conformance tests fixtures
 create-kind-cluster            Create a kind cluster
 delete-kind-cluster            Delete kind cluster
 help                           Display this help
-install-nkg                    Install NKG with provisioner on configured kind cluster
-prepare-nkg                    Build and load NKG container on configured kind cluster
+install-nkg-edge               Install NKG with provisioner from edge on configured kind cluster
+install-nkg-local-build        Install NKG from local build with provisioner on configured kind cluster
+preload-nginx-container        Preload NGINX container on configured kind cluster
+prepare-nkg-dependencies       Install NKG dependencies on configured kind cluster
 run-conformance-tests          Run conformance tests
+undo-image-update              Undo the NKG image name and tag in deployment manifest
 uninstall-nkg                  Uninstall NKG on configured kind cluster
-update-test-kind-config        Update kind config
 ```
+
+**Note:** The following variables are configurable when running the below `make` commands:
+
+| Variable  | Default | Description |
+| ------------- | ------------- | ------------- |
+| TAG | latest  | The tag for the conformance test image |
+| PREFIX | conformance-test-runner | The prefix for the conformance test image |
+| NKG_TAG  | edge  | The tag for the locally built NKG image |
+| NKG_PREFIX | nginx-kubernetes-gateway  | The prefix for the locally built NKG image |
+| KIND_KUBE_CONFIG_FOLDER | ~/.kube/kind  | The location of the kubeconfig folder |
+| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests |
+| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. |
+| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests |
+| NGINX_IMAGE | as defined in the ../deploy/manifests/deployment.yaml file  | The NGINX image for the NKG deployments |
+| NKG_DEPLOYMENT_MANIFEST | ../deploy/manifests/deployment.yaml | The location of the NKG deployment manifest |
+
 ### Step 1 - Create a kind Cluster
 
 ```bash
 $ make create-kind-cluster
 ```
+### Step 2 - Install Nginx Kubernetes Gateway to configured kind cluster
 
-### Step 2 - Update NKG deployment and provisioner manifests
-**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway
-* Set NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> to preferred values.
-* Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block.
-* Navigate to `conformance/provisioner` and update values in `provisioner.yaml` as specified in below code-block.
-* Save the changes.
-```
-.
-..
-containers:
-- image: <repo_name>:<image_tag>
-  imagePullPolicy: Never
-..
-.
+#### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster
+```bash
+$ make install-nkg-local-build
 ```
 
-### Step 3 - Build and load Nginx Kubernetes Gateway container to configured kind cluster
-**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway
+#### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster
+Instead of the above command, you can skip the build NKG image step and prepare the environment to instead
+use the `edge` image
 
 ```bash
-$ make NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> prepare-nkg
-
+$ make install-nkg-edge
 ```
-### Step 4 - Build conformance test runner image
+
+### Step 3 - Build conformance test runner image
 ```bash
 $ make build-test-runner-image
 ```
 
-### Step 5 - Install Nginx Kubernetes Gateway
+### Step 4 - Run Gateway conformance tests
 ```bash
-$ make install-nkg
+$ make run-conformance-tests
 ```
 
-### Step 6 - Run Gateway conformance tests
+### Step 5 - Cleanup the conformance test fixtures and uninstall Nginx Kubernetes Gateway
 ```bash
-$ make run-conformance-tests
+$ make cleanup-conformance-tests
+$ make uninstall-nkg
 ```
 
-### Step 7 - Uninstall Nginx Kubernetes Gateway
+### Step 6 - Revert changes to the NKG deployment manifest
+**Optional** Not required if using `edge` image
+**Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file!
 ```bash
-$ make uninstall-nkg
+$ make undo-image-update
 ```
 
-### Step 8 - Delete kind cluster
+### Step 7 - Delete kind cluster
 ```bash
 $ make delete-kind-cluster
 ```
diff --git a/conformance/tests/conformance-rbac.yaml b/conformance/tests/conformance-rbac.yaml
@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: conformance
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: conformance
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - secrets
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+- apiGroups:
+  - gateway.networking.k8s.io
+  resources:
+  - gatewayclasses
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - gateway.networking.k8s.io
+  resources:
+  - gateways
+  - httproutes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nginx-conformance
+subjects:
+- kind: ServiceAccount
+  name: conformance
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: conformance
+  apiGroup: rbac.authorization.k8s.io
diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md
@@ -14,6 +14,7 @@ Follow these steps to set up your development environment.
     - [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
     - [git](https://git-scm.com/)
     - [GNU Make](https://www.gnu.org/software/software.html)
+    - [yq](https://github.com/mikefarah/yq/#install)
     - [fieldalignment](https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/fieldalignment):
 
       ```shell
@@ -92,6 +93,10 @@ make unit-test
 
 For more details on testing, see the [testing](/docs/developer/testing.md) documentation.
 
+## Gateway API Conformance Testing
+
+To run Gateway API conformance tests, please follow the instructions on [this](/conformance/README.md) page.
+
 ## Run the Linter
 
 To lint the code, run the following make command from the project's root directory:

diff --git a/docs/developer/testing.md b/docs/developer/testing.md
@@ -1,9 +1,9 @@
 # Testing
 
 This document provides guidelines for testing, including instructions on running the unit tests, accessing the code
-coverage report, and performing manual testing. By following these guidelines, you will gain a thorough understanding of
-the project's approach to unit testing, enabling you to ensure code quality, validate functionality, and maintain robust
-test coverage.
+coverage report, performing manual testing, and running the conformance tests. By following these guidelines, you will 
+gain a thorough understanding of the project's approach to unit testing, enabling you to ensure code quality, validate
+functionality, and maintain robust test coverage.
 
 ## Unit Test Guidelines