From bae93c127bd5d5e7b295214b8e48b690372c8b69 Mon Sep 17 00:00:00 2001 From: Ciara Stacke <18287516+ciarams87@users.noreply.github.com> Date: Wed, 28 Jun 2023 09:00:11 +0100 Subject: [PATCH] Run conformance tests in pod on cluster (#787) * Run conformance tests in pod on cluster --- conformance/Makefile | 50 +++++++++++++---- conformance/README.md | 73 +++++++++++++++---------- conformance/tests/conformance-rbac.yaml | 63 +++++++++++++++++++++ docs/developer/quickstart.md | 5 ++ docs/developer/testing.md | 6 +- 5 files changed, 153 insertions(+), 44 deletions(-) create mode 100644 conformance/tests/conformance-rbac.yaml diff --git a/conformance/Makefile b/conformance/Makefile index 8e2ee41504..8a2963b5a3 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -6,6 +6,8 @@ EXEMPT_FEATURES = ReferenceGrant KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind TAG = latest PREFIX = conformance-test-runner +NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml +NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(NKG_DEPLOYMENT_MANIFEST)) .DEFAULT_GOAL := help .PHONY: help @@ -21,13 +23,19 @@ create-kind-cluster: ## Create a kind cluster kind create cluster --image kindest/node:v1.27.1 kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config -.PHONY: prepare-nkg -prepare-nkg: ## Build and load NKG container on configured kind cluster +.PHONY: preload-nginx-container +preload-nginx-container: ## Preload NGINX container on configured kind cluster + docker pull $(NGINX_IMAGE) + kind load docker-image $(NGINX_IMAGE) + +.PHONY: build-and-load-images +build-and-load-images: preload-nginx-container ## Build NKG container and load it and NGINX container on configured kind cluster + yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST) cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container kind load docker-image $(NKG_PREFIX):$(NKG_TAG) -.PHONY: install-nkg -install-nkg: ## Install NKG with provisioner on configured kind cluster +.PHONY: prepare-nkg-dependencies +prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system kubectl apply -f ../deploy/manifests/namespace.yaml @@ -36,16 +44,30 @@ install-nkg: ## Install NKG with provisioner on configured kind cluster kubectl apply -f ../deploy/manifests/rbac.yaml kubectl apply -f ../deploy/manifests/gatewayclass.yaml kubectl apply -f ../deploy/manifests/service/nodeport.yaml - kubectl apply -f provisioner/provisioner.yaml -.PHONY: update-test-kind-config -update-test-kind-config: ## Update kind config - sed -ir "s|server:.*|server: https://kind-control-plane:6443|" $(KIND_KUBE_CONFIG_FOLDER)/config +.PHONY: install-nkg-local-build +install-nkg-local-build: build-and-load-images prepare-nkg-dependencies ## Install NKG from local build with provisioner on configured kind cluster + yq '(select(di != 3))' provisioner/provisioner.yaml | kubectl apply -f - + yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' provisioner/provisioner.yaml | kubectl apply -f - + +.PHONY: install-nkg-edge +install-nkg-edge: preload-nginx-container prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster + kubectl apply -f provisioner/provisioner.yaml .PHONY: run-conformance-tests -run-conformance-tests: update-test-kind-config ## Run conformance tests - docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) \ - go test -timeout 25m -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug --supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES) +run-conformance-tests: ## Run conformance tests + kind load docker-image $(PREFIX):$(TAG) + kubectl apply -f tests/conformance-rbac.yaml + kubectl run -i conformance \ + --image=$(PREFIX):$(TAG) --image-pull-policy=Never \ + --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ + --restart=Never -- go test -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug \ + --supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES) + +.PHONY: cleanup-conformance-tests +cleanup-conformance-tests: ## Clean up conformance tests fixtures + kubectl delete pod conformance + kubectl delete -f tests/conformance-rbac.yaml .PHONY: uninstall-nkg uninstall-nkg: ## Uninstall NKG on configured kind cluster @@ -54,7 +76,11 @@ uninstall-nkg: ## Uninstall NKG on configured kind cluster kubectl delete -f ../deploy/manifests/namespace.yaml kubectl delete clusterrole nginx-gateway-provisioner kubectl delete clusterrolebinding nginx-gateway-provisioner - + +.PHONY: undo-image-update +undo-image-update: ## Undo the NKG image name and tag in deployment manifest + git checkout -- $(NKG_DEPLOYMENT_MANIFEST) + .PHONY: delete-kind-cluster delete-kind-cluster: ## Delete kind cluster kind delete cluster diff --git a/conformance/README.md b/conformance/README.md index d5c8d3dfc6..eecc911dc6 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -5,6 +5,7 @@ * [kind](https://kind.sigs.k8s.io/). * Docker. * Golang. +* [yq](https://github.com/mikefarah/yq/#install) **Note**: all commands in steps below are executed from the ```conformance``` directory @@ -13,66 +14,80 @@ List available commands: ```bash $ make +build-and-load-images Build NKG container and load it and NGINX container on configured kind cluster build-test-runner-image Build conformance test runner image +cleanup-conformance-tests Clean up conformance tests fixtures create-kind-cluster Create a kind cluster delete-kind-cluster Delete kind cluster help Display this help -install-nkg Install NKG with provisioner on configured kind cluster -prepare-nkg Build and load NKG container on configured kind cluster +install-nkg-edge Install NKG with provisioner from edge on configured kind cluster +install-nkg-local-build Install NKG from local build with provisioner on configured kind cluster +preload-nginx-container Preload NGINX container on configured kind cluster +prepare-nkg-dependencies Install NKG dependencies on configured kind cluster run-conformance-tests Run conformance tests +undo-image-update Undo the NKG image name and tag in deployment manifest uninstall-nkg Uninstall NKG on configured kind cluster -update-test-kind-config Update kind config ``` + +**Note:** The following variables are configurable when running the below `make` commands: + +| Variable | Default | Description | +| ------------- | ------------- | ------------- | +| TAG | latest | The tag for the conformance test image | +| PREFIX | conformance-test-runner | The prefix for the conformance test image | +| NKG_TAG | edge | The tag for the locally built NKG image | +| NKG_PREFIX | nginx-kubernetes-gateway | The prefix for the locally built NKG image | +| KIND_KUBE_CONFIG_FOLDER | ~/.kube/kind | The location of the kubeconfig folder | +| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests | +| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. | +| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests | +| NGINX_IMAGE | as defined in the ../deploy/manifests/deployment.yaml file | The NGINX image for the NKG deployments | +| NKG_DEPLOYMENT_MANIFEST | ../deploy/manifests/deployment.yaml | The location of the NKG deployment manifest | + ### Step 1 - Create a kind Cluster ```bash $ make create-kind-cluster ``` +### Step 2 - Install Nginx Kubernetes Gateway to configured kind cluster -### Step 2 - Update NKG deployment and provisioner manifests -**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway -* Set NKG_PREFIX= NKG_TAG= to preferred values. -* Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block. -* Navigate to `conformance/provisioner` and update values in `provisioner.yaml` as specified in below code-block. -* Save the changes. -``` -. -.. -containers: -- image: : - imagePullPolicy: Never -.. -. +#### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster +```bash +$ make install-nkg-local-build ``` -### Step 3 - Build and load Nginx Kubernetes Gateway container to configured kind cluster -**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway +#### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster +Instead of the above command, you can skip the build NKG image step and prepare the environment to instead +use the `edge` image ```bash -$ make NKG_PREFIX= NKG_TAG= prepare-nkg - +$ make install-nkg-edge ``` -### Step 4 - Build conformance test runner image + +### Step 3 - Build conformance test runner image ```bash $ make build-test-runner-image ``` -### Step 5 - Install Nginx Kubernetes Gateway +### Step 4 - Run Gateway conformance tests ```bash -$ make install-nkg +$ make run-conformance-tests ``` -### Step 6 - Run Gateway conformance tests +### Step 5 - Cleanup the conformance test fixtures and uninstall Nginx Kubernetes Gateway ```bash -$ make run-conformance-tests +$ make cleanup-conformance-tests +$ make uninstall-nkg ``` -### Step 7 - Uninstall Nginx Kubernetes Gateway +### Step 6 - Revert changes to the NKG deployment manifest +**Optional** Not required if using `edge` image +**Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file! ```bash -$ make uninstall-nkg +$ make undo-image-update ``` -### Step 8 - Delete kind cluster +### Step 7 - Delete kind cluster ```bash $ make delete-kind-cluster ``` diff --git a/conformance/tests/conformance-rbac.yaml b/conformance/tests/conformance-rbac.yaml new file mode 100644 index 0000000000..b6037bb09a --- /dev/null +++ b/conformance/tests/conformance-rbac.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: conformance +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: conformance +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - secrets + - services + verbs: + - create + - delete + - get + - list + - update +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + verbs: + - get + - list +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + - httproutes + verbs: + - create + - delete + - get + - list + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nginx-conformance +subjects: +- kind: ServiceAccount + name: conformance + namespace: default +roleRef: + kind: ClusterRole + name: conformance + apiGroup: rbac.authorization.k8s.io diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 080a68a898..a5a272328a 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -14,6 +14,7 @@ Follow these steps to set up your development environment. - [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) - [git](https://git-scm.com/) - [GNU Make](https://www.gnu.org/software/software.html) + - [yq](https://github.com/mikefarah/yq/#install) - [fieldalignment](https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/fieldalignment): ```shell @@ -92,6 +93,10 @@ make unit-test For more details on testing, see the [testing](/docs/developer/testing.md) documentation. +## Gateway API Conformance Testing + +To run Gateway API conformance tests, please follow the instructions on [this](/conformance/README.md) page. + ## Run the Linter To lint the code, run the following make command from the project's root directory: diff --git a/docs/developer/testing.md b/docs/developer/testing.md index 8fdbecc2d4..6cd0c0b61c 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -1,9 +1,9 @@ # Testing This document provides guidelines for testing, including instructions on running the unit tests, accessing the code -coverage report, and performing manual testing. By following these guidelines, you will gain a thorough understanding of -the project's approach to unit testing, enabling you to ensure code quality, validate functionality, and maintain robust -test coverage. +coverage report, performing manual testing, and running the conformance tests. By following these guidelines, you will +gain a thorough understanding of the project's approach to unit testing, enabling you to ensure code quality, validate +functionality, and maintain robust test coverage. ## Unit Test Guidelines