From a9c9bcf19e764a5bc0e3e19d7fc50f96667b5a5c Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Thu, 5 Sep 2024 14:54:37 -0600 Subject: [PATCH] Add crossplane framework for testing Problem: We want a way to verify nginx configuration reliably in our tests. This is especially useful when introducing new policies, without the desire for testing nginx functionality directly. Solution: Added a framework for getting the nginx config and passing through crossplane into a structured JSON format for easier parsing. Because we now use a local container for crossplane in our functional tests, we'll only support running these tests in a kind cluster. --- .github/workflows/functional.yml | 4 +- tests/Dockerfile.crossplane | 11 ++ tests/Makefile | 17 +- tests/README.md | 40 +--- tests/framework/crossplane.go | 218 ++++++++++++++++++++++ tests/framework/resourcemanager.go | 59 +++++- tests/scripts/remote-scripts/run-tests.sh | 7 - tests/scripts/run-tests-gcp-vm.sh | 21 +-- tests/suite/client_settings_test.go | 123 +++++++++++- tests/suite/system_suite_test.go | 1 + 10 files changed, 435 insertions(+), 66 deletions(-) create mode 100644 tests/Dockerfile.crossplane create mode 100644 tests/framework/crossplane.go delete mode 100755 tests/scripts/remote-scripts/run-tests.sh diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index abbc4c445c..2efb876c78 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -118,7 +118,7 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests - name: Run functional graceful-recovery tests @@ -132,5 +132,5 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests diff --git a/tests/Dockerfile.crossplane b/tests/Dockerfile.crossplane new file mode 100644 index 0000000000..eb79b16493 --- /dev/null +++ b/tests/Dockerfile.crossplane @@ -0,0 +1,11 @@ +FROM python:3.12-alpine + +ARG NGINX_CONF_DIR + +RUN pip install crossplane + +COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf + +USER 101:1001 + +ENTRYPOINT ["sh"] diff --git a/tests/Makefile b/tests/Makefile index abc4aa3ec3..89a2b217f9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,7 @@ GW_SERVICE_TYPE = NodePort## Service type to use for the gateway GW_SVC_GKE_INTERNAL = false NGF_VERSION ?= edge## NGF version to be tested PULL_POLICY = Never## Pull policy for the images +NGINX_CONF_DIR = internal/mode/static/nginx/conf PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC @@ -38,6 +39,10 @@ update-go-modules: ## Update the gateway-api go modules to latest main version build-test-runner-image: ## Build conformance test runner image docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile . +.PHONY: build-crossplane-image +build-crossplane-image: ## Build the crossplane image + docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f Dockerfile.crossplane .. + .PHONY: run-conformance-tests run-conformance-tests: ## Run conformance tests kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --name $(CLUSTER_NAME) @@ -80,9 +85,6 @@ ifeq ($(PLUS_ENABLED),true) NGINX_PREFIX := $(NGINX_PLUS_PREFIX) endif -.PHONY: setup-gcp-and-run-tests -setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests - .PHONY: setup-gcp-and-run-nfr-tests setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests @@ -102,13 +104,9 @@ create-gke-router: ## Create a GKE router to allow egress traffic from private n sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM ./scripts/sync-files-to-vm.sh -.PHONY: run-tests-on-vm -run-tests-on-vm: ## Run the functional tests on a GCP VM - ./scripts/run-tests-gcp-vm.sh - .PHONY: nfr-test nfr-test: ## Run the NFR tests on a GCP VM - NFR=true CI=$(CI) ./scripts/run-tests-gcp-vm.sh + CI=$(CI) ./scripts/run-tests-gcp-vm.sh .PHONY: start-longevity-test start-longevity-test: export START_LONGEVITY=true @@ -130,7 +128,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) .PHONY: test -test: ## Runs the functional tests on your default k8s cluster +test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster + kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME) go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \ --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ --label-filter "functional" $(GINKGO_FLAGS) ./suite -- \ diff --git a/tests/README.md b/tests/README.md index 62531de3e2..297bf8aece 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,10 +28,8 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide - [System Testing](#system-testing) - [Logging in tests](#logging-in-tests) - [Step 1 - Run the tests](#step-1---run-the-tests) - - [1a - Run the functional tests locally](#1a---run-the-functional-tests-locally) - - [1b - Run the tests on a GKE cluster from a GCP VM](#1b---run-the-tests-on-a-gke-cluster-from-a-gcp-vm) - - [Functional Tests](#functional-tests) - - [NFR tests](#nfr-tests) + - [Run the functional tests locally](#run-the-functional-tests-locally) + - [Run the NFR tests on a GKE cluster from a GCP VM](#run-the-nfr-tests-on-a-gke-cluster-from-a-gcp-vm) - [Longevity testing](#longevity-testing) - [Common test amendments](#common-test-amendments) - [Step 2 - Cleanup](#step-2---cleanup) @@ -47,7 +45,7 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide - [yq](https://github.com/mikefarah/yq/#install) - Make. -If running NFR tests, or running functional tests in GKE: +If running NFR tests: - The [gcloud CLI](https://cloud.google.com/sdk/docs/install) - A GKE cluster (if `master-authorized-networks` is enabled, please set `ADD_VM_IP_AUTH_NETWORKS=true` in your vars.env file) @@ -59,9 +57,7 @@ All the commands below are executed from the `tests` directory. You can see all ### Step 1 - Create a Kubernetes cluster -This can be done in a cloud provider of choice, or locally using `kind`. - -**Important**: NFR tests can only be run on a GKE cluster. +**Important**: Functional/conformance tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster. To create a local `kind` cluster: @@ -237,7 +233,7 @@ When running locally, the tests create a port-forward from your NGF Pod to local test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an internal LoadBalancer service which will receive the test traffic. -**Important**: NFR tests can only be run on a GKE cluster. +**Important**: Functional tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster. Directory structure is as follows: @@ -252,7 +248,7 @@ To log in the tests, use the `GinkgoWriter` interface described here: https://on ### Step 1 - Run the tests -#### 1a - Run the functional tests locally +#### Run the functional tests locally ```makefile make test TAG=$(whoami) @@ -273,9 +269,7 @@ To run the telemetry test: make test TAG=$(whoami) GINKGO_LABEL=telemetry ``` -#### 1b - Run the tests on a GKE cluster from a GCP VM - -This step only applies if you are running the NFR tests, or would like to run the functional tests on a GKE cluster from a GCP based VM. +#### Run the NFR tests on a GKE cluster from a GCP VM Before running the below `make` commands, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions. @@ -292,7 +286,7 @@ To just set up the VM with no router (this will not run the tests): make create-and-setup-vm ``` -Otherwise, you can set up the VM, router, and run the tests with a single command. See the options in the sections below. +Otherwise, you can set up the VM, router, and run the tests with a single command. See the options below. By default, the tests run using the version of NGF that was `git cloned` during the setup. If you want to make incremental changes and copy your local changes to the VM to test, you can run @@ -301,22 +295,6 @@ incremental changes and copy your local changes to the VM to test, you can run make sync-files-to-vm ``` -#### Functional Tests - -To set up the GCP environment with the router and VM and then run the tests, run the following command: - -```makefile -make setup-gcp-and-run-tests -``` - -To use an existing VM to run the tests, run the following - -```makefile -make run-tests-on-vm -``` - -#### NFR tests - To set up the GCP environment with the router and VM and then run the tests, run the following command: ```makefile @@ -374,7 +352,7 @@ or to pass a specific flag, e.g. run a specific test, use the GINKGO_FLAGS varia make test TAG=$(whoami) GINKGO_FLAGS='-ginkgo.focus "writes the system info to a results file"' ``` -> Note: if filtering on NFR tests (or functional tests on GKE), set the filter in the appropriate field in your `vars.env` file. +> Note: if filtering on NFR tests, set the filter in the appropriate field in your `vars.env` file. If you are running the tests in GCP, add your required label/ flags to `scripts/var.env`. diff --git a/tests/framework/crossplane.go b/tests/framework/crossplane.go new file mode 100644 index 0000000000..28a80d6cc9 --- /dev/null +++ b/tests/framework/crossplane.go @@ -0,0 +1,218 @@ +package framework + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +// ExpectedNginxField contains an nginx directive key and value, +// and the expected file, server, and location block that it should exist in. +type ExpectedNginxField struct { + // Key is the directive name. + Key string + // Value is the value for the directive. Can be the full value or a substring. + Value string + // File is the file name that should contain the directive. Can be a full filename or a substring. + File string + // Location is the location name that the directive should exist in. + Location string + // Servers are the server names that the directive should exist in. + Servers []string + // ValueSubstringAllowed allows the expected value to be a substring of the real value. + // This makes it easier for cases when real values are complex file names or contain things we + // don't care about, and we just want to check if a substring exists. + ValueSubstringAllowed bool +} + +// ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field, +// and returns whether or not that field exists where it should. +func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) bool { + for _, config := range conf.Config { + if !strings.Contains(config.File, expFieldCfg.File) { + continue + } + + for _, directive := range config.Parsed { + if len(expFieldCfg.Servers) == 0 { + if expFieldCfg.fieldFound(directive) { + return true + } + continue + } + + for _, serverName := range expFieldCfg.Servers { + if directive.Directive == "server" && getServerName(directive.Block) == serverName { + for _, serverDirective := range directive.Block { + if expFieldCfg.Location == "" && expFieldCfg.fieldFound(serverDirective) { + return true + } else if serverDirective.Directive == "location" && + fieldExistsInLocation(serverDirective, expFieldCfg) { + return true + } + } + } + } + } + } + + return false +} + +func getServerName(serverBlock Directives) string { + for _, directive := range serverBlock { + if directive.Directive == "server_name" { + return directive.Args[0] + } + } + + return "" +} + +func (e ExpectedNginxField) fieldFound(directive *Directive) bool { + arg := strings.Join(directive.Args, " ") + + valueMatch := arg == e.Value + if e.ValueSubstringAllowed { + valueMatch = strings.Contains(arg, e.Value) + } + + return directive.Directive == e.Key && valueMatch +} + +func fieldExistsInLocation(serverDirective *Directive, expFieldCfg ExpectedNginxField) bool { + // location could start with '=', so get the last element which is the path + loc := serverDirective.Args[len(serverDirective.Args)-1] + if loc == expFieldCfg.Location { + for _, locDirective := range serverDirective.Block { + if expFieldCfg.fieldFound(locDirective) { + return true + } + } + } + + return false +} + +// injectCrossplaneContainer adds an ephemeral container that contains crossplane for parsing +// nginx config. It attaches to the nginx container and shares volumes with it. +func injectCrossplaneContainer( + k8sClient kubernetes.Interface, + timeout time.Duration, + ngfPodName, + namespace string, +) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + pod := &core.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: ngfPodName, + Namespace: namespace, + }, + Spec: core.PodSpec{ + EphemeralContainers: []core.EphemeralContainer{ + { + TargetContainerName: "nginx", + EphemeralContainerCommon: core.EphemeralContainerCommon{ + Name: "crossplane", + Image: "nginx-crossplane:latest", + ImagePullPolicy: "Never", + Stdin: true, + VolumeMounts: []core.VolumeMount{ + { + MountPath: "/etc/nginx/conf.d", + Name: "nginx-conf", + }, + { + MountPath: "/etc/nginx/stream-conf.d", + Name: "nginx-stream-conf", + }, + { + MountPath: "/etc/nginx/module-includes", + Name: "module-includes", + }, + { + MountPath: "/etc/nginx/secrets", + Name: "nginx-secrets", + }, + { + MountPath: "/etc/nginx/includes", + Name: "nginx-includes", + }, + }, + }, + }, + }, + }, + } + + podClient := k8sClient.CoreV1().Pods(namespace) + if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("error adding ephemeral container: %w", err) + } + + return nil +} + +// createCrossplaneExecutor creates the executor for the crossplane command. +func createCrossplaneExecutor( + k8sClient kubernetes.Interface, + k8sConfig *rest.Config, + ngfPodName, + namespace string, +) (remotecommand.Executor, error) { + cmd := []string{"crossplane", "parse", "/etc/nginx/nginx.conf"} + opts := &core.PodExecOptions{ + Command: cmd, + Container: "crossplane", + Stdout: true, + Stderr: true, + } + + req := k8sClient.CoreV1().RESTClient().Post(). + Resource("pods"). + SubResource("exec"). + Name(ngfPodName). + Namespace(namespace). + VersionedParams(opts, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL()) + if err != nil { + return nil, fmt.Errorf("error creating executor: %w", err) + } + + return exec, nil +} + +// The following types are copied from https://github.com/nginxinc/nginx-go-crossplane, +// with unnecessary fields stripped out. +type Payload struct { + Config []Config `json:"config"` +} + +type Config struct { + File string `json:"file"` + Parsed Directives `json:"parsed"` +} + +type Directive struct { + Comment *string `json:"comment,omitempty"` + Directive string `json:"directive"` + File string `json:"file,omitempty"` + Args []string `json:"args"` + Includes []int `json:"includes,omitempty"` + Block Directives `json:"block,omitempty"` + Line int `json:"line"` +} + +type Directives []*Directive diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index 4ce3fec712..841e553ae7 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -23,6 +23,7 @@ import ( "bytes" "context" "embed" + "encoding/json" "errors" "fmt" "io" @@ -31,8 +32,6 @@ import ( "strings" "time" - "k8s.io/client-go/util/retry" - apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,6 +41,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" v1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -50,6 +52,7 @@ import ( type ResourceManager struct { K8sClient client.Client ClientGoClient kubernetes.Interface // used when k8sClient is not enough + K8sConfig *rest.Config FS embed.FS TimeoutConfig TimeoutConfig } @@ -809,3 +812,55 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( }, ) } + +// GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON. +func (rm *ResourceManager) GetNginxConfig(ngfPodName, namespace string) (*Payload, error) { + if err := injectCrossplaneContainer( + rm.ClientGoClient, + rm.TimeoutConfig.UpdateTimeout, + ngfPodName, + namespace, + ); err != nil { + return nil, err + } + + exec, err := createCrossplaneExecutor(rm.ClientGoClient, rm.K8sConfig, ngfPodName, namespace) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.RequestTimeout) + defer cancel() + + buf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + if err := wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: buf, + Stderr: errBuf, + }); err != nil { + return false, nil //nolint:nilerr // we want to retry if there's an error + } + + if errBuf.String() != "" { + return false, nil + } + + return true, nil + }, + ); err != nil { + return nil, fmt.Errorf("could not connect to ephemeral container: %w", err) + } + + conf := &Payload{} + if err := json.Unmarshal(buf.Bytes(), conf); err != nil { + return nil, fmt.Errorf("error unmarshaling nginx config: %w", err) + } + + return conf, nil +} diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh deleted file mode 100755 index a15ea5cbc2..0000000000 --- a/tests/scripts/remote-scripts/run-tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source "${HOME}"/vars.env - -cd nginx-gateway-fabric/tests && make test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 77dc6f4761..42a6d85c45 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -6,18 +6,13 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) source scripts/vars.env -SCRIPT=run-tests.sh -if [ "${NFR}" = "true" ]; then - SCRIPT=run-nfr-tests.sh -fi - gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~ gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ --command="export START_LONGEVITY=${START_LONGEVITY} &&\ export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ export CI=${CI} &&\ - bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT} + bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-nfr-tests.sh retcode=$? if [ ${retcode} -ne 0 ]; then @@ -25,14 +20,12 @@ if [ ${retcode} -ne 0 ]; then exit 1 fi -if [ "${NFR}" = "true" ]; then - ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config - if [ "${CI}" = "false" ]; then - gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null - rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . - else - gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . - fi +## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config +if [ "${CI}" = "false" ]; then + gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null + rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . +else + gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . fi ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index baffadb0ef..706d2651d8 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -91,9 +91,130 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" } }) + Context("nginx config", func() { + var conf *framework.Payload + + BeforeAll(func() { + podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(podNames).To(HaveLen(1)) + + ngfPodName := podNames[0] + + conf, err = resourceManager.GetNginxConfig(ngfPodName, ngfNamespace) + Expect(err).ToNot(HaveOccurred()) + }) + + DescribeTable("is set properly for", + func(expCfgs []framework.ExpectedNginxField) { + for _, expCfg := range expCfgs { + failureMsg := fmt.Sprintf( + "directive '%s' with value '%s' not found in expected place", + expCfg.Key, expCfg.Value, + ) + Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(BeTrue(), failureMsg) + } + }, + Entry("gateway policy", []framework.ExpectedNginxField{ + { + Key: "include", + Value: "gw-csp.conf", + ValueSubstringAllowed: true, + File: "http.conf", + Servers: []string{"*.example.com", "cafe.example.com"}, + }, + { + Key: "client_max_body_size", + Value: "1000", + File: "gw-csp.conf", + }, + { + Key: "client_body_timeout", + Value: "30s", + File: "gw-csp.conf", + }, + { + Key: "keepalive_requests", + Value: "100", + File: "gw-csp.conf", + }, + { + Key: "keepalive_time", + Value: "5s", + File: "gw-csp.conf", + }, + { + Key: "keepalive_timeout", + Value: "2s 1s", + File: "gw-csp.conf", + }, + }), + Entry("coffee route policy", []framework.ExpectedNginxField{ + { + Key: "include", + Value: "coffee-route-csp.conf", + ValueSubstringAllowed: true, + File: "http.conf", + Servers: []string{"cafe.example.com"}, + Location: "/coffee", + }, + { + Key: "client_max_body_size", + Value: "2000", + File: "coffee-route-csp.conf", + }, + }), + Entry("tea route policy", []framework.ExpectedNginxField{ + { + Key: "include", + Value: "tea-route-csp.conf", + ValueSubstringAllowed: true, + File: "http.conf", + Servers: []string{"cafe.example.com"}, + Location: "/tea", + }, + { + Key: "keepalive_requests", + Value: "200", + File: "tea-route-csp.conf", + }, + }), + Entry("soda route policy", []framework.ExpectedNginxField{ + { + Key: "include", + Value: "soda-route-csp.conf", + ValueSubstringAllowed: true, + File: "http.conf", + Servers: []string{"cafe.example.com"}, + Location: "/soda", + }, + { + Key: "client_max_body_size", + Value: "3000", + File: "soda-route-csp.conf", + }, + }), + Entry("grpc route policy", []framework.ExpectedNginxField{ + { + Key: "include", + Value: "grpc-route-csp.conf", + ValueSubstringAllowed: true, + File: "http.conf", + Servers: []string{"*.example.com"}, + Location: "/helloworld.Greeter/SayHello", + }, + { + Key: "client_max_body_size", + Value: "0", + File: "grpc-route-csp.conf", + }, + }), + ) + }) + // We only test that the client_max_body_size directive in this test is propagated correctly. // This is because we can easily verify this directive by sending requests with different sized payloads. - DescribeTable("the settings are propagated to the nginx config", + DescribeTable("client_max_body_size requests work as expected", func(uri string, byteLengthOfRequestBody, expStatus int) { url := baseURL + uri diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 524898ffd0..3fa10e0325 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -127,6 +127,7 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { resourceManager = framework.ResourceManager{ K8sClient: k8sClient, ClientGoClient: clientGoClient, + K8sConfig: k8sConfig, FS: manifests, TimeoutConfig: timeoutConfig, }