diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a690a47c7b..72aaf10056 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -131,7 +131,7 @@ jobs: DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - name: Build Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || '' }} context: "." diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 52a80044d4..cd89ca6c42 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -88,7 +88,7 @@ jobs: TELEMETRY_ENDPOINT_INSECURE: "false" - name: Build NGF Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile tags: ${{ steps.ngf-meta.outputs.tags }} @@ -99,7 +99,7 @@ jobs: pull: true - name: Build NGINX Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} tags: ${{ steps.nginx-meta.outputs.tags }} @@ -118,7 +118,7 @@ jobs: working-directory: ./tests - name: Build Test Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: tests/conformance/Dockerfile tags: conformance-test-runner:${{ github.sha }} diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 072c217783..e571e4f118 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -76,7 +76,7 @@ jobs: TELEMETRY_ENDPOINT_INSECURE: "true" - name: Build NGF Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile tags: ${{ steps.ngf-meta.outputs.tags }} @@ -87,7 +87,7 @@ jobs: target: goreleaser - name: Build NGINX Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} tags: ${{ steps.nginx-meta.outputs.tags }} @@ -118,19 +118,19 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true working-directory: ./tests - name: Run functional graceful-recovery tests run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests - name: Run functional tests run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true working-directory: ./tests diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 37654823e6..7cfb740637 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -60,7 +60,7 @@ jobs: type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - name: Build NGF Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile tags: ${{ steps.ngf-meta.outputs.tags }} @@ -71,7 +71,7 @@ jobs: pull: true - name: Build NGINX Docker Image - uses: docker/build-push-action@4f7cdeb0f05278b464e71357394bf2c61f94138e # v6.6.0 + uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755 # v6.6.1 with: file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} tags: ${{ steps.nginx-meta.outputs.tags }} diff --git a/Makefile b/Makefile index 231f2af933..2f1cf7fdb0 100644 --- a/Makefile +++ b/Makefile @@ -45,8 +45,8 @@ endif .PHONY: help help: Makefile ## Display this help - @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' - @grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' .PHONY: build-prod-images build-prod-images: build-prod-ngf-image build-prod-nginx-image ## Build the NGF and nginx docker images for production diff --git a/tests/Makefile b/tests/Makefile index 9ee6b307c5..87e5379fa7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,10 +1,11 @@ -CI=false +CI ?= false CLUSTER_NAME ?= kind CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image CONFORMANCE_TAG = latest## Tag for the conformance test runner image GATEWAY_CLASS = nginx## Gateway class to use GINKGO_FLAGS = GINKGO_LABEL = +GITHUB_OUTPUT = GW_API_VERSION ?= $(shell sed -n 's/.*ref=v\(.*\)/\1/p' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF GW_API_PREV_VERSION ?= 1.1.0## Supported Gateway API version from previous NGF release GW_SERVICE_TYPE = NodePort## Service type to use for the gateway @@ -15,7 +16,11 @@ PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml SUPPORTED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,GRPCExactMethodMatching,GRPCRouteListenerHostnameMatching,GRPCRouteHeaderMatching ifneq ($(GINKGO_LABEL),) - override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)" + override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)" +endif + +ifeq ($(CI),true) + GITHUB_OUTPUT := --github-output endif .PHONY: update-go-modules @@ -33,9 +38,9 @@ run-conformance-tests: ## Run conformance tests kubectl apply -f conformance/conformance-rbac.yaml kubectl run -i conformance \ --image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \ - --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ + --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ --restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \ - --supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \ + --supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \ --report-output=output.txt; cat output.txt" | tee output.txt ./scripts/check-pod-exit-code.sh sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml @@ -65,8 +70,8 @@ reset-go-modules: ## Reset the go modules changes # Check if PLUS_ENABLED is true ifeq ($(PLUS_ENABLED),true) - # If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX - NGINX_PREFIX := $(NGINX_PLUS_PREFIX) + # If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX + NGINX_PREFIX := $(NGINX_PLUS_PREFIX) endif .PHONY: setup-gcp-and-run-tests @@ -107,10 +112,11 @@ start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in stop-longevity-test: export STOP_LONGEVITY=true stop-longevity-test: nfr-test ## Stop the longevity test and collects results + .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) - go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \ - --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ + go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ + --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ --gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ --plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ --pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \ @@ -118,7 +124,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: test test: ## Runs the functional tests on your default k8s cluster - go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \ + go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ --label-filter "functional" $(GINKGO_FLAGS) ./suite -- \ --gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \ --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ diff --git a/tests/framework/info.go b/tests/framework/info.go new file mode 100644 index 0000000000..588b728631 --- /dev/null +++ b/tests/framework/info.go @@ -0,0 +1,83 @@ +package framework + +import ( + "fmt" + "runtime/debug" + + core "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetLogs returns the logs for all containers in all pods for a release. +func GetLogs(rm ResourceManager, namespace string, releaseName string) string { + var returnLogs string + pods, err := rm.GetPods(namespace, client.MatchingLabels{ + "app.kubernetes.io/instance": releaseName, + }) + if err != nil { + return fmt.Sprintf("failed to get pods: %v", err) + } + + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + returnLogs += fmt.Sprintf("Logs for container %s:\n", container.Name) + logs, err := rm.GetPodLogs(pod.Namespace, pod.Name, &core.PodLogOptions{ + Container: container.Name, + }) + if err != nil { + returnLogs += fmt.Sprintf(" failed to get logs: %v\n", err) + continue + } + returnLogs += fmt.Sprintf(" %s\n", logs) + } + } + return returnLogs +} + +// GetEvents returns the events for a namespace. +func GetEvents(rm ResourceManager, namespace string) string { + var returnEvents string + events, err := rm.GetEvents(namespace) + if err != nil { + return fmt.Sprintf("failed to get events: %v", err) + } + + eventGroups := make(map[string][]core.Event) + for _, event := range events.Items { + eventGroups[event.InvolvedObject.Name] = append(eventGroups[event.InvolvedObject.Name], event) + } + + for name, events := range eventGroups { + returnEvents += fmt.Sprintf("Events for %s:\n", name) + for _, event := range events { + returnEvents += fmt.Sprintf(" %s\n", event.Message) + } + returnEvents += "\n" + } + return returnEvents +} + +// GetBuildInfo returns the build information. +func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { + commitHash = "unknown" + commitTime = "unknown" + dirtyBuild = "unknown" + + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + for _, kv := range info.Settings { + switch kv.Key { + case "vcs.revision": + commitHash = kv.Value + case "vcs.time": + commitTime = kv.Value + case "vcs.modified": + dirtyBuild = kv.Value + } + } + + return +} diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index cf9e6d3fc4..e7a585cf79 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -622,6 +622,23 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app return &deployment, nil } +// GetEvents returns all Events in the specified namespace. +func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) + defer cancel() + + var eventList core.EventList + if err := rm.K8sClient.List( + ctx, + &eventList, + client.InNamespace(namespace), + ); err != nil { + return &core.EventList{}, fmt.Errorf("error getting list of Events: %w", err) + } + + return &eventList, nil +} + // ScaleDeployment scales the Deployment to the specified number of replicas. func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int32) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout) diff --git a/tests/framework/results.go b/tests/framework/results.go index ac65569d11..79bc594c10 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -49,10 +49,13 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error { if ci.IsGKE { clusterType = "GKE" } + + commit, date, dirty := GetBuildInfo() + + //nolint:lll text := fmt.Sprintf( - //nolint:lll - "# Results\n\n## Test environment\n\nNGINX Plus: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n", - plus, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, + "# Results\n\n## Test environment\n\nNGINX Plus: %v\n\nNGINX Gateway Fabric:\n\n- Commit: %s\n- Date: %s\n- Dirty: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n", + plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, ) if _, err := fmt.Fprint(file, text); err != nil { return err diff --git a/tests/scripts/cleanup-router.sh b/tests/scripts/cleanup-router.sh index fe7c215a65..ee3ea524b2 100755 --- a/tests/scripts/cleanup-router.sh +++ b/tests/scripts/cleanup-router.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eo pipefail +set -o pipefail source scripts/vars.env diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index b98589e463..ecb0420d18 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eo pipefail +set -o pipefail source scripts/vars.env diff --git a/tests/scripts/remote-scripts/run-nfr-tests.sh b/tests/scripts/remote-scripts/run-nfr-tests.sh index d2db339478..89eac20e1b 100755 --- a/tests/scripts/remote-scripts/run-nfr-tests.sh +++ b/tests/scripts/remote-scripts/run-nfr-tests.sh @@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then GINKGO_LABEL="longevity-teardown" fi -cd nginx-gateway-fabric/tests && make .vm-nfr-test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" +cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" if [ "${START_LONGEVITY}" == "true" ]; then suite/scripts/longevity-wrk.sh diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh index db681d2e32..a15ea5cbc2 100755 --- a/tests/scripts/remote-scripts/run-tests.sh +++ b/tests/scripts/remote-scripts/run-tests.sh @@ -4,4 +4,4 @@ set -e source "${HOME}"/vars.env -cd nginx-gateway-fabric/tests && make test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" +cd nginx-gateway-fabric/tests && make test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 91c30b0ec1..77dc6f4761 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -16,6 +16,7 @@ gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SC gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ --command="export START_LONGEVITY=${START_LONGEVITY} &&\ export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ + export CI=${CI} &&\ bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT} retcode=$? diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 26ef6fc4f8..baffadb0ef 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "bytes" diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index a1c3a6fb4f..703dbc3a21 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index a9edbbe2d8..b7360175c1 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "context" diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index d42ebc8b26..4054ddb99c 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index d573c5903c..62d482edb2 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index e5737ad210..053d575d26 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "bytes" @@ -130,39 +130,39 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { } const scaleResultTemplate = ` - ## Test {{ .Name }} +## Test {{ .Name }} - ### Reloads +### Reloads - - Total: {{ .ReloadCount }} - - Total Errors: {{ .ReloadErrsCount }} - - Average Time: {{ .ReloadAvgTime }}ms - - Reload distribution: - {{- range .ReloadBuckets }} - - {{ .Le }}ms: {{ .Val }} - {{- end }} +- Total: {{ .ReloadCount }} +- Total Errors: {{ .ReloadErrsCount }} +- Average Time: {{ .ReloadAvgTime }}ms +- Reload distribution: +{{- range .ReloadBuckets }} + - {{ .Le }}ms: {{ .Val }} +{{- end }} - ### Event Batch Processing +### Event Batch Processing - - Total: {{ .EventsCount }} - - Average Time: {{ .EventsAvgTime }}ms - - Event Batch Processing distribution: - {{- range .EventsBuckets }} - - {{ .Le }}ms: {{ .Val }} - {{- end }} +- Total: {{ .EventsCount }} +- Average Time: {{ .EventsAvgTime }}ms +- Event Batch Processing distribution: +{{- range .EventsBuckets }} + - {{ .Le }}ms: {{ .Val }} +{{- end }} - ### Errors +### Errors - - NGF errors: {{ .NGFErrors }} - - NGF container restarts: {{ .NGFContainerRestarts }} - - NGINX errors: {{ .NginxErrors }} - - NGINX container restarts: {{ .NginxContainerRestarts }} +- NGF errors: {{ .NGFErrors }} +- NGF container restarts: {{ .NGFContainerRestarts }} +- NGINX errors: {{ .NginxErrors }} +- NGINX container restarts: {{ .NginxContainerRestarts }} - ### Graphs and Logs +### Graphs and Logs - See [output directory](./{{ .Name }}) for more details. - The logs are attached only if there are errors. - ` +See [output directory](./{{ .Name }}) for more details. +The logs are attached only if there are errors. +` writeScaleResults := func(dest io.Writer, results scaleTestResults) error { tmpl, err := template.New("results").Parse(scaleResultTemplate) @@ -905,7 +905,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim Expect(framework.WriteMetricsResults(outFile, res.metrics)).To(Succeed()) - link := fmt.Sprintf("\n\n![%[1]v.png](%[1]v.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme)) + link := fmt.Sprintf("\n\n![%[1]v-oss.png](%[1]v-oss.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme)) if *plusEnabled { link = fmt.Sprintf("\n\n![%[1]v-plus.png](%[1]v-plus.png)\n", fmt.Sprintf("%s-%s", testFileNamePrefix, res.scheme)) } diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index f6913935ac..f0f3aa6993 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -1,4 +1,6 @@ -package suite +// This package needs to be named main to get build info +// because of https://github.com/golang/go/issues/33976 +package main import ( "context" @@ -21,7 +23,7 @@ import ( apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" k8sRuntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" + k8sTypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ctlr "sigs.k8s.io/controller-runtime" @@ -76,6 +78,7 @@ var ( chartVersion string clusterInfo framework.ClusterInfo skipNFRTests bool + logs string ) const ( @@ -235,7 +238,7 @@ func teardown(relName string) { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - key := types.NamespacedName{Name: ngfNamespace} + key := k8sTypes.NamespacedName{Name: ngfNamespace} if err := k8sClient.Get(ctx, key, &core.Namespace{}); err != nil && apierrors.IsNotFound(err) { return true, nil } @@ -292,6 +295,11 @@ var _ = AfterSuite(func() { if skipNFRTests { Skip("") } + events := framework.GetEvents(resourceManager, ngfNamespace) + AddReportEntry("Events", events, ReportEntryVisibilityNever) + + logs = framework.GetLogs(resourceManager, ngfNamespace, releaseName) + AddReportEntry("Logs", logs, ReportEntryVisibilityNever) labelFilter := GinkgoLabelFilter() if !strings.Contains(labelFilter, "longevity-setup") { @@ -311,3 +319,13 @@ func isNFR(labelFilter string) bool { strings.Contains(labelFilter, "upgrade") || strings.Contains(labelFilter, "scale") } + +var _ = ReportAfterSuite("Print info on failure", func(report Report) { + if !report.SuiteSucceeded { + for _, specReport := range report.SpecReports { + for _, entry := range specReport.ReportEntries { + fmt.Println(entry.GetRawValue()) + } + } + } +}) diff --git a/tests/suite/telemetry_test.go b/tests/suite/telemetry_test.go index c6a8b65fc0..3ce1ec6669 100644 --- a/tests/suite/telemetry_test.go +++ b/tests/suite/telemetry_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "fmt" diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index 9876792c18..4e4519e3c5 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "context" diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index 66c87922e8..1de1f45118 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -1,4 +1,4 @@ -package suite +package main import ( "bytes"