From 65f336168e6eea72221f18e8d5254a8ddb167977 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Mon, 9 Dec 2024 16:00:47 -0700 Subject: [PATCH 1/4] Use staging endpoint for n+ reporting in pipeline --- .github/workflows/conformance.yml | 3 ++- .github/workflows/functional.yml | 5 ++++- .github/workflows/helm.yml | 4 ++-- .github/workflows/nfr.yml | 2 +- Makefile | 12 ++++++++++-- docs/developer/quickstart.md | 1 + tests/Makefile | 8 ++++---- tests/framework/ngf.go | 13 ++++++++++++- tests/suite/system_suite_test.go | 16 +++++++++------- 9 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index b0b804d87c..6d68b817e5 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -139,7 +139,7 @@ jobs: - name: Setup license file for plus if: ${{ inputs.image == 'plus' }} env: - PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt - name: Setup conformance tests @@ -148,6 +148,7 @@ jobs: ngf_tag=${{ steps.ngf-meta.outputs.version }} if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi + export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag} working-directory: ./tests diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 629d24f450..2cb8f36827 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -104,7 +104,7 @@ jobs: - name: Setup license file for plus if: ${{ inputs.image == 'plus' }} env: - PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt - name: Install cloud-provider-kind @@ -125,6 +125,7 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} + export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests @@ -132,6 +133,7 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} + export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests @@ -139,5 +141,6 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} + export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 24f2391c26..a75a8c57cb 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -103,7 +103,7 @@ jobs: - name: Create plus secret if: ${{ inputs.image == 'plus' }} env: - PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: | echo "${PLUS_LICENSE}" > license.jwt kubectl create secret generic nplus-license --from-file license.jwt -n nginx-gateway @@ -155,7 +155,7 @@ jobs: - name: Create plus secrets if: ${{ inputs.image == 'plus' }} env: - PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: | echo "${PLUS_LICENSE}" > license.jwt kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 6ed48670c8..8334270bc6 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -114,7 +114,7 @@ jobs: - name: Setup license file for plus if: matrix.type == 'plus' env: - PLUS_LICENSE: ${{ secrets.JWT_PLUS_REGISTRY }} + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt - name: Create GKE cluster diff --git a/Makefile b/Makefile index ed92bef695..19d4dac4ef 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,8 @@ GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd6 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin PLUS_ENABLED ?= false PLUS_LICENSE_FILE ?= $(SELF_DIR)license.jwt +PLUS_USAGE_ENDPOINT ?=## The N+ usage endpoint. For development, please set to the N1 staging endpoint. + override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT) .DEFAULT_GOAL := help @@ -227,10 +229,16 @@ helm-install-local: install-gateway-crds ## Helm install NGF on configured kind helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway $(HELM_PARAMETERS) .PHONY: helm-install-local-with-plus -helm-install-local-with-plus: install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus. +helm-install-local-with-plus: check-for-plus-usage-endpoint install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus. kubectl create namespace nginx-gateway || true kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true - helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true $(HELM_PARAMETERS) + helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(NGINX_PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS) + +.PHONY: check-for-plus-usage-endpoint +check-for-plus-usage-endpoint: ## Checks that the PLUS_USAGE_ENDPOINT is set in the environment. This env var is required when deploying or testing with N+. +ifndef PLUS_USAGE_ENDPOINT + $(error PLUS_USAGE_ENDPOINT must be defined in your environment) +endif # Debug Targets .PHONY: debug-build diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 04549906ed..38bad3fbb6 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -122,6 +122,7 @@ This will build the docker images `nginx-gateway-fabric:` and `nginx- > Note: You will need a valid NGINX Plus license certificate and key named `nginx-repo.crt` and `nginx-repo.key` in the > root of this repo to build the NGINX Plus image. > You will also need a valid NGINX Plus JSON Web Token (JWT) to deploy NGF with NGINX Plus. That JWT should be stored in the `license.jwt` file in the root of the `nginx-gateway-fabric/` directory. See the [documentation](https://docs.nginx.com/nginx-gateway-fabric/installation/nginx-plus-jwt/) for instructions on how to download and set up the JWT. +> Additionally, you need to set the NGINX Plus usage endpoint in your environment. For development and testing, export `PLUS_USAGE_ENDPOINT=`. To build the NGINX Gateway Fabric and NGINX Plus container images from source run the following make command: diff --git a/tests/Makefile b/tests/Makefile index 3addb0843d..5182e7d161 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -109,7 +109,7 @@ sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM ./scripts/sync-files-to-vm.sh .PHONY: nfr-test -nfr-test: ## Run the NFR tests on a GCP VM +nfr-test: check-for-plus-usage-endpoint ## Run the NFR tests on a GCP VM CI=$(CI) ./scripts/run-tests-gcp-vm.sh .PHONY: start-longevity-test @@ -130,10 +130,10 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ --pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \ --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --plus-enabled=$(PLUS_ENABLED) \ - --plus-license-file-name=$(PLUS_LICENSE_FILE) + --plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT) .PHONY: test -test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster +test: check-for-plus-usage-endpoint build-crossplane-image ## Runs the functional tests on your kind k8s cluster kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME) go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \ --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ @@ -143,7 +143,7 @@ test: build-crossplane-image ## Runs the functional tests on your kind k8s clust --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ --pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \ --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --cluster-name=$(CLUSTER_NAME) --plus-enabled=$(PLUS_ENABLED) \ - --plus-license-file-name=$(PLUS_LICENSE_FILE) + --plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT) .PHONY: test-with-plus test-with-plus: PLUS_ENABLED=true diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index f19ea751b7..80c0f59abc 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -33,8 +33,9 @@ type InstallationConfig struct { ImageTag string ImagePullPolicy string ServiceType string - IsGKEInternalLB bool + PlusUsageEndpoint string Plus bool + IsGKEInternalLB bool Telemetry bool } @@ -79,6 +80,7 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { args = append(args, setImageArgs(cfg)...) args = append(args, setTelemetryArgs(cfg)...) + args = append(args, setPlusUsageEndpointArg(cfg)...) fullArgs := append(args, extraArgs...) //nolint:gocritic GinkgoWriter.Printf("Installing NGF with command: helm %v\n", strings.Join(fullArgs, " ")) @@ -238,6 +240,15 @@ func setImageArgs(cfg InstallationConfig) []string { return args } +func setPlusUsageEndpointArg(cfg InstallationConfig) []string { + var args []string + if cfg.Plus && cfg.PlusUsageEndpoint != "" { + args = append(args, formatValueSet("nginx.usage.endpoint", cfg.PlusUsageEndpoint)...) + } + + return args +} + func formatValueSet(key, value string) []string { return []string{"--set", fmt.Sprintf("%s=%s", key, value)} } diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index f4bce1b1b9..4bfb834ab8 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -62,6 +62,7 @@ var ( isGKEInternalLB = flag.Bool("is-gke-internal-lb", false, "Is the LB service GKE internal only") plusEnabled = flag.Bool("plus-enabled", false, "Is NGINX Plus enabled") plusLicenseFileName = flag.String("plus-license-file-name", "", "File name containing the NGINX Plus JWT") + plusUsageEndpoint = flag.String("plus-usage-endpoint", "", "Endpoint for reporting NGINX Plus usage") clusterName = flag.String("cluster-name", "kind", "Cluster name") ) @@ -195,13 +196,14 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framework.InstallationConfig { installCfg := framework.InstallationConfig{ - ReleaseName: cfg.releaseName, - Namespace: ngfNamespace, - ChartPath: cfg.chartPath, - ServiceType: *serviceType, - IsGKEInternalLB: *isGKEInternalLB, - Plus: *plusEnabled, - Telemetry: cfg.telemetry, + ReleaseName: cfg.releaseName, + Namespace: ngfNamespace, + ChartPath: cfg.chartPath, + ServiceType: *serviceType, + IsGKEInternalLB: *isGKEInternalLB, + Plus: *plusEnabled, + PlusUsageEndpoint: *plusUsageEndpoint, + Telemetry: cfg.telemetry, } // if we aren't installing from the public charts, then set the custom images From afeedb437474bb8a7546ff377da34cf91662e983 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 10 Dec 2024 11:49:51 -0700 Subject: [PATCH 2/4] Update tests makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 19d4dac4ef..dc88182410 100644 --- a/Makefile +++ b/Makefile @@ -222,7 +222,7 @@ load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind clus install-ngf-local-build: build-images load-images helm-install-local ## Install NGF from local build on configured kind cluster. .PHONY: install-ngf-local-build-with-plus -install-ngf-local-build-with-plus: build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster. +install-ngf-local-build-with-plus: check-for-plus-usage-endpoint build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster. .PHONY: helm-install-local helm-install-local: install-gateway-crds ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build. @@ -232,7 +232,7 @@ helm-install-local: install-gateway-crds ## Helm install NGF on configured kind helm-install-local-with-plus: check-for-plus-usage-endpoint install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus. kubectl create namespace nginx-gateway || true kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true - helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(NGINX_PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS) + helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS) .PHONY: check-for-plus-usage-endpoint check-for-plus-usage-endpoint: ## Checks that the PLUS_USAGE_ENDPOINT is set in the environment. This env var is required when deploying or testing with N+. From 792cfc73c086995cca24c19e46366929c09596a7 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 10 Dec 2024 13:15:18 -0700 Subject: [PATCH 3/4] More fixes --- tests/Makefile | 2 +- tests/suite/graceful_recovery_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 5182e7d161..bc2b54098d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -177,7 +177,7 @@ deploy-updated-provisioner: ## Update provisioner manifest and deploy to the con .PHONY: generate-static-deployment generate-static-deployment: - helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) > $(SELF_DIR)config/tests/static-deployment.yaml + helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) > $(SELF_DIR)config/tests/static-deployment.yaml # this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job) # it overrides the target in the main Makefile when the GW_API_VERSION is set to main diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index 4ec106c708..a45b23635a 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -367,7 +367,7 @@ func checkContainerLogsForErrors(ngfPodName string, checkNginxLogsOnly bool) { Expect(line).ToNot(ContainSubstring("[emerg]"), line) if strings.Contains(line, "[error]") { expectedError1 := "connect() failed (111: Connection refused)" - expectedError2 := "product.connect.nginx.com could not be resolved" + expectedError2 := "could not be resolved (host not found) during usage report" expectedError3 := "server returned 429" // FIXME(salonichf5) remove this error message check // when https://github.com/nginxinc/nginx-gateway-fabric/issues/2090 is completed. From 4e5a8894db51035433234af5db88b7324d4a0649 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 10 Dec 2024 13:24:36 -0700 Subject: [PATCH 4/4] Export endpoint for generate static deploy --- .github/workflows/conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6d68b817e5..87905c782b 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -77,6 +77,7 @@ jobs: ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi + export PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag} working-directory: ./tests