Support NginxProxy CRD and global tracing settings (#1870)

Problem: As a user of NGF I want to set the collection point for my traces for my installation of NGF So that I can ensure all my traces are sent to the same collection platform. Solution: Implement the NginxProxy CRD which contains the fields required to configure the collection point for tracing. This resource is attached to the GatewayClass. If the resource is not found, a condition will be set on the GatewayClass to indicate this. The GatewayClass will continue to be Accepted even if the parametersRef is invalid. This configuration sets the `http` context-level otel directives. The otel module is loaded conditionally based on the existence of this configuration. Note: tracing is not enabled by this configuration, this only sets high level options. #1828 is required to actually enable tracing on a per-route basis.
nginxinc · Apr 29, 2024 · 7c3da8d · 7c3da8d
1 parent c7fd089
commit 7c3da8d
Show file tree

Hide file tree

Showing 47 changed files with 1,793 additions and 61 deletions.
diff --git a/build/Dockerfile.nginx b/build/Dockerfile.nginx
@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.6
-FROM nginx:1.25.5-alpine
+FROM nginx:1.25.5-alpine-otel
 
 ARG NJS_DIR
 ARG NGINX_CONF_DIR

diff --git a/build/Dockerfile.nginxplus b/build/Dockerfile.nginxplus
@@ -18,7 +18,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
     addgroup -g 1001 -S nginx \
     && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
     && printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
-    && apk add --no-cache nginx-plus nginx-plus-module-njs libcap \
+    && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap \
     && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \

diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md
@@ -224,7 +224,7 @@ To uninstall/delete the release `ngf`:
 ```shell
 helm uninstall ngf -n nginx-gateway
 kubectl delete ns nginx-gateway
-kubectl delete crd nginxgateways.gateway.nginx.org
+kubectl delete crd nginxgateways.gateway.nginx.org nginxproxies.gateway.nginx.org
 ```
 
 These commands remove all the Kubernetes components associated with the release and deletes the release.
@@ -269,6 +269,7 @@ The following tables lists the configurable parameters of the NGINX Gateway Fabr
 | `nginx.image.tag`                                 | The tag for the NGINX image.                                                                                                                                                                             | edge                                                                                                            |
 | `nginx.image.pullPolicy`                          | The `imagePullPolicy` for the NGINX image.                                                                                                                                                               | Always                                                                                                          |
 | `nginx.plus`                                      | Is NGINX Plus image being used                                                                                                                                                                           | false                                                                                                           |
+| `nginx.config`                                    | The configuration for the data plane that is contained in the NginxProxy resource. | [See nginx.config section](values.yaml) |
 | `nginx.usage.secretName`                          | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | |
 | `nginx.usage.serverURL`                           | The base server URL of the NGINX Plus usage reporting server. | |
 | `nginx.usage.clusterName`                         | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | |

diff --git a/charts/nginx-gateway-fabric/templates/_helpers.tpl b/charts/nginx-gateway-fabric/templates/_helpers.tpl
@@ -31,6 +31,14 @@ Create control plane config name.
 {{- printf "%s-config" $name | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Create data plane config name.
+*/}}
+{{- define "nginx-gateway.proxy-config-name" -}}
+{{- $name := default .Release.Name .Values.nameOverride }}
+{{- printf "%s-proxy-config" $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}

diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml
@@ -118,6 +118,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -149,6 +151,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -181,6 +185,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: module-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

diff --git a/charts/nginx-gateway-fabric/templates/gatewayclass.yaml b/charts/nginx-gateway-fabric/templates/gatewayclass.yaml
@@ -6,3 +6,9 @@ metadata:
   {{- include "nginx-gateway.labels" . | nindent 4 }}
 spec:
   controllerName: {{ .Values.nginxGateway.gatewayControllerName }}
+  {{- if .Values.nginx.config }}
+  parametersRef:
+    group: gateway.nginx.org
+    kind: NginxProxy
+    name: {{ include "nginx-gateway.proxy-config-name" . }}
+  {{- end }}
diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml
@@ -0,0 +1,10 @@
+{{- if .Values.nginx.config }}
+apiVersion: gateway.nginx.org/v1alpha1
+kind: NginxProxy
+metadata:
+  name: {{ include "nginx-gateway.proxy-config-name" . }}
+  labels:
+  {{- include "nginx-gateway.labels" . | nindent 4 }}
+spec:
+  {{- toYaml .Values.nginx.config | nindent 2 }}
+{{- end }}
diff --git a/charts/nginx-gateway-fabric/templates/rbac.yaml b/charts/nginx-gateway-fabric/templates/rbac.yaml
@@ -10,7 +10,7 @@ metadata:
   {{- if or .Values.serviceAccount.imagePullSecret .Values.serviceAccount.imagePullSecrets }}
   imagePullSecrets:
     {{- if .Values.serviceAccount.imagePullSecret }}
-    - name: {{ .Values.serviceAccount.imagePullSecret}}
+    - name: {{ .Values.serviceAccount.imagePullSecret }}
     {{- end }}
     {{- if .Values.serviceAccount.imagePullSecrets }}
     {{- range .Values.serviceAccount.imagePullSecrets }}
@@ -115,6 +115,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - gateway.nginx.org
+  resources:
+  - nginxproxies
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - gateway.nginx.org
   resources:

diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml
@@ -70,6 +70,17 @@ nginx:
   ## Is NGINX Plus image being used
   plus: false
 
+  ## The configuration for the data plane that is contained in the NginxProxy resource.
+  config: {}
+    # telemetry:
+    #   exporter:
+    #     endpoint: otel-collector.default.svc:4317
+    #     interval: 5s
+    #     batchSize: 512
+    #     batchCount: 4
+    #   serviceName: ""
+    #   spanAttributes: []
+
   ## Configuration for NGINX Plus usage reporting.
   usage:
     ## The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting.

diff --git a/conformance/provisioner/static-deployment.yaml b/conformance/provisioner/static-deployment.yaml
@@ -70,6 +70,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -94,6 +96,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -111,6 +115,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: module-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

diff --git a/deploy/manifests/nginx-gateway-experimental.yaml b/deploy/manifests/nginx-gateway-experimental.yaml
@@ -97,6 +97,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - gateway.nginx.org
+  resources:
+  - nginxproxies
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
@@ -213,6 +220,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -237,6 +246,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -254,6 +265,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: module-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/nginx-gateway.yaml
@@ -94,6 +94,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - gateway.nginx.org
+  resources:
+  - nginxproxies
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
@@ -209,6 +216,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -233,6 +242,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -250,6 +261,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: module-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

diff --git a/deploy/manifests/nginx-plus-gateway-experimental.yaml b/deploy/manifests/nginx-plus-gateway-experimental.yaml
@@ -103,6 +103,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - gateway.nginx.org
+  resources:
+  - nginxproxies
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
@@ -220,6 +227,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -244,6 +253,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -261,6 +272,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: module-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

diff --git a/deploy/manifests/nginx-plus-gateway.yaml b/deploy/manifests/nginx-plus-gateway.yaml
@@ -100,6 +100,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - gateway.nginx.org
+  resources:
+  - nginxproxies
+  verbs:
+  - list
+  - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
@@ -216,6 +223,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -240,6 +249,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: module-includes
+          mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
           mountPath: /etc/nginx/secrets
         - name: nginx-run
@@ -257,6 +268,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: module-includes
+        emptyDir: {}
       - name: nginx-secrets
         emptyDir: {}
       - name: nginx-run

diff --git a/docs/proposals/gateway-settings.md b/docs/proposals/gateway-settings.md
@@ -1,7 +1,7 @@
 # Enhancement Proposal-1775: Gateway Settings
 
 - Issue: https://github.com/nginxinc/nginx-gateway-fabric/issues/1775
-- Status: Implementable
+- Status: Completed
 
 ## Summary
 
@@ -93,7 +93,7 @@ type Telemetry struct {
     // SpanAttributes are custom key/value attributes that are added to each span.
     //
     // +optional
-    SpanAttributes map[string]string `json:"spanAttributes,omitempty"`
+    SpanAttributes []SpanAttribute `json:"spanAttributes,omitempty"`
 }
 
 // TelemetryExporter specifies OpenTelemetry export parameters.
@@ -122,6 +122,15 @@ type TelemetryExporter struct {
 // The format is a subset of the syntax parsed by Golang time.ParseDuration.
 // Examples: 1h, 12m, 30s, 150ms.
 type Duration string
+
+// SpanAttribute is a key value pair to be added to a tracing span.
+type SpanAttribute struct {
+	// Key is the key for a span attribute.
+	Key string `json:"key"`
+
+	// Value is the value for a span attribute.
+	Value string `json:"value"`
+}
 ```
 
 ### Status

diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go
@@ -116,6 +116,7 @@ func StartManager(cfg config.Config) error {
 		Logger:           cfg.Logger.WithName("changeProcessor"),
 		Validators: validation.Validators{
 			HTTPFieldsValidator: ngxvalidation.HTTPValidator{},
+			GenericValidator:    ngxvalidation.GenericValidator{},
 		},
 		EventRecorder:  recorder,
 		Scheme:         scheme,
@@ -414,6 +415,12 @@ func registerControllers(
 				),
 			},
 		},
+		{
+			objectType: &ngfAPI.NginxProxy{},
+			options: []controller.Option{
+				controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+			},
+		},
 	}
 
 	if cfg.ExperimentalFeatures {
@@ -592,6 +599,7 @@ func prepareFirstEventBatchPreparerArgs(
 		&discoveryV1.EndpointSliceList{},
 		&gatewayv1.HTTPRouteList{},
 		&gatewayv1beta1.ReferenceGrantList{},
+		&ngfAPI.NginxProxyList{},
 		partialObjectMetadataList,
 	}