Watch UpstreamSettingsPolicies and translate into dataplane configura…

…tion (#2887) Problem: As a userI want NGF to take my configuration for an UpstreamSettingsPolicy and transform it into data plane configuration within NGF, so that NGF can then translate those settings into NGINX configuration, and so that NGF maintains an abstraction layer between data plane configuration and the specific data plane NGF uses. Solution: Add controller to watch UpstreamSettingsPolicies, and store them in the cluster state as generic NGF Policies. Update the graph to validate and process these policies and attach them to the relevant Services. When building the dataplane configuration, store the policies on the relevant http upstreams.
nginx · Dec 13, 2024 · b968639 · b968639
1 parent eabc8c9
commit b968639
Show file tree

Hide file tree

Showing 38 changed files with 1,883 additions and 238 deletions.
diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml
@@ -104,6 +104,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   {{- if .Values.nginxGateway.snippetsFilters.enable }}
   - snippetsfilters
   {{- end }}
@@ -116,6 +117,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   {{- if .Values.nginxGateway.snippetsFilters.enable }}
   - snippetsfilters/status
   {{- end }}

diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml
@@ -5,3 +5,5 @@ resources:
   - bases/gateway.nginx.org_nginxgateways.yaml
   - bases/gateway.nginx.org_nginxproxies.yaml
   - bases/gateway.nginx.org_observabilitypolicies.yaml
+  - bases/gateway.nginx.org_snippetsfilters.yaml
+  - bases/gateway.nginx.org_upstreamsettingspolicies.yaml
diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml
@@ -98,6 +98,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -107,6 +108,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml
@@ -98,6 +98,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -107,6 +108,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/crds.yaml b/deploy/crds.yaml
diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml
@@ -98,6 +98,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -107,6 +108,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml
@@ -111,6 +111,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -120,6 +121,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml
@@ -103,6 +103,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -112,6 +113,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml
@@ -106,6 +106,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -115,6 +116,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml
@@ -98,6 +98,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -107,6 +108,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml
@@ -98,6 +98,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   verbs:
   - list
   - watch
@@ -107,6 +108,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   verbs:
   - update
 - apiGroups:

diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml
@@ -106,6 +106,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   - snippetsfilters
   verbs:
   - list
@@ -116,6 +117,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   - snippetsfilters/status
   verbs:
   - update

diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml
@@ -98,6 +98,7 @@ rules:
   - nginxproxies
   - clientsettingspolicies
   - observabilitypolicies
+  - upstreamsettingspolicies
   - snippetsfilters
   verbs:
   - list
@@ -108,6 +109,7 @@ rules:
   - nginxgateways/status
   - clientsettingspolicies/status
   - observabilitypolicies/status
+  - upstreamsettingspolicies/status
   - snippetsfilters/status
   verbs:
   - update

diff --git a/examples/upstream-settings-policy/cafe-routes.yaml b/examples/upstream-settings-policy/cafe-routes.yaml
@@ -0,0 +1,37 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: coffee
+spec:
+  parentRefs:
+  - name: gateway
+    sectionName: http
+  hostnames:
+  - "cafe.example.com"
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /coffee
+    backendRefs:
+    - name: coffee
+      port: 80
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: tea
+spec:
+  parentRefs:
+  - name: gateway
+    sectionName: http
+  hostnames:
+  - "cafe.example.com"
+  rules:
+  - matches:
+    - path:
+        type: Exact
+        value: /tea
+    backendRefs:
+    - name: tea
+      port: 80
diff --git a/examples/upstream-settings-policy/cafe.yaml b/examples/upstream-settings-policy/cafe.yaml
@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coffee
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: coffee
+  template:
+    metadata:
+      labels:
+        app: coffee
+    spec:
+      containers:
+      - name: coffee
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: coffee
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: coffee
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tea
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: tea
+  template:
+    metadata:
+      labels:
+        app: tea
+    spec:
+      containers:
+      - name: tea
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tea
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: tea
diff --git a/examples/upstream-settings-policy/gateway.yaml b/examples/upstream-settings-policy/gateway.yaml
@@ -0,0 +1,11 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: gateway
+spec:
+  gatewayClassName: nginx
+  listeners:
+  - name: http
+    port: 80
+    protocol: HTTP
+    hostname: "*.example.com"
diff --git a/examples/upstream-settings-policy/upstream-settings-policy.yaml b/examples/upstream-settings-policy/upstream-settings-policy.yaml
@@ -7,7 +7,7 @@ spec:
   targetRefs:
     - group: core
       kind: Service
-      name: service
+      name: coffee
   keepAlive:
     connections: 32
     requests: 1001

diff --git a/internal/framework/kinds/kinds.go b/internal/framework/kinds/kinds.go
@@ -11,9 +11,9 @@ import (
 
 // Gateway API Kinds.
 const (
-	// Gateway is the Gateway Kind.
+	// Gateway is the Gateway kind.
 	Gateway = "Gateway"
-	// GatewayClass is the GatewayClass Kind.
+	// GatewayClass is the GatewayClass kind.
 	GatewayClass = "GatewayClass"
 	// HTTPRoute is the HTTPRoute kind.
 	HTTPRoute = "HTTPRoute"
@@ -23,6 +23,12 @@ const (
 	TLSRoute = "TLSRoute"
 )
 
+// Core API Kinds.
+const (
+	// Service is the Service kind.
+	Service = "Service"
+)
+
 // NGINX Gateway Fabric kinds.
 const (
 	// ClientSettingsPolicy is the ClientSettingsPolicy kind.
@@ -33,6 +39,8 @@ const (
 	NginxProxy = "NginxProxy"
 	// SnippetsFilter is the SnippetsFilter kind.
 	SnippetsFilter = "SnippetsFilter"
+	// UpstreamSettingsPolicy is the UpstreamSettingsPolicy kind.
+	UpstreamSettingsPolicy = "UpstreamSettingsPolicy"
 )
 
 // MustExtractGVK is a function that extracts the GroupVersionKind (GVK) of a client.object.

diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go
@@ -52,6 +52,7 @@ import (
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/clientsettings"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/observability"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/upstreamsettings"
 	ngxvalidation "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/validation"
 	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/file"
 	ngxruntime "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/runtime"
@@ -311,6 +312,10 @@ func createPolicyManager(
 			GVK:       mustExtractGVK(&ngfAPI.ObservabilityPolicy{}),
 			Validator: observability.NewValidator(validator),
 		},
+		{
+			GVK:       mustExtractGVK(&ngfAPI.UpstreamSettingsPolicy{}),
+			Validator: upstreamsettings.NewValidator(validator),
+		},
 	}
 
 	return policies.NewManager(mustExtractGVK, cfgs...)
@@ -492,6 +497,12 @@ func registerControllers(
 				controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
 			},
 		},
+		{
+			objectType: &ngfAPI.UpstreamSettingsPolicy{},
+			options: []controller.Option{
+				controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+			},
+		},
 	}
 
 	if cfg.ExperimentalFeatures {
@@ -728,6 +739,7 @@ func prepareFirstEventBatchPreparerArgs(cfg config.Config) ([]client.Object, []c
 		&gatewayv1.GRPCRouteList{},
 		&ngfAPI.ClientSettingsPolicyList{},
 		&ngfAPI.ObservabilityPolicyList{},
+		&ngfAPI.UpstreamSettingsPolicyList{},
 		partialObjectMetadataList,
 	}