ext auth impl (#2537)

* use BackendObjectReference to represent the ext auth service Signed-off-by: huabing zhao <[email protected]> remove type Signed-off-by: huabing zhao <[email protected]> fix gen Signed-off-by: huabing zhao <[email protected]> fix gen Signed-off-by: huabing zhao <[email protected]> fix test Signed-off-by: huabing zhao <[email protected]> ext auth impl Signed-off-by: huabing zhao <[email protected]> fix check Signed-off-by: huabing zhao <[email protected]> add test Signed-off-by: huabing zhao <[email protected]> address comments Signed-off-by: huabing zhao <[email protected]> fix test Signed-off-by: huabing zhao <[email protected]> change to backendref Signed-off-by: huabing zhao <[email protected]> * move indexers out of controller.go Signed-off-by: huabing zhao <[email protected]> * minor changes Signed-off-by: huabing zhao <[email protected]> * Add CEL validation to BackendObjectRef Signed-off-by: huabing zhao <[email protected]> * address comments Signed-off-by: huabing zhao <[email protected]> * change backendRef to an explicit reference Signed-off-by: huabing zhao <[email protected]> * reorder filters Signed-off-by: huabing zhao <[email protected]> --------- Signed-off-by: huabing zhao <[email protected]>
envoyproxy · Feb 8, 2024 · dc4a8d3 · dc4a8d3
1 parent a33c505
commit dc4a8d3
Show file tree

Hide file tree

Showing 52 changed files with 2,885 additions and 574 deletions.
diff --git a/api/v1alpha1/ext_auth_types.go b/api/v1alpha1/ext_auth_types.go
@@ -9,8 +9,12 @@ import (
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
-// +kubebuilder:validation:XValidation:message="one of grpc or http must be specified",rule="(has(self.grpc) || has(self.http))"
-// +kubebuilder:validation:XValidation:message="only one of grpc or http can be specified",rule="(has(self.grpc) && !has(self.http)) || (!has(self.grpc) && has(self.http))"
+// +kubebuilder:validation:XValidation:rule="(has(self.grpc) || has(self.http))",message="one of grpc or http must be specified"
+// +kubebuilder:validation:XValidation:rule="(has(self.grpc) && !has(self.http)) || (!has(self.grpc) && has(self.http))",message="only one of grpc or http can be specified"
+// +kubebuilder:validation:XValidation:rule="has(self.grpc) ? (!has(self.grpc.backendRef.group) || self.grpc.backendRef.group == \"\") : true", message="group is invalid, only the core API group (specified by omitting the group field or setting it to an empty string) is supported"
+// +kubebuilder:validation:XValidation:rule="has(self.grpc) ? (!has(self.grpc.backendRef.kind) || self.grpc.backendRef.kind == 'Service') : true", message="kind is invalid, only Service (specified by omitting the kind field or setting it to 'Service') is supported"
+// +kubebuilder:validation:XValidation:rule="has(self.http) ? (!has(self.http.backendRef.group) || self.http.backendRef.group == \"\") : true", message="group is invalid, only the core API group (specified by omitting the group field or setting it to an empty string) is supported"
+// +kubebuilder:validation:XValidation:rule="has(self.http) ? (!has(self.http.backendRef.kind) || self.http.backendRef.kind == 'Service') : true", message="kind is invalid, only Service (specified by omitting the kind field or setting it to 'Service') is supported"
 //
 // ExtAuth defines the configuration for External Authorization.
 type ExtAuth struct {
@@ -42,18 +46,18 @@ type ExtAuth struct {
 // The authorization request message is defined in
 // https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto
 type GRPCExtAuthService struct {
-	// BackendObjectReference references a Kubernetes object that represents the
+	// BackendRef references a Kubernetes object that represents the
 	// backend server to which the authorization request will be sent.
 	// Only service Kind is supported for now.
-	gwapiv1.BackendObjectReference `json:",inline"`
+	BackendRef gwapiv1.BackendObjectReference `json:"backendRef"`
 }
 
 // HTTPExtAuthService defines the HTTP External Authorization service
 type HTTPExtAuthService struct {
-	// BackendObjectReference references a Kubernetes object that represents the
+	// BackendRef references a Kubernetes object that represents the
 	// backend server to which the authorization request will be sent.
 	// Only service Kind is supported for now.
-	gwapiv1.BackendObjectReference `json:",inline"`
+	BackendRef gwapiv1.BackendObjectReference `json:"backendRef"`
 
 	// Path is the path of the HTTP External Authorization service.
 	// If path is specified, the authorization request will be sent to that path,

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml
diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json
@@ -329,7 +329,8 @@
                           "filename": "/etc/ssl/certs/ca-certificates.crt"
                         }
                       }
-                    }
+                    },
+                    "sni": "raw.githubusercontent.com"
                   }
                 },
                 "type": "STRICT_DNS"
@@ -412,7 +413,7 @@
                                       "cacheDuration": "300s",
                                       "httpUri": {
                                         "cluster": "raw_githubusercontent_com_443",
-                                        "timeout": "5s",
+                                        "timeout": "10s",
                                         "uri": "https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/jwt/jwks.json"
                                       },
                                       "retryPolicy": {}

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml
@@ -191,6 +191,7 @@ xds:
                 validationContext:
                   trustedCa:
                     filename: /etc/ssl/certs/ca-certificates.crt
+              sni: raw.githubusercontent.com
           type: STRICT_DNS
     - '@type': type.googleapis.com/envoy.admin.v3.ListenersConfigDump
       dynamicListeners:
@@ -246,7 +247,7 @@ xds:
                             cacheDuration: 300s
                             httpUri:
                               cluster: raw_githubusercontent_com_443
-                              timeout: 5s
+                              timeout: 10s
                               uri: https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/jwt/jwks.json
                             retryPolicy: {}
                       requirementMap:

diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.cluster.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.cluster.yaml
@@ -51,4 +51,5 @@ xds:
               validationContext:
                 trustedCa:
                   filename: /etc/ssl/certs/ca-certificates.crt
+            sni: raw.githubusercontent.com
         type: STRICT_DNS
diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.listener.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.listener.yaml
@@ -54,7 +54,7 @@ xds:
                           cacheDuration: 300s
                           httpUri:
                             cluster: raw_githubusercontent_com_443
-                            timeout: 5s
+                            timeout: 10s
                             uri: https://raw.githubusercontent.com/envoyproxy/gateway/main/examples/kubernetes/jwt/jwks.json
                           retryPolicy: {}
                     requirementMap:

diff --git a/internal/gatewayapi/securitypolicy.go b/internal/gatewayapi/securitypolicy.go
@@ -20,6 +20,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 	gwv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gwv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -269,6 +270,7 @@ func (t *Translator) translateSecurityPolicyForRoute(
 		jwt       *ir.JWT
 		oidc      *ir.OIDC
 		basicAuth *ir.BasicAuth
+		extAuth   *ir.ExtAuth
 		err, errs error
 	)
 
@@ -292,6 +294,12 @@ func (t *Translator) translateSecurityPolicyForRoute(
 		}
 	}
 
+	if policy.Spec.ExtAuth != nil {
+		if extAuth, err = t.buildExtAuth(policy, resources); err != nil {
+			errs = errors.Join(errs, err)
+		}
+	}
+
 	// Apply IR to all relevant routes
 	// Note: there are multiple features in a security policy, even if some of them
 	// are invalid, we still want to apply the valid ones.
@@ -307,6 +315,7 @@ func (t *Translator) translateSecurityPolicyForRoute(
 					r.JWT = jwt
 					r.OIDC = oidc
 					r.BasicAuth = basicAuth
+					r.ExtAuth = extAuth
 				}
 			}
 		}
@@ -323,6 +332,7 @@ func (t *Translator) translateSecurityPolicyForGateway(
 		jwt       *ir.JWT
 		oidc      *ir.OIDC
 		basicAuth *ir.BasicAuth
+		extAuth   *ir.ExtAuth
 		err, errs error
 	)
 
@@ -346,6 +356,12 @@ func (t *Translator) translateSecurityPolicyForGateway(
 		}
 	}
 
+	if policy.Spec.ExtAuth != nil {
+		if extAuth, err = t.buildExtAuth(policy, resources); err != nil {
+			errs = errors.Join(errs, err)
+		}
+	}
+
 	// Apply IR to all the routes within the specific Gateway
 	// If the feature is already set, then skip it, since it must have be
 	// set by a policy attaching to the route
@@ -371,6 +387,9 @@ func (t *Translator) translateSecurityPolicyForGateway(
 			if r.BasicAuth == nil {
 				r.BasicAuth = basicAuth
 			}
+			if r.ExtAuth == nil {
+				r.ExtAuth = extAuth
+			}
 		}
 	}
 	return errs
@@ -626,3 +645,138 @@ func (t *Translator) buildBasicAuth(
 
 	return &ir.BasicAuth{Users: usersSecretBytes}, nil
 }
+
+func (t *Translator) buildExtAuth(
+	policy *egv1a1.SecurityPolicy,
+	resources *Resources) (*ir.ExtAuth, error) {
+	var (
+		http       = policy.Spec.ExtAuth.HTTP
+		grpc       = policy.Spec.ExtAuth.GRPC
+		backendRef *gwapiv1.BackendObjectReference
+		protocol   ir.AppProtocol
+		ds         *ir.DestinationSetting
+		authority  string
+		err        error
+	)
+
+	switch {
+	// These are sanity checks, they should never happen because the API server
+	// should have caught them
+	case http == nil && grpc == nil:
+		return nil, errors.New("one of grpc or http must be specified")
+	case http != nil && grpc != nil:
+		return nil, errors.New("only one of grpc or http can be specified")
+	case http != nil:
+		backendRef = &http.BackendRef
+		protocol = ir.HTTP
+	case grpc != nil:
+		backendRef = &grpc.BackendRef
+		protocol = ir.GRPC
+	}
+
+	if err = t.validateExtServiceBackendReference(
+		backendRef,
+		policy.Namespace,
+		resources); err != nil {
+		return nil, err
+	}
+	authority = fmt.Sprintf(
+		"%s.%s:%d",
+		backendRef.Name,
+		NamespaceDerefOr(backendRef.Namespace, policy.Namespace),
+		*backendRef.Port)
+
+	if ds, err = t.processExtServiceDestination(
+		backendRef,
+		policy.Namespace,
+		protocol,
+		resources); err != nil {
+		return nil, err
+	}
+	rd := ir.RouteDestination{
+		Name:     irExtServiceDestinationName(policy, string(backendRef.Name)),
+		Settings: []*ir.DestinationSetting{ds},
+	}
+
+	extAuth := &ir.ExtAuth{
+		HeadersToExtAuth: policy.Spec.ExtAuth.HeadersToExtAuth,
+	}
+
+	if http != nil {
+		extAuth.HTTP = &ir.HTTPExtAuthService{
+			Destination:      rd,
+			Authority:        authority,
+			Path:             ptr.Deref(http.Path, ""),
+			HeadersToBackend: http.HeadersToBackend,
+		}
+	} else {
+		extAuth.GRPC = &ir.GRPCExtAuthService{
+			Destination: rd,
+			Authority:   authority,
+		}
+	}
+	return extAuth, nil
+}
+
+// TODO: zhaohuabing combine this function with the one in the route translator
+func (t *Translator) processExtServiceDestination(
+	backendRef *gwapiv1.BackendObjectReference,
+	ownerNamespace string,
+	protocol ir.AppProtocol,
+	resources *Resources) (*ir.DestinationSetting, error) {
+	var (
+		endpoints   []*ir.DestinationEndpoint
+		addrType    *ir.DestinationAddressType
+		servicePort v1.ServicePort
+	)
+
+	serviceNamespace := NamespaceDerefOr(backendRef.Namespace, ownerNamespace)
+	service := resources.GetService(serviceNamespace, string(backendRef.Name))
+	for _, port := range service.Spec.Ports {
+		if port.Port == int32(*backendRef.Port) {
+			servicePort = port
+			break
+		}
+	}
+
+	if servicePort.AppProtocol != nil &&
+		*servicePort.AppProtocol == "kubernetes.io/h2c" {
+		protocol = ir.HTTP2
+	}
+
+	// Route to endpoints by default
+	if !t.EndpointRoutingDisabled {
+		endpointSlices := resources.GetEndpointSlicesForBackend(
+			serviceNamespace, string(backendRef.Name), KindService)
+		endpoints, addrType = getIREndpointsFromEndpointSlices(
+			endpointSlices, servicePort.Name, servicePort.Protocol)
+	} else {
+		// Fall back to Service ClusterIP routing
+		ep := ir.NewDestEndpoint(
+			service.Spec.ClusterIP,
+			uint32(*backendRef.Port))
+		endpoints = append(endpoints, ep)
+	}
+
+	// TODO: support mixed endpointslice address type for the same backendRef
+	if !t.EndpointRoutingDisabled && addrType != nil && *addrType == ir.MIXED {
+		return nil, errors.New(
+			"mixed endpointslice address type for the same backendRef is not supported")
+	}
+
+	return &ir.DestinationSetting{
+		Weight:      ptr.To(uint32(1)),
+		Protocol:    protocol,
+		Endpoints:   endpoints,
+		AddressType: addrType,
+	}, nil
+}
+
+func irExtServiceDestinationName(policy *egv1a1.SecurityPolicy, service string) string {
+	return strings.ToLower(fmt.Sprintf(
+		"%s/%s/%s/%s",
+		KindSecurityPolicy,
+		policy.GetNamespace(),
+		policy.GetName(),
+		service))
+}
diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-group.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-group.out.yaml
@@ -71,7 +71,7 @@ httpRoutes:
       - lastTransitionTime: null
         message: Group is invalid, only the core API group (specified by omitting
           the group field or setting it to an empty string) and multicluster.x-k8s.io
-          is supported
+          are supported
         reason: InvalidKind
         status: "False"
         type: ResolvedRefs

diff --git a/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml b/internal/gatewayapi/testdata/httproute-with-invalid-backend-ref-invalid-kind.out.yaml
@@ -68,7 +68,7 @@ httpRoutes:
         status: "True"
         type: Accepted
       - lastTransitionTime: null
-        message: Kind is invalid, only Service and MCS ServiceImport is supported
+        message: Kind is invalid, only Service and MCS ServiceImport are supported
         reason: InvalidKind
         status: "False"
         type: ResolvedRefs

diff --git a/internal/gatewayapi/testdata/securitypolicy-with-extauth-invalid-no-matching-port.in.yaml b/internal/gatewayapi/testdata/securitypolicy-with-extauth-invalid-no-matching-port.in.yaml
@@ -0,0 +1,65 @@
+gateways:
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: Gateway
+    metadata:
+      namespace: default
+      name: gateway-1
+    spec:
+      gatewayClassName: envoy-gateway-class
+      listeners:
+        - name: http
+          protocol: HTTP
+          port: 80
+          allowedRoutes:
+            namespaces:
+              from: All
+httpRoutes:
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: HTTPRoute
+    metadata:
+      namespace: default
+      name: httproute-1
+    spec:
+      hostnames:
+        - www.foo.com
+      parentRefs:
+        - namespace: default
+          name: gateway-1
+          sectionName: http
+      rules:
+        - matches:
+            - path:
+                value: /foo
+          backendRefs:
+            - name: service-1
+              port: 8080
+services:
+  - apiVersion: v1
+    kind: Service
+    metadata:
+      namespace: default
+      name: http-backend
+    spec:
+      ports:
+        - port: 8080
+securityPolicies:
+  - apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: SecurityPolicy
+    metadata:
+      namespace: default
+      name: policy-for-gateway
+    spec:
+      targetRef:
+        group: gateway.networking.k8s.io
+        kind: Gateway
+        name: gateway-1
+        namespace: default
+      extAuth:
+        http:
+          backendRef:
+            Name: http-backend
+            Namespace: default
+            Port: 80
+          headersToBackend:
+            - header1
+            - header2