feat: downstream mTLS (#2490)

* feat: downstream mTLS Relates to #2483 Signed-off-by: Arko Dasgupta <[email protected]> * configmap provider logic Signed-off-by: Arko Dasgupta <[email protected]> * gatewayapi translation Signed-off-by: Arko Dasgupta <[email protected]> * fix charts Signed-off-by: Arko Dasgupta <[email protected]> * tests Signed-off-by: Arko Dasgupta <[email protected]> * lint Signed-off-by: Arko Dasgupta <[email protected]> --------- Signed-off-by: Arko Dasgupta <[email protected]>
envoyproxy · Feb 14, 2024 · 765903a · 765903a
1 parent 446997b
commit 765903a
Show file tree

Hide file tree

Showing 39 changed files with 1,114 additions and 159 deletions.
diff --git a/api/v1alpha1/tls_types.go b/api/v1alpha1/tls_types.go
@@ -6,7 +6,7 @@
 package v1alpha1
 
 import (
-	corev1 "k8s.io/api/core/v1"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 // +kubebuilder:validation:XValidation:rule="has(self.minVersion) && self.minVersion == '1.3' ? !has(self.ciphers) : true", message="setting ciphers has no effect if the minimum possible TLS version is 1.3"
@@ -115,7 +115,7 @@ type ClientValidationContext struct {
 	// the Certificate Authorities that can be used
 	// as a trust anchor to validate the certificates presented by the client.
 	//
-	// A single reference to a Kubernetes ConfigMap,
+	// A single reference to a Kubernetes ConfigMap or a Kubernetes Secret,
 	// with the CA certificate in a key named `ca.crt` is currently supported.
 	//
 	// References to a resource in different namespace are invalid UNLESS there
@@ -124,5 +124,5 @@ type ClientValidationContext struct {
 	//
 	// +kubebuilder:validation:MaxItems=8
 	// +optional
-	CACertificateRefs []corev1.ObjectReference `json:"caCertificateRefs,omitempty"`
+	CACertificateRefs []gwapiv1.SecretObjectReference `json:"caCertificateRefs,omitempty"`
 }
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml
@@ -243,77 +243,56 @@ spec:
                           to Kubernetes objects that contain TLS certificates of the
                           Certificate Authorities that can be used as a trust anchor
                           to validate the certificates presented by the client. \n
-                          A single reference to a Kubernetes ConfigMap, with the CA
-                          certificate in a key named `ca.crt` is currently supported.
-                          \n References to a resource in different namespace are invalid
-                          UNLESS there is a ReferenceGrant in the target namespace
-                          that allows the certificate to be attached."
+                          A single reference to a Kubernetes ConfigMap or a Kubernetes
+                          Secret, with the CA certificate in a key named `ca.crt`
+                          is currently supported. \n References to a resource in different
+                          namespace are invalid UNLESS there is a ReferenceGrant in
+                          the target namespace that allows the certificate to be attached."
                         items:
-                          description: "ObjectReference contains enough information
-                            to let you inspect or modify the referred object. ---
-                            New uses of this type are discouraged because of difficulty
-                            describing its usage when embedded in APIs. 1. Ignored
-                            fields.  It includes many fields which are not generally
-                            honored.  For instance, ResourceVersion and FieldPath
-                            are both very rarely valid in actual usage. 2. Invalid
-                            usage help.  It is impossible to add specific help for
-                            individual usage.  In most embedded usages, there are
-                            particular restrictions like, \"must refer only to types
-                            A and B\" or \"UID not honored\" or \"name must be restricted\".
-                            Those cannot be well described when embedded. 3. Inconsistent
-                            validation.  Because the usages are different, the validation
-                            rules are different by usage, which makes it hard for
-                            users to predict what will happen. 4. The fields are both
-                            imprecise and overly precise.  Kind is not a precise mapping
-                            to a URL. This can produce ambiguity during interpretation
-                            and require a REST mapping.  In most cases, the dependency
-                            is on the group,resource tuple and the version of the
-                            actual struct is irrelevant. 5. We cannot easily change
-                            it.  Because this type is embedded in many locations,
-                            updates to this type will affect numerous schemas.  Don't
-                            make new APIs embed an underspecified API type they do
-                            not control. \n Instead of using this type, create a locally
-                            provided and used type that is well-focused on your reference.
-                            For example, ServiceReferences for admission registration:
-                            https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
-                            ."
+                          description: "SecretObjectReference identifies an API object
+                            including its namespace, defaulting to Secret. \n The
+                            API object must be valid in the cluster; the Group and
+                            Kind must be registered in the cluster for this reference
+                            to be valid. \n References to objects with invalid Group
+                            and Kind are not valid, and must be rejected by the implementation,
+                            with appropriate Conditions set on the containing object."
                           properties:
-                            apiVersion:
-                              description: API version of the referent.
-                              type: string
-                            fieldPath:
-                              description: 'If referring to a piece of an object instead
-                                of an entire object, this string should contain a
-                                valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
-                                For example, if the object reference is to a container
-                                within a pod, this would take on a value like: "spec.containers{name}"
-                                (where "name" refers to the name of the container
-                                that triggered the event) or if no container name
-                                is specified "spec.containers[2]" (container with
-                                index 2 in this pod). This syntax is chosen only to
-                                have some well-defined way of referencing a part of
-                                an object. TODO: this design is not final and this
-                                field is subject to change in the future.'
+                            group:
+                              default: ""
+                              description: Group is the group of the referent. For
+                                example, "gateway.networking.k8s.io". When unspecified
+                                or empty string, core API group is inferred.
+                              maxLength: 253
+                              pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                               type: string
                             kind:
-                              description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                              default: Secret
+                              description: Kind is kind of the referent. For example
+                                "Secret".
+                              maxLength: 63
+                              minLength: 1
+                              pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$
                               type: string
                             name:
-                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                              description: Name is the name of the referent.
+                              maxLength: 253
+                              minLength: 1
                               type: string
                             namespace:
-                              description: 'Namespace of the referent. More info:
-                                https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
-                              type: string
-                            resourceVersion:
-                              description: 'Specific resourceVersion to which this
-                                reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
-                              type: string
-                            uid:
-                              description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                              description: "Namespace is the namespace of the referenced
+                                object. When unspecified, the local namespace is inferred.
+                                \n Note that when a namespace different than the local
+                                namespace is specified, a ReferenceGrant object is
+                                required in the referent namespace to allow that namespace's
+                                owner to accept the reference. See the ReferenceGrant
+                                documentation for details. \n Support: Core"
+                              maxLength: 63
+                              minLength: 1
+                              pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
                               type: string
+                          required:
+                          - name
                           type: object
-                          x-kubernetes-map-type: atomic
                         maxItems: 8
                         type: array
                     type: object

diff --git a/charts/gateway-helm/templates/_rbac.tpl b/charts/gateway-helm/templates/_rbac.tpl
@@ -29,6 +29,7 @@ Namespaced
 apiGroups:
 - ""
 resources:
+- configmaps
 - secrets
 - services
 verbs:

diff --git a/internal/gatewayapi/clienttrafficpolicy.go b/internal/gatewayapi/clienttrafficpolicy.go
@@ -32,11 +32,12 @@ func hasSectionName(policy *egv1a1.ClientTrafficPolicy) bool {
 	return policy.Spec.TargetRef.SectionName != nil
 }
 
-func (t *Translator) ProcessClientTrafficPolicies(clientTrafficPolicies []*egv1a1.ClientTrafficPolicy,
+func (t *Translator) ProcessClientTrafficPolicies(resources *Resources,
 	gateways []*GatewayContext,
 	xdsIR XdsIRMap, infraIR InfraIRMap) []*egv1a1.ClientTrafficPolicy {
 	var res []*egv1a1.ClientTrafficPolicy
 
+	clientTrafficPolicies := resources.ClientTrafficPolicies
 	// Sort based on timestamp
 	sort.Slice(clientTrafficPolicies, func(i, j int) bool {
 		return clientTrafficPolicies[i].CreationTimestamp.Before(&(clientTrafficPolicies[j].CreationTimestamp))
@@ -93,7 +94,7 @@ func (t *Translator) ProcessClientTrafficPolicies(clientTrafficPolicies []*egv1a
 			var err error
 			for _, l := range gateway.listeners {
 				if string(l.Name) == section {
-					err = t.translateClientTrafficPolicyForListener(&policy.Spec, l, xdsIR, infraIR)
+					err = t.translateClientTrafficPolicyForListener(policy, l, xdsIR, infraIR, resources)
 					break
 				}
 			}
@@ -180,7 +181,7 @@ func (t *Translator) ProcessClientTrafficPolicies(clientTrafficPolicies []*egv1a
 					continue
 				}
 
-				err = t.translateClientTrafficPolicyForListener(&policy.Spec, l, xdsIR, infraIR)
+				err = t.translateClientTrafficPolicyForListener(policy, l, xdsIR, infraIR, resources)
 			}
 
 			if err != nil {
@@ -286,7 +287,8 @@ func resolveCTPolicyTargetRef(policy *egv1a1.ClientTrafficPolicy, gateways []*Ga
 	return gateway
 }
 
-func (t *Translator) translateClientTrafficPolicyForListener(policySpec *egv1a1.ClientTrafficPolicySpec, l *ListenerContext, xdsIR XdsIRMap, infraIR InfraIRMap) error {
+func (t *Translator) translateClientTrafficPolicyForListener(policy *egv1a1.ClientTrafficPolicy, l *ListenerContext,
+	xdsIR XdsIRMap, infraIR InfraIRMap, resources *Resources) error {
 	// Find IR
 	irKey := irStringKey(l.gateway.Namespace, l.gateway.Name)
 	// It must exist since we've already finished processing the gateways
@@ -308,27 +310,27 @@ func (t *Translator) translateClientTrafficPolicyForListener(policySpec *egv1a1.
 	// IR must exist since we're past validation
 	if httpIR != nil {
 		// Translate TCPKeepalive
-		translateListenerTCPKeepalive(policySpec.TCPKeepalive, httpIR)
+		translateListenerTCPKeepalive(policy.Spec.TCPKeepalive, httpIR)
 
 		// Translate Proxy Protocol
-		translateListenerProxyProtocol(policySpec.EnableProxyProtocol, httpIR)
+		translateListenerProxyProtocol(policy.Spec.EnableProxyProtocol, httpIR)
 
 		// Translate Client IP Detection
-		translateClientIPDetection(policySpec.ClientIPDetection, httpIR)
+		translateClientIPDetection(policy.Spec.ClientIPDetection, httpIR)
 
 		// Translate Header Settings
-		translateListenerHeaderSettings(policySpec.Headers, httpIR)
+		translateListenerHeaderSettings(policy.Spec.Headers, httpIR)
 
 		// Translate Path Settings
-		translatePathSettings(policySpec.Path, httpIR)
+		translatePathSettings(policy.Spec.Path, httpIR)
 
 		// Translate HTTP1 Settings
-		if err := translateHTTP1Settings(policySpec.HTTP1, httpIR); err != nil {
+		if err := translateHTTP1Settings(policy.Spec.HTTP1, httpIR); err != nil {
 			return err
 		}
 
 		// enable http3 if set and TLS is enabled
-		if httpIR.TLS != nil && policySpec.HTTP3 != nil {
+		if httpIR.TLS != nil && policy.Spec.HTTP3 != nil {
 			httpIR.HTTP3 = &ir.HTTP3Settings{}
 			var proxyListenerIR *ir.ProxyListener
 			for _, proxyListener := range infraIR[irKey].Proxy.Listeners {
@@ -343,7 +345,9 @@ func (t *Translator) translateClientTrafficPolicyForListener(policySpec *egv1a1.
 		}
 
 		// Translate TLS parameters
-		translateListenerTLSParameters(policySpec.TLS, httpIR)
+		if err := t.translateListenerTLSParameters(policy, httpIR, resources); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -452,13 +456,17 @@ func translateHTTP1Settings(http1Settings *egv1a1.HTTP1Settings, httpIR *ir.HTTP
 	return nil
 }
 
-func translateListenerTLSParameters(tlsParams *egv1a1.TLSSettings, httpIR *ir.HTTPListener) {
+func (t *Translator) translateListenerTLSParameters(policy *egv1a1.ClientTrafficPolicy,
+	httpIR *ir.HTTPListener, resources *Resources) error {
 	// Return if this listener isn't a TLS listener. There has to be
 	// at least one certificate defined, which would cause httpIR to
 	// have a TLS structure.
 	if httpIR.TLS == nil {
-		return
+		return nil
 	}
+
+	tlsParams := policy.Spec.TLS
+
 	// Make sure that the negotiated TLS protocol version is as expected if TLS is used,
 	// regardless of if TLS parameters were used in the ClientTrafficPolicy or not
 	httpIR.TLS.MinVersion = ptr.To(ir.TLSv12)
@@ -473,10 +481,12 @@ func translateListenerTLSParameters(tlsParams *egv1a1.TLSSettings, httpIR *ir.HT
 			httpIR.TLS.ALPNProtocols[i] = string(tlsParams.ALPNProtocols[i])
 		}
 	}
+
 	// Return early if not set
 	if tlsParams == nil {
-		return
+		return nil
 	}
+
 	if tlsParams.MinVersion != nil {
 		httpIR.TLS.MinVersion = ptr.To(ir.TLSVersion(*tlsParams.MinVersion))
 	}
@@ -492,4 +502,55 @@ func translateListenerTLSParameters(tlsParams *egv1a1.TLSSettings, httpIR *ir.HT
 	if len(tlsParams.SignatureAlgorithms) > 0 {
 		httpIR.TLS.SignatureAlgorithms = tlsParams.SignatureAlgorithms
 	}
+
+	if tlsParams.ClientValidation != nil {
+		from := crossNamespaceFrom{
+			group:     egv1a1.GroupName,
+			kind:      KindClientTrafficPolicy,
+			namespace: policy.Namespace,
+		}
+
+		irCACert := &ir.TLSCACertificate{
+			Name: irTLSCACertName(policy.Namespace, policy.Name),
+		}
+
+		for _, caCertRef := range tlsParams.ClientValidation.CACertificateRefs {
+			if caCertRef.Kind == nil || string(*caCertRef.Kind) == KindSecret { // nolint
+				secret, err := t.validateSecretRef(false, from, caCertRef, resources)
+				if err != nil {
+					return err
+				}
+
+				secretBytes, ok := secret.Data[caCertKey]
+				if !ok || len(secretBytes) == 0 {
+					return fmt.Errorf(
+						"caCertificateRef not found in secret %s", caCertRef.Name)
+				}
+
+				irCACert.Certificate = append(irCACert.Certificate, secretBytes...)
+
+			} else if string(*caCertRef.Kind) == KindConfigMap {
+				configMap, err := t.validateConfigMapRef(false, from, caCertRef, resources)
+				if err != nil {
+					return err
+				}
+
+				configMapBytes, ok := configMap.Data[caCertKey]
+				if !ok || len(configMapBytes) == 0 {
+					return fmt.Errorf(
+						"caCertificateRef not found in configMap %s", caCertRef.Name)
+				}
+
+				irCACert.Certificate = append(irCACert.Certificate, configMapBytes...)
+			} else {
+				return fmt.Errorf("unsupported caCertificateRef kind:%s", string(*caCertRef.Kind))
+			}
+		}
+
+		if len(irCACert.Certificate) > 0 {
+			httpIR.TLS.CACertificate = irCACert
+		}
+	}
+
+	return nil
 }
diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go
@@ -26,6 +26,8 @@ const (
 
 	L4Protocol = "L4"
 	L7Protocol = "L7"
+
+	caCertKey = "ca.crt"
 )
 
 type protocolPort struct {
@@ -389,7 +391,11 @@ func irTLSConfigs(tlsSecrets []*v1.Secret) *ir.TLSConfig {
 }
 
 func irTLSListenerConfigName(secret *v1.Secret) string {
-	return fmt.Sprintf("%s-%s", secret.Namespace, secret.Name)
+	return fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)
+}
+
+func irTLSCACertName(namespace, name string) string {
+	return fmt.Sprintf("%s/%s/%s", namespace, name, caCertKey)
 }
 
 func isMergeGatewaysEnabled(resources *Resources) bool {

diff --git a/internal/gatewayapi/resource.go b/internal/gatewayapi/resource.go
@@ -40,6 +40,7 @@ type Resources struct {
 	ServiceImports         []*mcsapi.ServiceImport        `json:"serviceImports,omitempty" yaml:"serviceImports,omitempty"`
 	EndpointSlices         []*discoveryv1.EndpointSlice   `json:"endpointSlices,omitempty" yaml:"endpointSlices,omitempty"`
 	Secrets                []*v1.Secret                   `json:"secrets,omitempty" yaml:"secrets,omitempty"`
+	ConfigMaps             []*v1.ConfigMap                `json:"configMaps,omitempty" yaml:"configMaps,omitempty"`
 	EnvoyProxy             *egv1a1.EnvoyProxy             `json:"envoyProxy,omitempty" yaml:"envoyProxy,omitempty"`
 	ExtensionRefFilters    []unstructured.Unstructured    `json:"extensionRefFilters,omitempty" yaml:"extensionRefFilters,omitempty"`
 	EnvoyPatchPolicies     []*egv1a1.EnvoyPatchPolicy     `json:"envoyPatchPolicies,omitempty" yaml:"envoyPatchPolicies,omitempty"`
@@ -57,6 +58,7 @@ func NewResources() *Resources {
 		Services:               []*v1.Service{},
 		EndpointSlices:         []*discoveryv1.EndpointSlice{},
 		Secrets:                []*v1.Secret{},
+		ConfigMaps:             []*v1.ConfigMap{},
 		ReferenceGrants:        []*gwapiv1b1.ReferenceGrant{},
 		Namespaces:             []*v1.Namespace{},
 		ExtensionRefFilters:    []unstructured.Unstructured{},
@@ -107,6 +109,16 @@ func (r *Resources) GetSecret(namespace, name string) *v1.Secret {
 	return nil
 }
 
+func (r *Resources) GetConfigMap(namespace, name string) *v1.ConfigMap {
+	for _, configMap := range r.ConfigMaps {
+		if configMap.Namespace == namespace && configMap.Name == name {
+			return configMap
+		}
+	}
+
+	return nil
+}
+
 func (r *Resources) GetEndpointSlicesForBackend(svcNamespace, svcName string, backendKind string) []*discoveryv1.EndpointSlice {
 	var endpointSlices []*discoveryv1.EndpointSlice
 	for _, endpointSlice := range r.EndpointSlices {