Refactor to allow services to specify the cacert and issuer

Also specifies a cert for libvirt and nova that contains the ip and
hostname of the control plane.

Add top level flag for TLSEnabled

Just like the control plane, we need a flag to indicate whether the
dataplane has TLS enabled.  This is because - per adoption requirements,
internal TLS may not be enabled.

The hasTLSCerts parameter at the service level is simply for developers
to specify whether or not TLS certs are needed/supported with TLS is
enabled at the top level.

api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml

@@ -1876,6 +1876,8 @@ spec:
                 items:
                   type: string
                 type: array
+              tlsEnabled:
+                type: boolean
             required:
             - nodeTemplate
             - nodes

api/bases/dataplane.openstack.org_openstackdataplaneservices.yaml

@@ -30,12 +30,18 @@ spec:
             type: object
           spec:
             properties:
+              caCerts:
+                type: string
               configMaps:
                 items:
                   type: string
                 type: array
               hasTLSCerts:
                 type: boolean
+              issuers:
+                additionalProperties:
+                  type: string
+                type: object
               label:
                 maxLength: 53
                 pattern: '[a-z]([-a-z0-9]*[a-z0-9])'

api/v1beta1/openstackdataplanenodeset_types.go

@@ -59,6 +59,11 @@ type OpenStackDataPlaneNodeSetSpec struct {
 	// +kubebuilder:default={download-cache,bootstrap,configure-network,validate-network,install-os,configure-os,run-os,ovn,neutron-metadata,libvirt,nova,telemetry}
 	// Services list
 	Services []string `json:"services"`
+
+	// TLSEnabled - Whether the node set has TLS enabled.
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
+	TLSEnabled *bool `json:"tlsEnabled,omitempty" yaml:"tlsEnabled,omitempty"`
 }
 
 //+kubebuilder:object:root=true

api/v1beta1/openstackdataplaneservice_types.go

@@ -81,6 +81,14 @@ type OpenStackDataPlaneServiceSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
 	HasTLSCerts *bool `json:"hasTLSCerts,omitempty" yaml:"hasTLSCerts,omitempty"`
+
+	// Issuers - Issuers to issue TLS Certificates
+	// +kubebuilder:validation:Optional
+	Issuers map[string]string `json:"issuers,omitempty"`
+
+	// CACerts - Secret containing the CA certificate chain
+	// +kubebuilder:validation:Optional
+	CACerts string `json:"caCerts,omitempty" yaml:"caCerts,omitempty"`
 }
 
 // OpenStackDataPlaneServiceStatus defines the observed state of OpenStackDataPlaneService

config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml

@@ -1876,6 +1876,8 @@ spec:
                 items:
                   type: string
                 type: array
+              tlsEnabled:
+                type: boolean
             required:
             - nodeTemplate
             - nodes

config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml

@@ -30,12 +30,18 @@ spec:
             type: object
           spec:
             properties:
+              caCerts:
+                type: string
               configMaps:
                 items:
                   type: string
                 type: array
               hasTLSCerts:
                 type: boolean
+              issuers:
+                additionalProperties:
+                  type: string
+                type: object
               label:
                 maxLength: 53
                 pattern: '[a-z]([-a-z0-9]*[a-z0-9])'

config/services/dataplane_v1beta1_openstackdataplaneservice_libvirt.yaml

@@ -6,3 +6,5 @@ spec:
   label: libvirt
   playbook: osp.edpm.libvirt
   hasTLSCerts: True
+  issuers:
+    default: osp-rootca-issuer-internal

config/services/dataplane_v1beta1_openstackdataplaneservice_nova.yaml

@@ -12,3 +12,5 @@ spec:
     - nova-migration-ssh-key
   playbook: osp.edpm.nova
   hasTLSCerts: True
+  issuers:
+    default: osp-rootca-issuer-internal

controllers/openstackdataplanenodeset_controller.go

@@ -210,17 +210,19 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req
 	instance.Status.CtlplaneSearchDomain = ctlplaneSearchDomain
 
 	// Issue certs for TLS for services that need them
-	for _, serviceName := range instance.Spec.Services {
-		service, err := deployment.GetService(ctx, helper, serviceName)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-		if service.Spec.HasTLSCerts != nil && *service.Spec.HasTLSCerts {
-			result, err = deployment.EnsureTLSCerts(ctx, helper, instance, allHostnames, allIPs, serviceName)
+	if instance.Spec.TLSEnabled != nil && *instance.Spec.TLSEnabled {
+		for _, serviceName := range instance.Spec.Services {
+			service, err := deployment.GetService(ctx, helper, serviceName)
 			if err != nil {
 				return ctrl.Result{}, err
-			} else if (result != ctrl.Result{}) {
-				return result, nil
+			}
+			if service.Spec.HasTLSCerts != nil && *service.Spec.HasTLSCerts {
+				result, err = deployment.EnsureTLSCerts(ctx, helper, instance, allHostnames, allIPs, service)
+				if err != nil {
+					return ctrl.Result{}, err
+				} else if (result != ctrl.Result{}) {
+					return result, nil
+				}
 			}
 		}
 	}

docs/openstack_dataplanenodeset.md

@@ -129,6 +129,7 @@ OpenStackDataPlaneNodeSetSpec defines the desired state of OpenStackDataPlaneNod
 | env | Env is a list containing the environment variables to pass to the pod | []corev1.EnvVar | false |
 | networkAttachments | NetworkAttachments is a list of NetworkAttachment resource names to pass to the ansibleee resource which allows to connect the ansibleee runner to the given network | []string | false |
 | services | Services list | []string | true |
+| tlsEnabled | TLSEnabled - Whether the node set has TLS enabled. | *bool | false |
 
 [Back to Custom Resources](#custom-resources)
 

docs/openstack_dataplaneservice.md

@@ -126,6 +126,8 @@ OpenStackDataPlaneServiceSpec defines the desired state of OpenStackDataPlaneSer
 | secrets | Secrets list of Secret names to mount as ExtraMounts for the OpenStackAnsibleEE | []string | false |
 | openStackAnsibleEERunnerImage | OpenStackAnsibleEERunnerImage image to use as the ansibleEE runner image | string | false |
 | hasTLSCerts | HasTLSCerts - Whether the nodes have TLS certs | *bool | false |
+| issuers | Issuers - Issuers to issue TLS Certificates | map[string]string | false |
+| caCerts | CACerts - Secret containing the CA certificate chain | string | false |
 
 [Back to Custom Resources](#custom-resources)
 

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/openstack-k8s-operators/dataplane-operator/api v0.0.0-20230724101130-2d6fe1f4706b
 	github.com/openstack-k8s-operators/infra-operator/apis v0.3.1-0.20231122104142-3b449040167e
 	github.com/openstack-k8s-operators/lib-common/modules/ansible v0.3.1-0.20231122111552-6bd6025ade37
-	github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231006072650-7fe7fe16bcd1
+	github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231109064837-a0ac89bc5a39
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231122111552-6bd6025ade37
 	github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20231122111552-6bd6025ade37
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20231122111552-6bd6025ade37
@@ -28,6 +28,7 @@ require (
 	k8s.io/api v0.26.11
 	k8s.io/apimachinery v0.26.11
 	k8s.io/client-go v0.26.11
+	k8s.io/utils v0.0.0-20231121161247-cf03d44ff3cf
 	sigs.k8s.io/controller-runtime v0.14.7
 )
 
@@ -86,7 +87,7 @@ require (
 	k8s.io/component-base v0.26.11 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 // indirect
-	k8s.io/utils v0.0.0-20231121161247-cf03d44ff3cf // indirect
+	sigs.k8s.io/gateway-api v0.6.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

go.sum

@@ -241,8 +241,8 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.3.1-0.20231122104142-3
 github.com/openstack-k8s-operators/infra-operator/apis v0.3.1-0.20231122104142-3b449040167e/go.mod h1:FnKU6sravC43Uj0iq2bhZaPMjoPCBhkNlVdiVoGi5/E=
 github.com/openstack-k8s-operators/lib-common/modules/ansible v0.3.1-0.20231122111552-6bd6025ade37 h1:DpfHnE6YERlwPX/zd0eaZ59uv/l6+uXThwHTaxdRPQ4=
 github.com/openstack-k8s-operators/lib-common/modules/ansible v0.3.1-0.20231122111552-6bd6025ade37/go.mod h1:A9sWNibvjr1a9B/mpy4k6J9xkH11fnn0Dx/X1EZ3On8=
-github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231006072650-7fe7fe16bcd1 h1:sE/qio/WNUEng0VBmefSr46e/cq4R83payEzge/Y48U=
-github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231006072650-7fe7fe16bcd1/go.mod h1:u1pqzqGNLcof95aqhLfU6xHVTD6ZTc5gWy2FE03UrZQ=
+github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231109064837-a0ac89bc5a39 h1:f6F22jZ6HNBdlrTBDziaoWM1HqW2LOME3nq+07SuC+s=
+github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231109064837-a0ac89bc5a39/go.mod h1:Gr8E0kTkczsoUJ1AIzj9Z5vhl6V21ZrNJXICMB527qI=
 github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231122111552-6bd6025ade37 h1:ftwUwA41Y6R8U883Saz7aDg2lqUTh2ewm0qKwLuGLcU=
 github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231122111552-6bd6025ade37/go.mod h1:/6//JWNEY68jOMoaoaSI0koL2jzpEKim3m60+jFCbqY=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20231122111552-6bd6025ade37 h1:F/sQ5+TzB1dVf4VyeyLDtcyNQDHnIkqZPK9V+cr/f6s=

pkg/deployment/cert.go

@@ -28,7 +28,9 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1"
+	infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
 	"github.com/openstack-k8s-operators/lib-common/modules/certmanager"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
@@ -38,18 +40,18 @@ import (
 // This secret will be mounted by the ansibleEE pod as an extra mount when the service is deployed.
 func EnsureTLSCerts(ctx context.Context, helper *helper.Helper,
 	instance *dataplanev1.OpenStackDataPlaneNodeSet,
-	allHostnames map[string][]string,
-	allIPs map[string][]string,
-	serviceName string) (ctrl.Result, error) {
+	allHostnames map[string]map[infranetworkv1.NetNameStr]string,
+	allIPs map[string]map[infranetworkv1.NetNameStr]string,
+	service dataplanev1.OpenStackDataPlaneService) (ctrl.Result, error) {
 
 	certsData := map[string][]byte{}
 
 	// for each node in the nodeset, issue all the TLS certs needed based on the
 	// ips or DNS Names
 	for nodeName := range instance.Spec.Nodes {
-		var dnsNames []string
+		var dnsNames map[infranetworkv1.NetNameStr]string
+		var ips map[infranetworkv1.NetNameStr]string
 		var secretName string
-		var certName string
 		var certSecret *corev1.Secret = nil
 		var err error
 		var result ctrl.Result
@@ -58,59 +60,59 @@ func EnsureTLSCerts(ctx context.Context, helper *helper.Helper,
 		// For now we just add the hostname so we can select all the certs on one node
 		labels := map[string]string{
 			"hostname": nodeName,
+			"service":  service.Name,
 		}
+		secretName = "cert-" + service.Name + "-" + nodeName
 
 		dnsNames = allHostnames[nodeName]
-		// ips = allIPs[nodeName]
-
-		switch serviceName {
+		ips = allIPs[nodeName]
+
+		switch service.Name {
+		case "nova", "libvirt":
+			// nova and libvirt want a cert with ctlplane ip and dns name
+			hosts := []string{dnsNames[CtlPlaneNetwork]}
+			ctlIPs := []string{ips[CtlPlaneNetwork]}
+			certSecret, result, err = GetTLSNodeCert(ctx, helper, instance, secretName,
+				service.Spec.Issuers["default"], labels,
+				nodeName, hosts, ctlIPs, nil)
 		default:
 			// The default case provides a cert with all the dns names for the host.
 			// This will probably be sufficient for most services.  If a service needs
 			// a different kind of cert (for example, containing ips, or using a different
 			// issuer)  then add a case for the service in this switch statement
-
-			secretName = "cert-" + nodeName
-			certSecret, _, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace)
-			if err != nil {
-				if !k8serrors.IsNotFound(err) {
-					err = fmt.Errorf("Error retrieving secret %s - %w", secretName, err)
-					return ctrl.Result{}, err
-				}
-
-				certName = secretName
-				duration := ptr.To(time.Hour * 24 * 365)
-				certSecret, result, err = certmanager.EnsureCert(ctx, helper, certmanager.RootCAIssuerInternalLabel,
-					certName, duration, dnsNames, nil, labels)
-				if err != nil {
-					return ctrl.Result{}, err
-				} else if (result != ctrl.Result{}) {
-					return result, nil
-				}
+			hosts := make([]string, 0, len(dnsNames))
+			for _, host := range dnsNames {
+				hosts = append(hosts, host)
 			}
+			secretName = "cert-default-" + nodeName
+			certSecret, result, err = GetTLSNodeCert(ctx, helper, instance, secretName,
+				certmanager.RootCAIssuerInternalLabel, labels,
+				nodeName, hosts, nil, nil)
 		}
 
+		// handle cert request errors
+		if (err != nil) || (result != ctrl.Result{}) {
+			return result, err
+		}
 		// TODO(alee) Add an owner reference to the secret so it can be monitored
 		// We'll do this once stuggi adds a function to do this in libcommon
 
 		// To use this cert, add it to the relevant service data
-		// TODO(alee) We only need the cert and key.  The cacert will come from another label
-		for key, value := range certSecret.Data {
-			certsData[nodeName+"-"+key] = value
-		}
+		certsData[nodeName+"-tls.key"] = certSecret.Data["tls.key"]
+		certsData[nodeName+"-tls.crt"] = certSecret.Data["tls.crt"]
 	}
 
 	// create a secret to hold the certs for the service
 	serviceCertsSecret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      GetServiceCertsSecretName(instance, serviceName),
+			Name:      GetServiceCertsSecretName(instance, service.Name),
 			Namespace: instance.Namespace,
 		},
 		Data: certsData,
 	}
 	_, result, err := secret.CreateOrPatchSecret(ctx, helper, instance, serviceCertsSecret)
 	if err != nil {
-		err = fmt.Errorf("Error creating certs secret for %s - %w", serviceName, err)
+		err = fmt.Errorf("Error creating certs secret for %s - %w", service.Name, err)
 		return ctrl.Result{}, err
 	} else if result != controllerutil.OperationResultNone {
 		return ctrl.Result{RequeueAfter: time.Second * 5}, nil
@@ -119,6 +121,42 @@ func EnsureTLSCerts(ctx context.Context, helper *helper.Helper,
 	return ctrl.Result{}, nil
 }
 
+// GetTLSNodeCert creates or retrieves the cert for a node for a given service
+func GetTLSNodeCert(ctx context.Context, helper *helper.Helper,
+	instance *dataplanev1.OpenStackDataPlaneNodeSet,
+	secretName string, issuer string,
+	labels map[string]string, nodeName string,
+	hostnames []string, ips []string, usages []certmgrv1.KeyUsage) (*corev1.Secret, ctrl.Result, error) {
+	certSecret, _, err := secret.GetSecret(ctx, helper, secretName, instance.Namespace)
+	var result ctrl.Result
+	if err != nil {
+		if !k8serrors.IsNotFound(err) {
+			err = fmt.Errorf("Error retrieving secret %s - %w", secretName, err)
+			return nil, ctrl.Result{}, err
+		}
+
+		duration := ptr.To(time.Hour * 24 * 365)
+		request := certmanager.CertificateRequest{
+			IssuerName:  issuer,
+			CertName:    secretName,
+			Duration:    duration,
+			Hostnames:   hostnames,
+			Ips:         ips,
+			Annotations: nil,
+			Labels:      labels,
+			Usages:      usages,
+		}
+
+		certSecret, result, err = certmanager.EnsureCert(ctx, helper, request)
+		if err != nil {
+			return nil, ctrl.Result{}, err
+		} else if (result != ctrl.Result{}) {
+			return nil, result, nil
+		}
+	}
+	return certSecret, ctrl.Result{}, nil
+}
+
 // GetServiceCertsSecretName - return name of secret to be mounted in ansibleEE which contains
 // all the TLS certs for the relevant service
 // The convention we use here is "<nodeset.name>-<service>-certs", so for example,

pkg/deployment/const.go

@@ -50,4 +50,7 @@ const (
 
 	// CertPaths base path for cert volume mount in OpenStackAnsibleEE pod
 	CertPaths = "/var/lib/openstack/certs"
+
+	// CACertPaths base path for CA cert volume mount in OpenStackAnsibleEE pod
+	CACertPaths = "/var/lib/openstack/cacerts"
 )