[wip] create CA bundle

controllers/core/openstackcontrolplane_controller.go

@@ -31,6 +31,7 @@ import (
 	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	corev1 "k8s.io/api/core/v1"
 
 	manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1"
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
@@ -328,6 +329,7 @@ func (r *OpenStackControlPlaneReconciler) reconcileNormal(ctx context.Context, i
 func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&corev1beta1.OpenStackControlPlane{}).
+		Owns(&corev1.Secret{}).
 		Owns(&mariadbv1.MariaDB{}).
 		Owns(&mariadbv1.Galera{}).
 		Owns(&memcachedv1.Memcached{}).

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/operator-framework/api v0.17.6
 	github.com/rabbitmq/cluster-operator/v2 v2.5.0
 	go.uber.org/zap v1.26.0
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	k8s.io/api v0.27.2
 	k8s.io/apimachinery v0.27.4
 	k8s.io/client-go v0.27.2
@@ -48,7 +49,6 @@ require (
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 // indirect
 	github.com/metal3-io/baremetal-operator/apis v0.3.1 // indirect
 	github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.2.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect
 	sigs.k8s.io/gateway-api v0.6.0 // indirect

pkg/openstack/ca.go

@@ -2,6 +2,11 @@ package openstack
 
 import (
 	"context"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"math"
+	"os"
 	"time"
 
 	certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
@@ -12,6 +17,8 @@ import (
 	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
+	"golang.org/x/exp/slices"
+	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 
 	corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1"
 
@@ -25,6 +32,10 @@ const (
 	DefaultPublicCAName = "rootca-" + string(service.EndpointPublic)
 	// DefaultInternalCAName -
 	DefaultInternalCAName = "rootca-" + string(service.EndpointInternal)
+	// TLSCABundleFile -
+	TLSCABundleFile = "tls-ca-bundle.pem"
+	// TLSCABundlePath -
+	TLSCABundlePath = "/etc/pki/ca-trust/extracted/pem/" + TLSCABundleFile
 )
 
 // ReconcileCAs -
@@ -72,26 +83,49 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h
 		return ctrlResult, nil
 	}
 
-	caCerts := map[string]string{}
+	bundle := newBundle()
+
+	// load current CA bundle from secret if exist
+	currentCASecret, _, err := secret.GetSecret(ctx, helper, CombinedCASecret, instance.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return ctrl.Result{}, err
+	}
+	if currentCASecret != nil {
+		if _, ok := currentCASecret.Data[TLSCABundleFile]; ok {
+			err = bundle.getCertsFromPEM(currentCASecret.Data[TLSCABundleFile])
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+	}
 
 	// create RootCA cert and Issuer that uses the generated CA certificate to issue certs
-	if instance.Spec.TLS.PublicEndpoints.Enabled && instance.Spec.TLS.PublicEndpoints.Issuer == nil {
-		caCert, ctrlResult, err := createRootCACertAndIssuer(
-			ctx,
-			instance,
-			helper,
-			issuerReq,
-			DefaultPublicCAName,
-			map[string]string{},
-		)
-		if err != nil {
-			return ctrlResult, err
-		} else if (ctrlResult != ctrl.Result{}) {
-			return ctrlResult, nil
+	if instance.Spec.TLS.PublicEndpoints.Enabled {
+		var caCert []byte
+		if instance.Spec.TLS.PublicEndpoints.Issuer == nil {
+			caCert, ctrlResult, err = createRootCACertAndIssuer(
+				ctx,
+				instance,
+				helper,
+				issuerReq,
+				DefaultPublicCAName,
+				map[string]string{},
+			)
+			if err != nil {
+				return ctrlResult, err
+			} else if (ctrlResult != ctrl.Result{}) {
+				return ctrlResult, nil
+			}
+		} else {
+			// TODO get secret name from issuer and get ca.crt
 		}
 
-		caCerts[DefaultPublicCAName] = string(caCert)
+		err = bundle.getCertsFromPEM(caCert)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
 	}
+
 	if instance.Spec.TLS.InternalEndpoints.Enabled {
 		caCert, ctrlResult, err := createRootCACertAndIssuer(
 			ctx,
@@ -109,7 +143,10 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h
 			return ctrlResult, nil
 		}
 
-		caCerts[DefaultInternalCAName] = string(caCert)
+		err = bundle.getCertsFromPEM(caCert)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
 	}
 	instance.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneCAReadyCondition, corev1.OpenStackControlPlaneCAReadyMessage)
 
@@ -129,12 +166,24 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h
 			return ctrlResult, err
 		}
 
-		for key, ca := range caSecret.Data {
-			key := instance.Spec.TLS.CaSecretName + "-" + key
-			caCerts[key] = string(ca)
+		for _, caCert := range caSecret.Data {
+			err = bundle.getCertsFromPEM(caCert)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
 		}
 	}
 
+	// get CA bundle from operator
+	caBundle, err := getOperatorCABundle(TLSCABundlePath)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	err = bundle.getCertsFromPEM(caBundle)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	saSecretTemplate := []util.Template{
 		{
 			Name:               CombinedCASecret,
@@ -147,7 +196,7 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h
 				CombinedCASecret: "",
 			},
 			ConfigOptions: nil,
-			CustomData:    caCerts,
+			CustomData:    map[string]string{TLSCABundleFile: string(caBundle)},
 		},
 	}
 
@@ -174,8 +223,7 @@ func createRootCACertAndIssuer(
 	selfsignedIssuerReq *certmgrv1.Issuer,
 	caName string,
 	labels map[string]string,
-) (string, ctrl.Result, error) {
-	var caCert string
+) ([]byte, ctrl.Result, error) {
 	// create RootCA Certificate used to sign certificates
 	caCertReq := certmanager.Cert(
 		caName,
@@ -208,15 +256,15 @@ func createRootCACertAndIssuer(
 			caCertReq.Name,
 			err.Error()))
 
-		return caCert, ctrlResult, err
+		return nil, ctrlResult, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			corev1.OpenStackControlPlaneCAReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			corev1.OpenStackControlPlaneCAReadyRunningMessage))
 
-		return caCert, ctrlResult, nil
+		return nil, ctrlResult, nil
 	}
 
 	// create Issuer that uses the generated CA certificate to issue certs
@@ -239,22 +287,22 @@ func createRootCACertAndIssuer(
 			issuerReq.GetName(),
 			err.Error()))
 
-		return caCert, ctrlResult, err
+		return nil, ctrlResult, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			corev1.OpenStackControlPlaneCAReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			corev1.OpenStackControlPlaneCAReadyRunningMessage))
 
-		return caCert, ctrlResult, nil
+		return nil, ctrlResult, nil
 	}
 
-	caCert, ctrlResult, err = getCAFromSecret(ctx, instance, helper, caName)
+	caCert, ctrlResult, err := getCAFromSecret(ctx, instance, helper, caName)
 	if err != nil {
-		return caCert, ctrl.Result{}, err
+		return nil, ctrl.Result{}, err
 	} else if (ctrlResult != ctrl.Result{}) {
-		return caCert, ctrlResult, nil
+		return nil, ctrlResult, nil
 	}
 
 	return caCert, ctrl.Result{}, nil
@@ -265,7 +313,7 @@ func getCAFromSecret(
 	instance *corev1.OpenStackControlPlane,
 	helper *helper.Helper,
 	caName string,
-) (string, ctrl.Result, error) {
+) ([]byte, ctrl.Result, error) {
 	caSecret, ctrlResult, err := secret.GetDataFromSecret(ctx, helper, caName, time.Duration(5), "ca.crt")
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
@@ -277,16 +325,117 @@ func getCAFromSecret(
 			caName,
 			err.Error()))
 
-		return caSecret, ctrlResult, err
+		return nil, ctrlResult, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			corev1.OpenStackControlPlaneCAReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			corev1.OpenStackControlPlaneCAReadyRunningMessage))
 
-		return caSecret, ctrlResult, nil
+		return nil, ctrlResult, nil
+	}
+
+	return []byte(caSecret), ctrl.Result{}, nil
+}
+
+func getOperatorCABundle(caFile string) ([]byte, error) {
+	contents, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("File reading error %w", err)
+	}
+
+	return contents, nil
+}
+
+func days(t time.Time) int {
+	return int(math.Round(time.Since(t).Hours() / 24))
+}
+
+type caBundle struct {
+	certs []caCert
+}
+
+type caCert struct {
+	hash string
+	cert *x509.Certificate
+}
+
+// newBundle returns a new, empty Bundle
+func newBundle() *caBundle {
+	return &caBundle{
+		certs: make([]caCert, 0),
+	}
+}
+
+func (ca *caBundle) getCertsFromPEM(PEMdata []byte) error {
+	if PEMdata == nil {
+		return fmt.Errorf("certificate data can't be nil")
+	}
+
+	for {
+		var block *pem.Block
+		block, PEMdata = pem.Decode(PEMdata)
+
+		if block == nil {
+			break
+		}
+
+		if block.Type != "CERTIFICATE" {
+			// only certificates are allowed in a bundle
+			return fmt.Errorf("invalid PEM block in bundle: only CERTIFICATE blocks are permitted but found '%s'", block.Type)
+		}
+
+		if len(block.Headers) != 0 {
+			return fmt.Errorf("invalid PEM block in bundle; blocks are not permitted to have PEM headers")
+		}
+
+		certificate, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			// the presence of an invalid cert (including things which aren't certs)
+			// should cause the bundle to be rejected
+			return fmt.Errorf("invalid PEM block in bundle; invalid PEM certificate: %w", err)
+		}
+
+		if certificate == nil {
+			return fmt.Errorf("failed appending a certificate: certificate is nil")
+		}
+
+		// validate if the CA expired
+		if -days(certificate.NotAfter) <= 0 {
+			continue
+		}
+
+		blockHash, err := util.ObjectHash(block.Bytes)
+		if err != nil {
+			return fmt.Errorf("failed calc hash of PEM block : %w", err)
+		}
+
+		// if cert is not already in bundle list add it
+		// validate of nextip is already in a reservation and its not us
+		f := func(c caCert) bool {
+			return c.hash == blockHash
+		}
+		idx := slices.IndexFunc(ca.certs, f)
+		if idx == -1 {
+			ca.certs = append(ca.certs,
+				caCert{
+					hash: blockHash,
+					cert: certificate,
+				})
+		}
+	}
+
+	return nil
+}
+
+// Create PEM bundle from certificates
+func (ca *caBundle) getBundlePEM() [][]byte {
+	var certsData = make([][]byte, len(ca.certs))
+
+	for i, cert := range ca.certs {
+		certsData[i] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.cert.Raw})
 	}
 
-	return caSecret, ctrl.Result{}, nil
+	return certsData
 }