diff --git a/config/samples/core_v1beta1_openstackcontrolplane.yaml b/config/samples/core_v1beta1_openstackcontrolplane.yaml index 4a1623aeb..2debd498e 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane.yaml @@ -10,9 +10,6 @@ spec: enabled: false publicEndpoints: enabled: true - openstackclient: - template: - containerImage: quay.io/mschuppe/openstack-openstackclient:current-podified keystone: template: databaseInstance: openstack diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml index bc42ec9d4..72c41ce5e 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation.yaml @@ -10,9 +10,6 @@ spec: enabled: false publicEndpoints: enabled: true - openstackclient: - template: - containerImage: quay.io/mschuppe/openstack-openstackclient:current-podified dns: template: override: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml index 2bd1623bb..d3eaff2c5 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_galera_network_isolation_3replicas.yaml @@ -10,9 +10,6 @@ spec: enabled: false publicEndpoints: enabled: true - openstackclient: - template: - containerImage: quay.io/mschuppe/openstack-openstackclient:current-podified dns: template: override: diff --git a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml index bff45b8cb..3cf74dcc5 100644 --- a/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml +++ b/config/samples/core_v1beta1_openstackcontrolplane_network_isolation.yaml @@ -10,9 +10,6 @@ spec: enabled: false publicEndpoints: enabled: true - openstackclient: - template: - containerImage: quay.io/mschuppe/openstack-openstackclient:current-podified dns: template: override: diff --git a/controllers/core/openstackcontrolplane_controller.go b/controllers/core/openstackcontrolplane_controller.go index fc74de612..bcc4e8517 100644 --- a/controllers/core/openstackcontrolplane_controller.go +++ b/controllers/core/openstackcontrolplane_controller.go @@ -31,6 +31,7 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + corev1 "k8s.io/api/core/v1" manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -328,6 +329,7 @@ func (r *OpenStackControlPlaneReconciler) reconcileNormal(ctx context.Context, i func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1beta1.OpenStackControlPlane{}). + Owns(&corev1.Secret{}). Owns(&mariadbv1.MariaDB{}). Owns(&mariadbv1.Galera{}). Owns(&memcachedv1.Memcached{}). diff --git a/go.mod b/go.mod index 86dd52bfe..6fe69fa49 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/operator-framework/api v0.17.6 github.com/rabbitmq/cluster-operator/v2 v2.5.0 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.4 k8s.io/client-go v0.27.2 @@ -46,7 +47,6 @@ require ( github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect github.com/metal3-io/baremetal-operator/apis v0.3.1 // indirect github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.2.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/tools v0.14.0 // indirect sigs.k8s.io/gateway-api v0.6.0 // indirect ) diff --git a/pkg/openstack/ca.go b/pkg/openstack/ca.go index 823a23ae6..eaf4caffc 100644 --- a/pkg/openstack/ca.go +++ b/pkg/openstack/ca.go @@ -2,6 +2,12 @@ package openstack import ( "context" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "math" + "os" "time" certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -12,6 +18,8 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + "golang.org/x/exp/slices" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" @@ -25,6 +33,12 @@ const ( DefaultPublicCAName = "rootca-" + string(service.EndpointPublic) // DefaultInternalCAName - DefaultInternalCAName = "rootca-" + string(service.EndpointInternal) + // TLSCABundleFile - + TLSCABundleFile = "tls-ca-bundle.pem" + // DownstreamTLSCABundlePath - + DownstreamTLSCABundlePath = "/etc/pki/ca-trust/extracted/pem/" + TLSCABundleFile + // UpstreamTLSCABundlePath - + UpstreamTLSCABundlePath = "/etc/ssl/certs/ca-certificates.crt" ) // ReconcileCAs - @@ -72,26 +86,49 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h return ctrlResult, nil } - caCerts := map[string]string{} + bundle := newBundle() + + // load current CA bundle from secret if exist + currentCASecret, _, err := secret.GetSecret(ctx, helper, CombinedCASecret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if currentCASecret != nil { + if _, ok := currentCASecret.Data[TLSCABundleFile]; ok { + err = bundle.getCertsFromPEM(currentCASecret.Data[TLSCABundleFile]) + if err != nil { + return ctrl.Result{}, err + } + } + } // create RootCA cert and Issuer that uses the generated CA certificate to issue certs - if instance.Spec.TLS.PublicEndpoints.Enabled && instance.Spec.TLS.PublicEndpoints.Issuer == nil { - caCert, ctrlResult, err := createRootCACertAndIssuer( - ctx, - instance, - helper, - issuerReq, - DefaultPublicCAName, - map[string]string{}, - ) - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil + if instance.Spec.TLS.PublicEndpoints.Enabled { + var caCert []byte + if instance.Spec.TLS.PublicEndpoints.Issuer == nil { + caCert, ctrlResult, err = createRootCACertAndIssuer( + ctx, + instance, + helper, + issuerReq, + DefaultPublicCAName, + map[string]string{}, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } else { + // TODO get secret name from issuer and get ca.crt } - caCerts[DefaultPublicCAName] = string(caCert) + err = bundle.getCertsFromPEM(caCert) + if err != nil { + return ctrl.Result{}, err + } } + if instance.Spec.TLS.InternalEndpoints.Enabled { caCert, ctrlResult, err := createRootCACertAndIssuer( ctx, @@ -109,7 +146,10 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h return ctrlResult, nil } - caCerts[DefaultInternalCAName] = string(caCert) + err = bundle.getCertsFromPEM(caCert) + if err != nil { + return ctrl.Result{}, err + } } instance.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneCAReadyCondition, corev1.OpenStackControlPlaneCAReadyMessage) @@ -129,12 +169,35 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h return ctrlResult, err } - for key, ca := range caSecret.Data { - key := instance.Spec.TLS.CaSecretName + "-" + key - caCerts[key] = string(ca) + for _, caCert := range caSecret.Data { + err = bundle.getCertsFromPEM(caCert) + if err != nil { + return ctrl.Result{}, err + } } } + // get CA bundle from operator image. Downstream and upstream build use a different + // base image, so the ca bundle cert file can be in different locations + caBundle, err := getOperatorCABundle(DownstreamTLSCABundlePath) + if err != nil { + // if the DownstreamTLSCABundlePath does not exist in the operator image, + // check for UpstreamTLSCABundlePath + if errors.Is(err, os.ErrNotExist) { + helper.GetLogger().Info(fmt.Sprintf("Downstream CA bundle not found using: %s", UpstreamTLSCABundlePath)) + caBundle, err = getOperatorCABundle(UpstreamTLSCABundlePath) + if err != nil { + return ctrl.Result{}, err + } + } else { + return ctrl.Result{}, err + } + } + err = bundle.getCertsFromPEM(caBundle) + if err != nil { + return ctrl.Result{}, err + } + saSecretTemplate := []util.Template{ { Name: CombinedCASecret, @@ -147,7 +210,7 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h CombinedCASecret: "", }, ConfigOptions: nil, - CustomData: caCerts, + CustomData: map[string]string{TLSCABundleFile: bundle.getBundlePEM()}, }, } @@ -174,8 +237,7 @@ func createRootCACertAndIssuer( selfsignedIssuerReq *certmgrv1.Issuer, caName string, labels map[string]string, -) (string, ctrl.Result, error) { - var caCert string +) ([]byte, ctrl.Result, error) { // create RootCA Certificate used to sign certificates caCertReq := certmanager.Cert( caName, @@ -208,7 +270,7 @@ func createRootCACertAndIssuer( caCertReq.Name, err.Error())) - return caCert, ctrlResult, err + return nil, ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( corev1.OpenStackControlPlaneCAReadyCondition, @@ -216,7 +278,7 @@ func createRootCACertAndIssuer( condition.SeverityInfo, corev1.OpenStackControlPlaneCAReadyRunningMessage)) - return caCert, ctrlResult, nil + return nil, ctrlResult, nil } // create Issuer that uses the generated CA certificate to issue certs @@ -239,7 +301,7 @@ func createRootCACertAndIssuer( issuerReq.GetName(), err.Error())) - return caCert, ctrlResult, err + return nil, ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( corev1.OpenStackControlPlaneCAReadyCondition, @@ -247,14 +309,14 @@ func createRootCACertAndIssuer( condition.SeverityInfo, corev1.OpenStackControlPlaneCAReadyRunningMessage)) - return caCert, ctrlResult, nil + return nil, ctrlResult, nil } - caCert, ctrlResult, err = getCAFromSecret(ctx, instance, helper, caName) + caCert, ctrlResult, err := getCAFromSecret(ctx, instance, helper, caName) if err != nil { - return caCert, ctrl.Result{}, err + return nil, ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { - return caCert, ctrlResult, nil + return nil, ctrlResult, nil } return caCert, ctrl.Result{}, nil @@ -265,7 +327,7 @@ func getCAFromSecret( instance *corev1.OpenStackControlPlane, helper *helper.Helper, caName string, -) (string, ctrl.Result, error) { +) ([]byte, ctrl.Result, error) { caSecret, ctrlResult, err := secret.GetDataFromSecret(ctx, helper, caName, time.Duration(5), "ca.crt") if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( @@ -277,7 +339,7 @@ func getCAFromSecret( caName, err.Error())) - return caSecret, ctrlResult, err + return nil, ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( corev1.OpenStackControlPlaneCAReadyCondition, @@ -285,8 +347,110 @@ func getCAFromSecret( condition.SeverityInfo, corev1.OpenStackControlPlaneCAReadyRunningMessage)) - return caSecret, ctrlResult, nil + return nil, ctrlResult, nil + } + + return []byte(caSecret), ctrl.Result{}, nil +} + +func getOperatorCABundle(caFile string) ([]byte, error) { + contents, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("File reading error %w", err) + } + + return contents, nil +} + +func days(t time.Time) int { + return int(math.Round(time.Since(t).Hours() / 24)) +} + +type caBundle struct { + certs []caCert +} + +type caCert struct { + hash string + cert *x509.Certificate +} + +// newBundle returns a new, empty Bundle +func newBundle() *caBundle { + return &caBundle{ + certs: make([]caCert, 0), + } +} + +func (cab *caBundle) getCertsFromPEM(PEMdata []byte) error { + if PEMdata == nil { + return fmt.Errorf("certificate data can't be nil") + } + + for { + var block *pem.Block + block, PEMdata = pem.Decode(PEMdata) + + if block == nil { + break + } + + if block.Type != "CERTIFICATE" { + // only certificates are allowed in a bundle + return fmt.Errorf("invalid PEM block in bundle: only CERTIFICATE blocks are permitted but found '%s'", block.Type) + } + + if len(block.Headers) != 0 { + return fmt.Errorf("invalid PEM block in bundle; blocks are not permitted to have PEM headers") + } + + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + // the presence of an invalid cert (including things which aren't certs) + // should cause the bundle to be rejected + return fmt.Errorf("invalid PEM block in bundle; invalid PEM certificate: %w", err) + } + + if certificate == nil { + return fmt.Errorf("failed appending a certificate: certificate is nil") + } + + // validate if the CA expired + if -days(certificate.NotAfter) <= 0 { + continue + } + + blockHash, err := util.ObjectHash(block.Bytes) + if err != nil { + return fmt.Errorf("failed calc hash of PEM block : %w", err) + } + + // if cert is not already in bundle list add it + // validate of nextip is already in a reservation and its not us + f := func(c caCert) bool { + return c.hash == blockHash + } + idx := slices.IndexFunc(cab.certs, f) + if idx == -1 { + cab.certs = append(cab.certs, + caCert{ + hash: blockHash, + cert: certificate, + }) + } + } + + return nil +} + +// Create PEM bundle from certificates +func (cab *caBundle) getBundlePEM() string { + var bundleData string + + for _, cert := range cab.certs { + bundleData += "# " + cert.cert.Issuer.CommonName + "\n" + + string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.cert.Raw})) } - return caSecret, ctrl.Result{}, nil + return bundleData }