From 6010e3b190b8aca749b408c8f99e5b6b8245fb13 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Tue, 20 Feb 2024 15:25:54 +0100 Subject: [PATCH] [tlse] TLS database connection The my.cnf file gets added to the secret holding the service configs. The content of my.cnf is centrally managed in the mariadb-operator and retrieved calling db.GetDatabaseClientConfig(tlsCfg) Depends-On: https://github.com/openstack-k8s-operators/mariadb-operator/pull/190 Depends-On: https://github.com/openstack-k8s-operators/mariadb-operator/pull/191 Jira: OSPRH-4547 --- controllers/cinder_controller.go | 153 +++++++++++------- pkg/cinder/const.go | 2 + pkg/cinder/cronjob.go | 11 +- pkg/cinder/volumes.go | 6 + test/functional/cinder_controller_test.go | 26 ++- .../common/assert_sample_deployment.yaml | 4 + .../common/assert_tls_sample_deployment.yaml | 20 +++ 7 files changed, 156 insertions(+), 66 deletions(-) diff --git a/controllers/cinder_controller.go b/controllers/cinder_controller.go index c84e31e2..ef739bd8 100644 --- a/controllers/cinder_controller.go +++ b/controllers/cinder_controller.go @@ -51,6 +51,7 @@ import ( common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -380,63 +381,6 @@ func (r *CinderReconciler) reconcileInit( Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) - // - // create service DB instance - // - db := mariadbv1.NewDatabase( - instance.Name, - instance.Spec.DatabaseUser, - instance.Spec.Secret, - map[string]string{ - "dbName": instance.Spec.DatabaseInstance, - }, - ) - // create or patch the DB - ctrlResult, err := db.CreateOrPatchDB( - ctx, - helper, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DBReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err - } - if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DBReadyRunningMessage)) - return ctrlResult, nil - } - // wait for the DB to be setup - ctrlResult, err = db.WaitForDBCreated(ctx, helper) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.DBReadyErrorMessage, - err.Error())) - return ctrlResult, err - } - if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DBReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.DBReadyRunningMessage)) - return ctrlResult, nil - } - // update Status.DatabaseHostname, used to config the service - instance.Status.DatabaseHostname = db.GetDatabaseHostname() - instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) - // create service DB - end - // // run Cinder db sync // @@ -450,7 +394,7 @@ func (r *CinderReconciler) reconcileInit( time.Duration(5)*time.Second, dbSyncHash, ) - ctrlResult, err = dbSyncjob.DoJob( + ctrlResult, err := dbSyncjob.DoJob( ctx, helper, ) @@ -615,10 +559,17 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) // run check OpenStack secret - end + db, result, err := r.ensureDB(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + // // Create Secrets required as input for the Service and calculate an overall hash of hashes // - err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached) + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -911,6 +862,7 @@ func (r *CinderReconciler) generateServiceConfigs( envVars *map[string]env.Setter, serviceLabels map[string]string, memcached *memcachedv1.Memcached, + db *mariadbv1.Database, ) error { // // create Secret required for cinder input @@ -920,8 +872,20 @@ func (r *CinderReconciler) generateServiceConfigs( labels := labels.GetLabels(instance, labels.GetGroupLabel(cinder.ServiceName), serviceLabels) + db, err := mariadbv1.GetDatabaseByName(ctx, h, cinder.DatabaseName) + if err != nil { + return err + } + + var tlsCfg *tls.Service + if instance.Spec.CinderAPI.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } // customData hold any customization for all cinder services. - customData := map[string]string{cinder.CustomConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{ + cinder.CustomConfigFileName: instance.Spec.CustomServiceConfig, + cinder.MyCnfFileName: db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) if err != nil { @@ -952,7 +916,7 @@ func (r *CinderReconciler) generateServiceConfigs( templateParameters["KeystoneInternalURL"] = keystoneInternalURL templateParameters["KeystonePublicURL"] = keystonePublicURL templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) - templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s", + templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", instance.Spec.DatabaseUser, string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]), instance.Status.DatabaseHostname, @@ -1272,3 +1236,70 @@ func (r *CinderReconciler) volumeCleanupDeployments(ctx context.Context, instanc return nil } + +func (r *CinderReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *cinderv1beta1.Cinder, +) (*mariadbv1.Database, ctrl.Result, error) { + // + // create service DB instance + // + db := mariadbv1.NewDatabase( + instance.Name, + instance.Spec.DatabaseUser, + instance.Spec.Secret, + map[string]string{ + "dbName": instance.Spec.DatabaseInstance, + }, + ) + + // create or patch the DB + ctrlResult, err := db.CreateOrPatchDBByName( + ctx, + h, + instance.Spec.DatabaseInstance, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + // wait for the DB to be setup + // (ksambor) should we use WaitForDBCreatedWithTimeout instead? + ctrlResult, err = db.WaitForDBCreated(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + return db, ctrlResult, nil +} diff --git a/pkg/cinder/const.go b/pkg/cinder/const.go index 31774792..560b39d1 100644 --- a/pkg/cinder/const.go +++ b/pkg/cinder/const.go @@ -41,6 +41,8 @@ const ( CustomServiceConfigFileName = "03-service-custom.conf" // CustomServiceConfigSecretsFileName - CustomServiceConfigSecretsFileName = "04-service-custom-secrets.conf" + // MyCnfFileName - + MyCnfFileName = "my.cnf" // CinderPublicPort - CinderPublicPort int32 = 8776 diff --git a/pkg/cinder/cronjob.go b/pkg/cinder/cronjob.go index 70797e4d..53e4f551 100644 --- a/pkg/cinder/cronjob.go +++ b/pkg/cinder/cronjob.go @@ -17,6 +17,7 @@ package cinder import ( cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" + cinderv1beta1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" "fmt" @@ -75,6 +76,12 @@ func CronJob( MountPath: "/etc/cinder/cinder.conf.d", ReadOnly: true, }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: MyCnfFileName, + ReadOnly: true, + }, } // add CA cert if defined @@ -83,6 +90,8 @@ func CronJob( cronJobVolumeMounts = append(cronJobVolumeMounts, instance.Spec.CinderAPI.TLS.CreateVolumeMounts(nil)...) } + cronJobExtraMounts := []cinderv1beta1.CinderExtraVolMounts{} + cronjob := &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-db-purge", ServiceName), @@ -122,7 +131,7 @@ func CronJob( }, }, }, - Volumes: cronJobVolumes, + Volumes: append(GetVolumes(instance.Name, false, cronJobExtraMounts, DbsyncPropagation), cronJobVolumes...), RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: instance.RbacResourceName(), }, diff --git a/pkg/cinder/volumes.go b/pkg/cinder/volumes.go index d69b8bee..25a00fe1 100644 --- a/pkg/cinder/volumes.go +++ b/pkg/cinder/volumes.go @@ -172,6 +172,12 @@ func GetVolumeMounts(storageSvc bool, extraVol []cinderv1beta1.CinderExtraVolMou MountPath: "/var/lib/config-data/merged", ReadOnly: true, }, + { + Name: "config-data", + MountPath: "/etc/my.cnf", + SubPath: MyCnfFileName, + ReadOnly: true, + }, } // Volume and backup services require extra directories diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go index bba5adeb..72f834e5 100644 --- a/test/functional/cinder_controller_test.go +++ b/test/functional/cinder_controller_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/utils/ptr" cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" + "github.com/openstack-k8s-operators/cinder-operator/pkg/cinder" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" @@ -218,13 +219,17 @@ var _ = Describe("Cinder controller", func() { infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, "memcached", memcachedSpec)) infra.SimulateMemcachedReady(cinderTest.CinderMemcached) + mariadb.SimulateMariaDBAccountCompleted(cinderTest.Instance) + mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Instance) }) It("should create config-data and scripts ConfigMaps", func() { keystoneAPI := keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace) DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPI) - Eventually(func() corev1.Secret { - return th.GetSecret(cinderTest.CinderConfigSecret) - }, timeout, interval).ShouldNot(BeNil()) + cf := th.GetSecret(cinderTest.CinderConfigSecret) + Expect(cf).ShouldNot(BeNil()) + conf := cf.Data[cinder.MyCnfFileName] + Expect(conf).To( + ContainSubstring("[client]\nssl=0")) Eventually(func() corev1.Secret { return th.GetSecret(cinderTest.CinderConfigScripts) }, timeout, interval).ShouldNot(BeNil()) @@ -461,7 +466,7 @@ var _ = Describe("Cinder controller", func() { infra.SimulateMemcachedReady(cinderTest.CinderMemcached) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) mariadb.SimulateMariaDBAccountCompleted(cinderTest.Instance) - mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Instance) + mariadb.SimulateMariaDBTLSDatabaseCompleted(cinderTest.Instance) th.SimulateJobSuccess(cinderTest.CinderDBSync) }) @@ -510,6 +515,19 @@ var _ = Describe("Cinder controller", func() { ) }) + It("should create config-data and scripts ConfigMaps", func() { + keystoneAPI := keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPI) + cf := th.GetSecret(cinderTest.CinderConfigSecret) + Expect(cf).ShouldNot(BeNil()) + conf := cf.Data[cinder.MyCnfFileName] + Expect(conf).To( + ContainSubstring("[client]\nssl-ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nssl=1")) + Eventually(func() corev1.Secret { + return th.GetSecret(cinderTest.CinderConfigScripts) + }, timeout, interval).ShouldNot(BeNil()) + }) + It("Creates CinderAPI", func() { DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(cinderTest.CABundleSecret)) DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(cinderTest.InternalCertSecret)) diff --git a/test/kuttl/common/assert_sample_deployment.yaml b/test/kuttl/common/assert_sample_deployment.yaml index c774b6ad..0a49beb7 100644 --- a/test/kuttl/common/assert_sample_deployment.yaml +++ b/test/kuttl/common/assert_sample_deployment.yaml @@ -121,6 +121,10 @@ spec: - mountPath: /var/lib/config-data/merged name: config-data readOnly: true + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /etc/cinder/cinder.conf.d name: config-data-custom readOnly: true diff --git a/test/kuttl/common/assert_tls_sample_deployment.yaml b/test/kuttl/common/assert_tls_sample_deployment.yaml index 9d00ba88..178c48b0 100644 --- a/test/kuttl/common/assert_tls_sample_deployment.yaml +++ b/test/kuttl/common/assert_tls_sample_deployment.yaml @@ -83,6 +83,10 @@ spec: - mountPath: /var/lib/config-data/merged name: config-data readOnly: true + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /etc/cinder/cinder.conf.d name: config-data-custom readOnly: true @@ -175,6 +179,10 @@ spec: - mountPath: /var/lib/config-data/merged name: config-data readOnly: true + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /etc/cinder/cinder.conf.d name: config-data-custom readOnly: true @@ -203,6 +211,10 @@ spec: - mountPath: /var/lib/config-data/merged name: config-data readOnly: true + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /etc/cinder/cinder.conf.d name: config-data-custom readOnly: true @@ -267,6 +279,10 @@ spec: - mountPath: /var/lib/config-data/merged name: config-data readOnly: true + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/cinder name: var-lib-cinder - mountPath: /etc/nvme @@ -315,6 +331,10 @@ spec: - mountPath: /var/lib/config-data/merged name: config-data readOnly: true + - mountPath: /etc/my.cnf + name: config-data + readOnly: true + subPath: my.cnf - mountPath: /var/lib/cinder name: var-lib-cinder - mountPath: /etc/nvme