Skip to content

Commit

Permalink
[tlse] TLS database connection
Browse files Browse the repository at this point in the history 
The my.cnf file gets added to the secret holding the service configs.
The content of my.cnf is centrally managed in the mariadb-operator
and retrieved calling db.GetDatabaseClientConfig(tlsCfg)

Depends-On: openstack-k8s-operators/mariadb-operator#190
Depends-On: openstack-k8s-operators/mariadb-operator#191
Depends-On: openstack-k8s-operators/mariadb-operator#200

Jira: OSPRH-4547
  • Loading branch information
@stuggi
stuggi committed Feb 26, 2024
1 parent fb901e9 commit c782618
Show file tree
Hide file tree
Showing 8 changed files with 336 additions and 69 deletions.
154 changes: 91 additions & 63 deletions controllers/glance_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"time"

"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
rbacv1 "k8s.io/api/rbac/v1"
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/fields"
Expand Down Expand Up @@ -363,67 +364,10 @@ func (r *GlanceReconciler) reconcileInit(
) (ctrl.Result, error) {
r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name))

//
// create service DB instance
//
db := mariadbv1.NewDatabase(
instance.Name,
instance.Spec.DatabaseUser,
instance.Spec.Secret,
map[string]string{
"dbName": instance.Spec.DatabaseInstance,
},
)
// create or patch the DB
ctrlResult, err := db.CreateOrPatchDB(
ctx,
helper,
)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return ctrl.Result{}, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return ctrlResult, nil
}
// wait for the DB to be setup
ctrlResult, err = db.WaitForDBCreated(ctx, helper)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return ctrlResult, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return ctrlResult, nil
}
// update Status.DatabaseHostname, used to config the service
instance.Status.DatabaseHostname = db.GetDatabaseHostname()
instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
// create service DB - end

//
// create Keystone service and users - https://docs.openstack.org/Glance/latest/install/install-rdo.html#configure-user-and-endpoints
//
_, _, err = oko_secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace)
_, _, err := oko_secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace)
if err != nil {
if k8s_errors.IsNotFound(err) {
return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Secret)
Expand All @@ -442,7 +386,7 @@ func (r *GlanceReconciler) reconcileInit(
}

ksSvc := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, time.Duration(10)*time.Second)
ctrlResult, err = ksSvc.CreateOrPatch(ctx, helper)
ctrlResult, err := ksSvc.CreateOrPatch(ctx, helper)
if err != nil {
return ctrlResult, err
}
Expand Down Expand Up @@ -588,12 +532,19 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
// run check OpenStack secret - end

db, ctrlResult, err := r.ensureDB(ctx, helper, instance)
if err != nil {
return ctrl.Result{}, err
} else if (ctrlResult != ctrl.Result{}) {
return ctrlResult, nil
}

//
// Create Secrets required as input for the Service and calculate an overall hash of hashes
//

//
err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels)
err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels, db)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.ServiceConfigReadyCondition,
Expand All @@ -611,7 +562,6 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
//

var serviceAnnotations map[string]string
var ctrlResult ctrl.Result
// networks to attach to
for _, glanceAPI := range instance.Spec.GlanceAPIs {
serviceAnnotations, ctrlResult, err = ensureNAD(ctx, &instance.Status.Conditions, glanceAPI.NetworkAttachments, helper)
Expand Down Expand Up @@ -898,6 +848,7 @@ func (r *GlanceReconciler) generateServiceConfig(
instance *glancev1.Glance,
envVars *map[string]env.Setter,
serviceLabels map[string]string,
db *mariadbv1.Database,
) error {
labels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), serviceLabels)

Expand All @@ -910,7 +861,7 @@ func (r *GlanceReconciler) generateServiceConfig(
// hence only passing the database related parameters
templateParameters := map[string]interface{}{
"MinimalConfig": true, // This tells the template to generate a minimal config
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
instance.Spec.DatabaseUser,
string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
instance.Status.DatabaseHostname,
Expand All @@ -930,7 +881,17 @@ func (r *GlanceReconciler) generateServiceConfig(
templateParameters["ImageCacheDir"] = glance.ImageCacheDir
}

customData := map[string]string{glance.CustomConfigFileName: instance.Spec.CustomServiceConfig}
var tlsCfg *tls.Service
for _, api := range instance.Spec.GlanceAPIs {
if api.TLS.CaBundleSecretName != "" {
tlsCfg = &tls.Service{}
break
}
}
customData := map[string]string{
glance.CustomConfigFileName: instance.Spec.CustomServiceConfig,
"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
}

// Generate both default 00-config.conf and -scripts
return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, true)
Expand Down Expand Up @@ -1085,3 +1046,70 @@ func (r *GlanceReconciler) glanceAPICleanup(ctx context.Context, instance *glanc
}
return nil
}

func (r *GlanceReconciler) ensureDB(
ctx context.Context,
h *helper.Helper,
instance *glancev1.Glance,
) (*mariadbv1.Database, ctrl.Result, error) {
//
// create service DB instance
//
db := mariadbv1.NewDatabase(
instance.Name,
instance.Spec.DatabaseUser,
instance.Spec.Secret,
map[string]string{
"dbName": instance.Spec.DatabaseInstance,
},
)

// create or patch the DB
ctrlResult, err := db.CreateOrPatchDBByName(
ctx,
h,
instance.Spec.DatabaseInstance,
)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return db, ctrl.Result{}, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return db, ctrlResult, nil
}
// wait for the DB to be setup
// (ksambor) should we use WaitForDBCreatedWithTimeout instead?
ctrlResult, err = db.WaitForDBCreated(ctx, h)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.DBReadyErrorMessage,
err.Error()))
return db, ctrlResult, err
}
if (ctrlResult != ctrl.Result{}) {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.DBReadyCondition,
condition.RequestedReason,
condition.SeverityInfo,
condition.DBReadyRunningMessage))
return db, ctrlResult, nil
}

// update Status.DatabaseHostname, used to config the service
instance.Status.DatabaseHostname = db.GetDatabaseHostname()
instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
return db, ctrlResult, nil
}
17 changes: 15 additions & 2 deletions controllers/glanceapi_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import (
"github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
"github.com/openstack-k8s-operators/lib-common/modules/common/util"
mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -791,8 +792,20 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
) error {
labels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), serviceLabels)

db, err := mariadbv1.GetDatabaseByName(ctx, h, glance.DatabaseName)
if err != nil {
return err
}

var tlsCfg *tls.Service
if instance.Spec.TLS.Ca.CaBundleSecretName != "" {
tlsCfg = &tls.Service{}
}
// 02-config.conf
customData := map[string]string{glance.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
customData := map[string]string{
glance.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig,
"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
}

// 03-config.conf
customSecrets := ""
Expand Down Expand Up @@ -845,7 +858,7 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
"ServicePassword": string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]),
"KeystoneInternalURL": keystoneInternalURL,
"KeystonePublicURL": keystonePublicURL,
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
instance.Spec.DatabaseUser,
string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
instance.Spec.DatabaseHostname,
Expand Down
15 changes: 15 additions & 0 deletions pkg/glance/cronjob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,28 @@ func CronJob(
},
},
},
{
Name: "config-data",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
DefaultMode: &config0644AccessMode,
SecretName: ServiceName + "-config-data",
},
},
},
}
cronJobVolumeMounts := []corev1.VolumeMount{
{
Name: "db-purge-config-data",
MountPath: "/etc/glance/glance.conf.d",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
}

// add CA cert if defined from the first api
Expand Down
6 changes: 6 additions & 0 deletions pkg/glance/dbsync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ func DbSyncJob(
MountPath: "/etc/glance/glance.conf.d",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/var/lib/kolla/config_files/config.json",
Expand Down
6 changes: 6 additions & 0 deletions pkg/glance/volumes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,12 @@ func GetVolumeMounts(secretNames []string, hasCinder bool, extraVol []glancev1.G
MountPath: "/var/lib/config-data/default",
ReadOnly: true,
},
{
Name: "config-data",
MountPath: "/etc/my.cnf",
SubPath: "my.cnf",
ReadOnly: true,
},
{
Name: ServiceName,
MountPath: "/var/lib/glance",
Expand Down
7 changes: 7 additions & 0 deletions test/functional/glance_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ var _ = Describe("Glance controller", func() {
th.SimulateJobSuccess(glanceTest.GlanceDBSync)
Glance := GetGlance(glanceTest.Instance)
Expect(Glance.Status.DatabaseHostname).To(Equal(fmt.Sprintf("hostname-for-openstack.%s.svc", namespace)))

secretDataMap := th.GetSecret(glanceTest.GlanceConfigMapData)
Expect(secretDataMap).ShouldNot(BeNil())
myCnf := secretDataMap.Data["my.cnf"]
Expect(myCnf).To(
ContainSubstring("[client]\nssl=0"))

th.ExpectCondition(
glanceName,
ConditionGetterFunc(GlanceConditionGetter),
Expand Down
Loading

0 comments on commit c782618

Please sign in to comment.