From 54812e3c028fe61c1dcd2b70854bceb04d95231a Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Fri, 25 Aug 2023 11:37:57 +0200 Subject: [PATCH] Rework the whole Manila configuration process This patch represents an implementation of the proposal [1] and aligns the manila-operator with the work already done in Cinder and the other operators. There are a few relevant changes in the bootstrap process of Manila, in particular: 1. It stops using an InitContainer to generate the snippet files that configure each Manila service. The logic that was previously implemented in the InitContainer has been moved to the Controllers, where config files are generated and stored in k8s Secrets. 2. InitContainers are fully removed from the bootstrap process; It uses to copy additional files (httpd and wsgi config in manila-api) to the target directories, and mount the generated config (0{0,1,2,3}-config.conf to /etc/manila/manila.conf.d, which is used by each service to run; 3. The relevant content, including scripts, previously stored in a ConfigMap, are now stored in a corresponding k8s Secret, which is mounted to the Service deployment Pods; A total of 4 config snippet files are generated: - 00-config.conf contains global settings that are common to every Manila Pod, including ones that are derived from deployment secrets (e.g. database password, etc.) - 01-config.conf contains the global customServiceConfig settings that apply to every Manila service. - 02-config.conf contains the customServiceConfig settings that are specific to each service. - 03-config.conf contains secrets specified by each service's customServiceConfigSecrets. logging.conf has been removed as it' s no longer required in the switch to a side container approach for logging purposes, which will be part of a follow up PR. Finally, functional tests are aligned to the use of k8s Secrets instead of the old pattern based on ConfigMaps; kuttl tests are updated and the initContainer has been removed. DBsync now mounts only the required files (a minimal 00-config.conf) and a db-sync-config.json containing the command run through kolla. [1] openstack-k8s-operators/docs#31 Signed-off-by: Francesco Pantano --- .../manila.openstack.org_manilaapis.yaml | 3 - api/bases/manila.openstack.org_manilas.yaml | 12 - ...manila.openstack.org_manilaschedulers.yaml | 3 - .../manila.openstack.org_manilashares.yaml | 3 - api/v1beta1/common_types.go | 10 +- api/v1beta1/manila_types.go | 2 +- api/v1beta1/manilashare_types.go | 1 - .../manila.openstack.org_manilaapis.yaml | 3 - .../bases/manila.openstack.org_manilas.yaml | 12 - ...manila.openstack.org_manilaschedulers.yaml | 3 - .../manila.openstack.org_manilashares.yaml | 3 - controllers/manila_controller.go | 60 +++-- controllers/manilaapi_controller.go | 232 +++++++++--------- controllers/manilascheduler_controller.go | 223 ++++++++--------- controllers/manilashare_controller.go | 229 ++++++++--------- pkg/manila/const.go | 9 + pkg/manila/dbsync.go | 66 +++-- pkg/manila/initcontainer.go | 128 ---------- pkg/manila/volumes.go | 58 ++--- pkg/manilaapi/deployment.go | 27 -- pkg/manilaapi/volumes.go | 40 +-- pkg/manilascheduler/statefulset.go | 27 -- pkg/manilascheduler/volumes.go | 39 +-- pkg/manilashare/statefulset.go | 28 --- pkg/manilashare/volumes.go | 44 +--- templates/common/common.sh | 36 --- templates/manila/bin/healthcheck.sh | 17 ++ templates/manila/bin/init.sh | 104 -------- .../config/{manila.conf => 00-config.conf} | 8 +- templates/manila/config/db-sync-config.json | 10 +- templates/manila/config/logging.conf | 34 --- .../manila/config/manila-api-config.json | 10 +- .../config/manila-scheduler-config.json | 16 +- .../manila/config/manila-share-config.json | 16 +- tests/functional/manila_controller_test.go | 33 +-- tests/functional/manila_test_data.go | 8 +- .../common/assert_sample_deployment.yaml | 4 - 37 files changed, 496 insertions(+), 1065 deletions(-) delete mode 100644 pkg/manila/initcontainer.go delete mode 100755 templates/common/common.sh create mode 100755 templates/manila/bin/healthcheck.sh delete mode 100755 templates/manila/bin/init.sh rename templates/manila/config/{manila.conf => 00-config.conf} (86%) delete mode 100644 templates/manila/config/logging.conf diff --git a/api/bases/manila.openstack.org_manilaapis.yaml b/api/bases/manila.openstack.org_manilaapis.yaml index 05ed8a4a..bbe54763 100644 --- a/api/bases/manila.openstack.org_manilaapis.yaml +++ b/api/bases/manila.openstack.org_manilaapis.yaml @@ -56,9 +56,6 @@ spec: type: string debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/api/bases/manila.openstack.org_manilas.yaml b/api/bases/manila.openstack.org_manilas.yaml index 36738f8a..b70d307f 100644 --- a/api/bases/manila.openstack.org_manilas.yaml +++ b/api/bases/manila.openstack.org_manilas.yaml @@ -46,9 +46,6 @@ spec: type: string debug: properties: - dbInitContainer: - default: false - type: boolean dbSync: default: false type: boolean @@ -826,9 +823,6 @@ spec: type: array debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean @@ -923,9 +917,6 @@ spec: type: array debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean @@ -996,9 +987,6 @@ spec: type: array debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/api/bases/manila.openstack.org_manilaschedulers.yaml b/api/bases/manila.openstack.org_manilaschedulers.yaml index dea70450..56210236 100644 --- a/api/bases/manila.openstack.org_manilaschedulers.yaml +++ b/api/bases/manila.openstack.org_manilaschedulers.yaml @@ -56,9 +56,6 @@ spec: type: string debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/api/bases/manila.openstack.org_manilashares.yaml b/api/bases/manila.openstack.org_manilashares.yaml index 089710c1..28061fa1 100644 --- a/api/bases/manila.openstack.org_manilashares.yaml +++ b/api/bases/manila.openstack.org_manilashares.yaml @@ -56,9 +56,6 @@ spec: type: string debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index fee012a6..47a21256 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -79,7 +79,7 @@ type ManilaServiceTemplate struct { // +kubebuilder:default="# add your customization here" // CustomServiceConfig - customize the service config using this parameter to change service defaults, // or overwrite rendered information using raw OpenStack config format. The content gets added to - // to /etc//.conf.d directory as custom.conf file. + // to /etc//.conf.d directory a custom config file. CustomServiceConfig string `json:"customServiceConfig,omitempty"` // +kubebuilder:validation:Optional @@ -118,10 +118,6 @@ type PasswordSelector struct { // ManilaDebug indicates whether certain stages of Manila deployment should // pause in debug mode type ManilaDebug struct { - // +kubebuilder:validation:Optional - // +kubebuilder:default=false - // dbInitContainer enable debug (waits until /tmp/stop-init-container disappears) - DBInitContainer bool `json:"dbInitContainer,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=false // dbSync enable debug @@ -131,10 +127,6 @@ type ManilaDebug struct { // ManilaServiceDebug indicates whether certain stages of Manila service // deployment should pause in debug mode type ManilaServiceDebug struct { - // +kubebuilder:validation:Optional - // +kubebuilder:default=false - // initContainer enable debug (waits until /tmp/stop-init-container disappears) - InitContainer bool `json:"initContainer,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=false // service enable debug diff --git a/api/v1beta1/manila_types.go b/api/v1beta1/manila_types.go index c54143b7..9f734dfb 100644 --- a/api/v1beta1/manila_types.go +++ b/api/v1beta1/manila_types.go @@ -60,7 +60,7 @@ type ManilaSpec struct { // +kubebuilder:default="# add your customization here" // CustomServiceConfig - customize the service config for all Manila services using this parameter to change service defaults, // or overwrite rendered information using raw OpenStack config format. The content gets added to - // to /etc//.conf.d directory as custom.conf file. + // to /etc//.conf.d directory a custom config file. CustomServiceConfig string `json:"customServiceConfig,omitempty"` // +kubebuilder:validation:Optional diff --git a/api/v1beta1/manilashare_types.go b/api/v1beta1/manilashare_types.go index 3bbd8a1c..b46fedb6 100644 --- a/api/v1beta1/manilashare_types.go +++ b/api/v1beta1/manilashare_types.go @@ -106,7 +106,6 @@ func init() { SchemeBuilder.Register(&ManilaShare{}, &ManilaShareList{}) } - // IsReady - returns true if ManilaShare is reconciled successfully func (instance ManilaShare) IsReady() bool { return instance.Status.Conditions.IsTrue(condition.ReadyCondition) diff --git a/config/crd/bases/manila.openstack.org_manilaapis.yaml b/config/crd/bases/manila.openstack.org_manilaapis.yaml index 05ed8a4a..bbe54763 100644 --- a/config/crd/bases/manila.openstack.org_manilaapis.yaml +++ b/config/crd/bases/manila.openstack.org_manilaapis.yaml @@ -56,9 +56,6 @@ spec: type: string debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/config/crd/bases/manila.openstack.org_manilas.yaml b/config/crd/bases/manila.openstack.org_manilas.yaml index 36738f8a..b70d307f 100644 --- a/config/crd/bases/manila.openstack.org_manilas.yaml +++ b/config/crd/bases/manila.openstack.org_manilas.yaml @@ -46,9 +46,6 @@ spec: type: string debug: properties: - dbInitContainer: - default: false - type: boolean dbSync: default: false type: boolean @@ -826,9 +823,6 @@ spec: type: array debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean @@ -923,9 +917,6 @@ spec: type: array debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean @@ -996,9 +987,6 @@ spec: type: array debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/config/crd/bases/manila.openstack.org_manilaschedulers.yaml b/config/crd/bases/manila.openstack.org_manilaschedulers.yaml index dea70450..56210236 100644 --- a/config/crd/bases/manila.openstack.org_manilaschedulers.yaml +++ b/config/crd/bases/manila.openstack.org_manilaschedulers.yaml @@ -56,9 +56,6 @@ spec: type: string debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/config/crd/bases/manila.openstack.org_manilashares.yaml b/config/crd/bases/manila.openstack.org_manilashares.yaml index 089710c1..28061fa1 100644 --- a/config/crd/bases/manila.openstack.org_manilashares.yaml +++ b/config/crd/bases/manila.openstack.org_manilashares.yaml @@ -56,9 +56,6 @@ spec: type: string debug: properties: - initContainer: - default: false - type: boolean service: default: false type: boolean diff --git a/controllers/manila_controller.go b/controllers/manila_controller.go index 4845fd41..bcb6586e 100644 --- a/controllers/manila_controller.go +++ b/controllers/manila_controller.go @@ -26,7 +26,6 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -258,7 +257,7 @@ func (r *ManilaReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&manilav1beta1.ManilaShare{}). Owns(&rabbitmqv1.TransportURL{}). Owns(&batchv1.Job{}). - Owns(&corev1.ConfigMap{}). + Owns(&corev1.Secret{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). @@ -429,7 +428,7 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila } // ConfigMap - configMapVars := make(map[string]env.Setter) + configVars := make(map[string]env.Setter) // // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created @@ -488,7 +487,7 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila err.Error())) return ctrl.Result{}, err } - configMapVars[ospSecret.Name] = env.SetValue(hash) + configVars[ospSecret.Name] = env.SetValue(hash) instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) // run check OpenStack secret - end @@ -500,12 +499,12 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila common.AppSelector: manila.ServiceName, } // - // create Configmap required for manila input + // create Config required for Manila input // - %-scripts configmap holding scripts to e.g. bootstrap the service // - %-config configmap holding minimal manila config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateServiceConfigMaps(ctx, helper, instance, &configMapVars, serviceLabels) + err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -520,7 +519,7 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -534,7 +533,7 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila // so we need to return and reconcile again return ctrl.Result{}, nil } - // Create ConfigMaps and Secrets - end + // Create Service Config and Secrets - end instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) @@ -718,8 +717,7 @@ func (r *ManilaReconciler) reconcileUpgrade(ctx context.Context, instance *manil } // generateServiceConfigMaps - create create configmaps which hold scripts and service configuration -// TODO add DefaultConfigOverwrite -func (r *ManilaReconciler) generateServiceConfigMaps( +func (r *ManilaReconciler) generateServiceConfig( ctx context.Context, h *helper.Helper, instance *manilav1beta1.Manila, @@ -727,19 +725,18 @@ func (r *ManilaReconciler) generateServiceConfigMaps( serviceLabels map[string]string, ) error { // - // create Configmap/Secret required for manila input + // create Secret required for manila input // - %-scripts configmap holding scripts to e.g. bootstrap the service // - %-config configmap holding minimal manila config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the ospSecret via the init container // - cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) + labels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) // customData hold any customization for the service. // custom.conf is going to /etc//.conf.d // all other files get placed into /etc/ to allow overwrite of e.g. policy.json - // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{manila.CustomConfigFileName: instance.Spec.CustomServiceConfig} for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data @@ -758,19 +755,38 @@ func (r *ManilaReconciler) generateServiceConfigMaps( return err } - templateParameters := make(map[string]interface{}) - templateParameters["ServiceUser"] = instance.Spec.ServiceUser - templateParameters["KeystonePublicURL"] = keystonePublicURL - templateParameters["KeystoneInternalURL"] = keystoneInternalURL + ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Secret, instance.Namespace) + if err != nil { + return err + } + + transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + return err + } + + //templateParameters := make(map[string]interface{}) + templateParameters := map[string]interface{}{ + "ServiceUser": instance.Spec.ServiceUser, + "ServicePassword": string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]), + "KeystonePublicURL": keystonePublicURL, + "KeystoneInternalURL": keystoneInternalURL, + "TransportURL": string(transportURLSecret.Data["transport_url"]), + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s", + instance.Spec.DatabaseUser, + string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]), + instance.Status.DatabaseHostname, + manila.DatabaseName), + } - cms := []util.Template{ + configTemplates := []util.Template{ // ScriptsConfigMap { Name: fmt.Sprintf("%s-scripts", instance.Name), Namespace: instance.Namespace, Type: util.TemplateTypeScripts, InstanceType: instance.Kind, - Labels: cmLabels, + Labels: labels, }, // ConfigMap { @@ -780,11 +796,11 @@ func (r *ManilaReconciler) generateServiceConfigMaps( InstanceType: instance.Kind, CustomData: customData, ConfigOptions: templateParameters, - Labels: cmLabels, + Labels: labels, }, } - return configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars) + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) } // createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart diff --git a/controllers/manilaapi_controller.go b/controllers/manilaapi_controller.go index 159632d4..7441629b 100644 --- a/controllers/manilaapi_controller.go +++ b/controllers/manilaapi_controller.go @@ -33,7 +33,6 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/deployment" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" @@ -81,7 +80,6 @@ var ( //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilaapis,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilaapis/status,verbs=get;update;patch //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilaapis/finalizers,verbs=update -// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete @@ -209,7 +207,7 @@ func (r *ManilaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for changes to any CustomServiceConfigSecrets. Global secrets // (e.g. TransportURLSecret) are handled by the top Manila controller. - svcSecretFn := func(o client.Object) []reconcile.Request { + secretFn := func(o client.Object) []reconcile.Request { var namespace string = o.GetNamespace() var secretName string = o.GetName() result := []reconcile.Request{} @@ -223,49 +221,32 @@ func (r *ManilaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { r.Log.Error(err, "Unable to retrieve API CRs %v") return nil } - for _, cr := range apis.Items { - for _, v := range cr.Spec.CustomServiceConfigSecrets { - if v == secretName { + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(manila.ServiceName))]; ok { + for _, cr := range apis.Items { + // return reconcile event for the CR where the owner label AND the parentCinderName matches + if l == manila.GetOwningManilaName(&cr) { + // return namespace and Name of CR name := client.ObjectKey{ Namespace: namespace, Name: cr.Name, } - r.Log.Info(fmt.Sprintf("Secret %s is used by Manila CR %s", secretName, cr.Name)) + r.Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + result = append(result, reconcile.Request{NamespacedName: name}) } } } - if len(result) > 0 { - return result - } - return nil - } - // watch for configmap where the CM owner label AND the CR.Spec.ManagingCrName label matches - configMapFn := func(o client.Object) []reconcile.Request { - result := []reconcile.Request{} - - // get all API CRs - apis := &manilav1beta1.ManilaAPIList{} - listOpts := []client.ListOption{ - client.InNamespace(o.GetNamespace()), - } - if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { - r.Log.Error(err, "Unable to retrieve API CRs %v") - return nil - } - - label := o.GetLabels() - // TODO: Just trying to verify that the CM is owned by this CR's managing CR - if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(manila.ServiceName))]; ok { - for _, cr := range apis.Items { - // return reconcil event for the CR where the CM owner label AND the parentManilaName matches - if l == manila.GetOwningManilaName(&cr) { - // return namespace and Name of CR + for _, cr := range apis.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { name := client.ObjectKey{ - Namespace: o.GetNamespace(), + Namespace: namespace, Name: cr.Name, } - r.Log.Info(fmt.Sprintf("ConfigMap object %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + r.Log.Info(fmt.Sprintf("Secret %s is used by Manila CR %s", secretName, cr.Name)) result = append(result, reconcile.Request{NamespacedName: name}) } } @@ -286,10 +267,7 @@ func (r *ManilaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Service{}). // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, - handler.EnqueueRequestsFromMapFunc(svcSecretFn)). - // watch the config CMs we don't own - Watches(&source.Kind{Type: &corev1.ConfigMap{}}, - handler.EnqueueRequestsFromMapFunc(configMapFn)). + handler.EnqueueRequestsFromMapFunc(secretFn)). Complete(r) } @@ -531,92 +509,51 @@ func (r *ManilaAPIReconciler) reconcileInit( func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *manilav1beta1.ManilaAPI, helper *helper.Helper) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) - // ConfigMap - configMapVars := make(map[string]env.Setter) + // ConfigVars + configVars := make(map[string]env.Setter) // // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map // - ospSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + ctrlResult, err := r.getSecret(ctx, helper, instance, instance.Spec.TransportURLSecret, &configVars) if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Secret) - } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + return ctrlResult, err } - configMapVars[ospSecret.Name] = env.SetValue(hash) - // run check OpenStack secret - end // // check for required TransportURL secret holding transport URL string // - transportURLSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.TransportURLSecret, instance.Namespace) + ctrlResult, err = r.getSecret(ctx, helper, instance, instance.Spec.TransportURLSecret, &configVars) if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("TransportURL secret %s not found", instance.Spec.TransportURLSecret) - } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + return ctrlResult, err } - configMapVars[transportURLSecret.Name] = env.SetValue(hash) - // run check TransportURL secret - end - // - // check for required Manila config maps that should have been created by parent Manila CR + // check for required service secrets // + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + ctrlResult, err = r.getSecret(ctx, helper, instance, secretName, &configVars) + if err != nil { + return ctrlResult, err + } + } parentManilaName := manila.GetOwningManilaName(instance) - configMaps := []string{ - fmt.Sprintf("%s-scripts", parentManilaName), //ScriptsConfigMap - fmt.Sprintf("%s-config-data", parentManilaName), //ConfigMap + parentSecrets := []string{ + fmt.Sprintf("%s-scripts", parentManilaName), //ScriptsSecret + fmt.Sprintf("%s-config-data", parentManilaName), //ConfigSecret } - _, err = configmap.GetConfigMaps(ctx, helper, instance, configMaps, instance.Namespace, &configMapVars) - if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Could not find all config maps for parent Manila CR %s", parentManilaName) + // + // Create Secrets required as input for the Service and calculate an overall hash of hashes + // + for _, parentSecret := range parentSecrets { + ctrlResult, err = r.getSecret(ctx, helper, instance, parentSecret, &configVars) + if err != nil { + return ctrlResult, err } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err } instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - // run check parent Manila CR config maps - end - - // - // Create ConfigMaps required as input for the Service and calculate an overall hash of hashes - // serviceLabels := map[string]string{ common.AppSelector: manila.ServiceName, @@ -624,9 +561,9 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man } // - // create custom Configmap for this manila-api service + // create Secrets for manila-api service // - err = r.generateServiceConfigMaps(ctx, helper, instance, &configMapVars, serviceLabels) + err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -636,13 +573,12 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man err.Error())) return ctrl.Result{}, err } - // Create ConfigMaps - end // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -657,7 +593,6 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man return ctrl.Result{}, nil } instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - // Create ConfigMaps and Secrets - end // // TODO check when/if Init, Update, or Upgrade should/could be skipped @@ -693,7 +628,7 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -807,9 +742,42 @@ func (r *ManilaAPIReconciler) reconcileUpgrade(ctx context.Context, instance *ma return ctrl.Result{}, nil } -// generateServiceConfigMaps - create custom configmap to hold service-specific config -// TODO add DefaultConfigOverwrite -func (r *ManilaAPIReconciler) generateServiceConfigMaps( +// getSecret - get the specified secret, and add its hash to envVars +func (r *ManilaAPIReconciler) getSecret( + ctx context.Context, + h *helper.Helper, + instance *manilav1beta1.ManilaAPI, + secretName string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + secret, hash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Secret %s not found", secretName) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secret.Name] = env.SetValue(hash) + + return ctrl.Result{}, nil +} + +// generateServiceConfig - create secrets to hold service-specific config +func (r *ManilaAPIReconciler) generateServiceConfig( ctx context.Context, h *helper.Helper, instance *manilav1beta1.ManilaAPI, @@ -817,24 +785,44 @@ func (r *ManilaAPIReconciler) generateServiceConfigMaps( serviceLabels map[string]string, ) error { // - // create custom Configmap for manila-api-specific config input - // - %-config-data configmap holding custom config for the service's manila.conf + // create custom Secret for manila-api-specific config input // - cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) + labels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) // customData hold any customization for the service. - // custom.conf is going to be merged into /etc/manila/manila.conf - // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{manila.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data } - customData[common.CustomServiceConfigFileName] = instance.Spec.CustomServiceConfig + customData[manila.CustomServiceConfigFileName] = instance.Spec.CustomServiceConfig + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cinder controller, and add them to this service specific Secret. + manilaSecretName := manila.GetOwningManilaName(instance) + "-config-data" + manilaSecret, _, err := secret.GetSecret(ctx, h, manilaSecretName, instance.Namespace) + if err != nil { + return err + } + customData[manila.DefaultsConfigFileName] = string(manilaSecret.Data[manila.DefaultsConfigFileName]) + customData[manila.CustomConfigFileName] = string(manilaSecret.Data[manila.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[manila.CustomServiceConfigSecretsFileName] = customSecrets - cms := []util.Template{ + configTemplates := []util.Template{ // Custom ConfigMap { Name: fmt.Sprintf("%s-config-data", instance.Name), @@ -842,11 +830,11 @@ func (r *ManilaAPIReconciler) generateServiceConfigMaps( Type: util.TemplateTypeConfig, InstanceType: instance.Kind, CustomData: customData, - Labels: cmLabels, + Labels: labels, }, } - return configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars) + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) } // createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart diff --git a/controllers/manilascheduler_controller.go b/controllers/manilascheduler_controller.go index 86f1890f..4b4b3cfd 100644 --- a/controllers/manilascheduler_controller.go +++ b/controllers/manilascheduler_controller.go @@ -37,7 +37,6 @@ import ( "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" @@ -82,7 +81,6 @@ type ManilaSchedulerReconciler struct { //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilaschedulers/status,verbs=get;update;patch //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilaschedulers/finalizers,verbs=update // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; -//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch @@ -178,7 +176,7 @@ func (r *ManilaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for changes to any CustomServiceConfigSecrets. Global secrets // (e.g. TransportURLSecret) are handled by the top Manila controller. - svcSecretFn := func(o client.Object) []reconcile.Request { + secretFn := func(o client.Object) []reconcile.Request { var namespace string = o.GetNamespace() var secretName string = o.GetName() result := []reconcile.Request{} @@ -192,46 +190,17 @@ func (r *ManilaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { r.Log.Error(err, "Unable to retrieve API CRs %v") return nil } - for _, cr := range schedulers.Items { - for _, v := range cr.Spec.CustomServiceConfigSecrets { - if v == secretName { - name := client.ObjectKey{ - Namespace: namespace, - Name: cr.Name, - } - r.Log.Info(fmt.Sprintf("Secret %s is used by Manila CR %s", secretName, cr.Name)) - result = append(result, reconcile.Request{NamespacedName: name}) - } - } - } - if len(result) > 0 { - return result - } - return nil - } - // watch for configmap where the CM owner label AND the CR.Spec.ManagingCrName label matches - configMapFn := func(o client.Object) []reconcile.Request { - result := []reconcile.Request{} - - // get all scheduler CRs - schedulers := &manilav1beta1.ManilaSchedulerList{} - listOpts := []client.ListOption{ - client.InNamespace(o.GetNamespace()), - } - if err := r.Client.List(context.Background(), schedulers, listOpts...); err != nil { - r.Log.Error(err, "Unable to retrieve scheduler CRs %v") - return nil - } + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches label := o.GetLabels() - // TODO: Just trying to verify that the CM is owned by this CR's managing CR if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(manila.ServiceName))]; ok { for _, cr := range schedulers.Items { // return reconcil event for the CR where the CM owner label AND the parentName matches if l == manila.GetOwningManilaName(&cr) { // return namespace and Name of CR name := client.ObjectKey{ - Namespace: o.GetNamespace(), + Namespace: namespace, Name: cr.Name, } r.Log.Info(fmt.Sprintf("ConfigMap object %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) @@ -240,6 +209,18 @@ func (r *ManilaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { } } } + for _, cr := range schedulers.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + r.Log.Info(fmt.Sprintf("Secret %s is used by Manila CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } if len(result) > 0 { return result } @@ -252,10 +233,7 @@ func (r *ManilaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, - handler.EnqueueRequestsFromMapFunc(svcSecretFn)). - // watch the config CMs we don't own - Watches(&source.Kind{Type: &corev1.ConfigMap{}}, - handler.EnqueueRequestsFromMapFunc(configMapFn)). + handler.EnqueueRequestsFromMapFunc(secretFn)). Complete(r) } @@ -285,99 +263,58 @@ func (r *ManilaSchedulerReconciler) reconcileNormal(ctx context.Context, instanc r.Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) // ConfigMap - configMapVars := make(map[string]env.Setter) + configVars := make(map[string]env.Setter) // // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map // - ospSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + ctrlResult, err := r.getSecret(ctx, helper, instance, instance.Spec.Secret, &configVars) if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Secret) - } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + return ctrlResult, err } - configMapVars[ospSecret.Name] = env.SetValue(hash) - // run check OpenStack secret - end - // // check for required TransportURL secret holding transport URL string // - transportURLSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.TransportURLSecret, instance.Namespace) + ctrlResult, err = r.getSecret(ctx, helper, instance, instance.Spec.TransportURLSecret, &configVars) if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("TransportURL secret %s not found", instance.Spec.TransportURLSecret) - } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + return ctrlResult, err } - configMapVars[transportURLSecret.Name] = env.SetValue(hash) - // run check TransportURL secret - end // - // check for required Manila config maps that should have been created by parent Manila CR + // check for required service secrets // + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + ctrlResult, err = r.getSecret(ctx, helper, instance, secretName, &configVars) + if err != nil { + return ctrlResult, err + } + } parentManilaName := manila.GetOwningManilaName(instance) - - configMaps := []string{ + parentSecrets := []string{ fmt.Sprintf("%s-scripts", parentManilaName), //ScriptsConfigMap fmt.Sprintf("%s-config-data", parentManilaName), //ConfigMap } - _, err = configmap.GetConfigMaps(ctx, helper, instance, configMaps, instance.Namespace, &configMapVars) - if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Could not find all config maps for parent Manila CR %s", parentManilaName) + for _, parentSecret := range parentSecrets { + ctrlResult, err = r.getSecret(ctx, helper, instance, parentSecret, &configVars) + if err != nil { + return ctrlResult, err } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err } instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - // run check parent Manila CR config maps - end // - // Create ConfigMaps required as input for the Service and calculate an overall hash of hashes + // Create Secrets required as input for the Service and calculate an overall hash of hashes // serviceLabels := map[string]string{ common.AppSelector: manila.ServiceName, common.ComponentSelector: manilascheduler.Component, } // - // create custom Configmap for this manila-scheduler service + // create custom Secrets for manila-scheduler service // - err = r.generateServiceConfigMaps(ctx, helper, instance, &configMapVars, serviceLabels) + err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -387,13 +324,12 @@ func (r *ManilaSchedulerReconciler) reconcileNormal(ctx context.Context, instanc err.Error())) return ctrl.Result{}, err } - // Create ConfigMaps - end // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -408,7 +344,6 @@ func (r *ManilaSchedulerReconciler) reconcileNormal(ctx context.Context, instanc return ctrl.Result{}, nil } instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - // Create ConfigMaps and Secrets - end // // TODO check when/if Init, Update, or Upgrade should/could be skipped @@ -444,7 +379,7 @@ func (r *ManilaSchedulerReconciler) reconcileNormal(ctx context.Context, instanc } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -558,9 +493,43 @@ func (r *ManilaSchedulerReconciler) reconcileUpgrade(ctx context.Context, instan return ctrl.Result{}, nil } -// generateServiceConfigMaps - create custom configmap to hold service-specific config +// getSecret - get the specified secret, and add its hash to envVars +func (r *ManilaSchedulerReconciler) getSecret( + ctx context.Context, + h *helper.Helper, + instance *manilav1beta1.ManilaScheduler, + secretName string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + secret, hash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Secret %s not found", secretName) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secret.Name] = env.SetValue(hash) + + return ctrl.Result{}, nil +} + +// generateServiceConfig - create Secret to hold service-specific config // TODO add DefaultConfigOverwrite -func (r *ManilaSchedulerReconciler) generateServiceConfigMaps( +func (r *ManilaSchedulerReconciler) generateServiceConfig( ctx context.Context, h *helper.Helper, instance *manilav1beta1.ManilaScheduler, @@ -568,24 +537,44 @@ func (r *ManilaSchedulerReconciler) generateServiceConfigMaps( serviceLabels map[string]string, ) error { // - // create custom Configmap for manila-scheduler-specific config input - // - %-config-data configmap holding custom config for the service's manila.conf + // create custom Secret for manila-scheduler-specific config input + // - %-config-data Secret holding custom config for the service's manila.conf // - cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) + labels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) - // customData hold any customization for the service. - // custom.conf is going to be merged into /etc/manila/manila.conf - // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{manila.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data } - customData[common.CustomServiceConfigFileName] = instance.Spec.CustomServiceConfig + customData[manila.CustomServiceConfigFileName] = instance.Spec.CustomServiceConfig + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // Manila controller, and add them to this service specific Secret. + manilaSecretName := manila.GetOwningManilaName(instance) + "-config-data" + manilaSecret, _, err := secret.GetSecret(ctx, h, manilaSecretName, instance.Namespace) + if err != nil { + return err + } + customData[manila.DefaultsConfigFileName] = string(manilaSecret.Data[manila.DefaultsConfigFileName]) + customData[manila.CustomConfigFileName] = string(manilaSecret.Data[manila.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[manila.CustomServiceConfigSecretsFileName] = customSecrets - cms := []util.Template{ + configTemplates := []util.Template{ // Custom ConfigMap { Name: fmt.Sprintf("%s-config-data", instance.Name), @@ -593,11 +582,11 @@ func (r *ManilaSchedulerReconciler) generateServiceConfigMaps( Type: util.TemplateTypeConfig, InstanceType: instance.Kind, CustomData: customData, - Labels: cmLabels, + Labels: labels, }, } - return configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars) + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) } // createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart diff --git a/controllers/manilashare_controller.go b/controllers/manilashare_controller.go index f0d406cf..e06add46 100644 --- a/controllers/manilashare_controller.go +++ b/controllers/manilashare_controller.go @@ -37,7 +37,6 @@ import ( "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" @@ -81,7 +80,6 @@ type ManilaShareReconciler struct { //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilashares,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilashares/status,verbs=get;update;patch //+kubebuilder:rbac:groups=manila.openstack.org,resources=manilashares/finalizers,verbs=update -//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch @@ -179,7 +177,7 @@ func (r *ManilaShareReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for changes to any CustomServiceConfigSecrets. Global secrets // (e.g. TransportURLSecret) are handled by the top Manila controller. - svcSecretFn := func(o client.Object) []reconcile.Request { + secretFn := func(o client.Object) []reconcile.Request { var namespace string = o.GetNamespace() var secretName string = o.GetName() result := []reconcile.Request{} @@ -193,50 +191,30 @@ func (r *ManilaShareReconciler) SetupWithManager(mgr ctrl.Manager) error { r.Log.Error(err, "Unable to retrieve API CRs %v") return nil } - for _, cr := range shares.Items { - for _, v := range cr.Spec.CustomServiceConfigSecrets { - if v == secretName { + + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(manila.ServiceName))]; ok { + for _, cr := range shares.Items { + // return reconcil event for the CR where the CM owner label AND the parentManilaName matches + if l == manila.GetOwningManilaName(&cr) { + // return namespace and Name of CR name := client.ObjectKey{ Namespace: namespace, Name: cr.Name, } - r.Log.Info(fmt.Sprintf("Secret %s is used by Manila CR %s", secretName, cr.Name)) + r.Log.Info(fmt.Sprintf("Secret object %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) result = append(result, reconcile.Request{NamespacedName: name}) } } } - if len(result) > 0 { - return result - } - return nil - } - // watch for configmap where the CM owner label AND the CR.Spec.ManagingCrName label matches - configMapFn := func(o client.Object) []reconcile.Request { - result := []reconcile.Request{} - - // get all manila shares CRs - shares := &manilav1beta1.ManilaShareList{} - listOpts := []client.ListOption{ - client.InNamespace(o.GetNamespace()), - } - if err := r.Client.List(context.Background(), shares, listOpts...); err != nil { - r.Log.Error(err, "Unable to retrieve share CRs %v") - return nil - } - - label := o.GetLabels() - // TODO: Just trying to verify that the CM is owned by this CR's managing CR - if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(manila.ServiceName))]; ok { - for _, cr := range shares.Items { - // return reconcil event for the CR where the CM owner label AND the parentManilaName matches - if l == manila.GetOwningManilaName(&cr) { - // return namespace and Name of CR + for _, cr := range shares.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { name := client.ObjectKey{ - Namespace: o.GetNamespace(), + Namespace: namespace, Name: cr.Name, } - r.Log.Info(fmt.Sprintf("ConfigMap object %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) - + r.Log.Info(fmt.Sprintf("Secret %s is used by Manila CR %s", secretName, cr.Name)) result = append(result, reconcile.Request{NamespacedName: name}) } } @@ -253,10 +231,7 @@ func (r *ManilaShareReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, - handler.EnqueueRequestsFromMapFunc(svcSecretFn)). - // watch the config CMs we don't own - Watches(&source.Kind{Type: &corev1.ConfigMap{}}, - handler.EnqueueRequestsFromMapFunc(configMapFn)). + handler.EnqueueRequestsFromMapFunc(secretFn)). Complete(r) } @@ -285,100 +260,57 @@ func (r *ManilaShareReconciler) reconcileInit( func (r *ManilaShareReconciler) reconcileNormal(ctx context.Context, instance *manilav1beta1.ManilaShare, helper *helper.Helper) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) - // ConfigMap - configMapVars := make(map[string]env.Setter) + // configVars + configVars := make(map[string]env.Setter) // // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map // - ospSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + ctrlResult, err := r.getSecret(ctx, helper, instance, instance.Spec.Secret, &configVars) if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("OpenStack secret %s not found", instance.Spec.Secret) - } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + return ctrlResult, err } - configMapVars[ospSecret.Name] = env.SetValue(hash) - // run check OpenStack secret - end // // check for required TransportURL secret holding transport URL string // - transportURLSecret, hash, err := secret.GetSecret(ctx, helper, instance.Spec.TransportURLSecret, instance.Namespace) + ctrlResult, err = r.getSecret(ctx, helper, instance, instance.Spec.TransportURLSecret, &configVars) if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("TransportURL secret %s not found", instance.Spec.TransportURLSecret) - } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err + return ctrlResult, err } - configMapVars[transportURLSecret.Name] = env.SetValue(hash) - // run check TransportURL secret - end // - // check for required Manila config maps that should have been created by parent Manila CR + // check for required service secrets // + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + ctrlResult, err = r.getSecret(ctx, helper, instance, secretName, &configVars) + if err != nil { + return ctrlResult, err + } + } parentManilaName := manila.GetOwningManilaName(instance) - - configMaps := []string{ - fmt.Sprintf("%s-scripts", parentManilaName), //ScriptsConfigMap - fmt.Sprintf("%s-config-data", parentManilaName), //ConfigMap + parentSecrets := []string{ + fmt.Sprintf("%s-scripts", parentManilaName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentManilaName), // Secret used for ServiceConfig } - _, err = configmap.GetConfigMaps(ctx, helper, instance, configMaps, instance.Namespace, &configMapVars) - if err != nil { - if k8s_errors.IsNotFound(err) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.InputReadyWaitingMessage)) - return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Could not find all config maps for parent Manila CR %s", parentManilaName) + for _, parentSecret := range parentSecrets { + ctrlResult, err = r.getSecret(ctx, helper, instance, parentSecret, &configVars) + if err != nil { + return ctrlResult, err } - instance.Status.Conditions.Set(condition.FalseCondition( - condition.InputReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.InputReadyErrorMessage, - err.Error())) - return ctrl.Result{}, err } instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - // run check parent Manila CR config maps - end - // - // Create ConfigMaps required as input for the Service and calculate an overall hash of hashes - // serviceLabels := map[string]string{ common.AppSelector: manila.ServiceName, common.ComponentSelector: manilashare.Component, } // - // create custom Configmap for this manila-share service + // create service Secrets for manila-share service // - err = r.generateServiceConfigMaps(ctx, helper, instance, &configMapVars, serviceLabels) + err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -388,13 +320,12 @@ func (r *ManilaShareReconciler) reconcileNormal(ctx context.Context, instance *m err.Error())) return ctrl.Result{}, err } - // Create ConfigMaps - end // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -409,7 +340,6 @@ func (r *ManilaShareReconciler) reconcileNormal(ctx context.Context, instance *m return ctrl.Result{}, nil } instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) - // Create ConfigMaps and Secrets - end // // TODO check when/if Init, Update, or Upgrade should/could be skipped @@ -445,7 +375,7 @@ func (r *ManilaShareReconciler) reconcileNormal(ctx context.Context, instance *m } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -560,9 +490,43 @@ func (r *ManilaShareReconciler) reconcileUpgrade(ctx context.Context, instance * return ctrl.Result{}, nil } -// generateServiceConfigMaps - create custom configmap to hold service-specific config +// getSecret - get the specified secret, and add its hash to envVars +func (r *ManilaShareReconciler) getSecret( + ctx context.Context, + h *helper.Helper, + instance *manilav1beta1.ManilaShare, + secretName string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + secret, hash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, fmt.Errorf("Secret %s not found", secretName) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secret.Name] = env.SetValue(hash) + + return ctrl.Result{}, nil +} + +// generateServiceConfig - create custom Secret to hold service-specific config // TODO add DefaultConfigOverwrite -func (r *ManilaShareReconciler) generateServiceConfigMaps( +func (r *ManilaShareReconciler) generateServiceConfig( ctx context.Context, h *helper.Helper, instance *manilav1beta1.ManilaShare, @@ -570,24 +534,45 @@ func (r *ManilaShareReconciler) generateServiceConfigMaps( serviceLabels map[string]string, ) error { // - // create custom Configmap for manila-share-specific config input - // - %-config-data configmap holding custom config for the service's manila.conf + // create custom Secret for manila-share-specific config input + // - %-config-data Secret holding custom config for the service's manila.conf // - cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) + labels := labels.GetLabels(instance, labels.GetGroupLabel(manila.ServiceName), serviceLabels) - // customData hold any customization for the service. - // custom.conf is going to be merged into /etc/manila/manila.conf - // TODO: make sure custom.conf can not be overwritten - customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + customData := map[string]string{manila.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} for key, data := range instance.Spec.DefaultConfigOverwrite { customData[key] = data } - customData[common.CustomServiceConfigFileName] = instance.Spec.CustomServiceConfig + customData[manila.CustomServiceConfigFileName] = instance.Spec.CustomServiceConfig + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // manila controller, and add them to this service specific Secret. + manilaSecretName := manila.GetOwningManilaName(instance) + "-config-data" + manilaSecret, _, err := secret.GetSecret(ctx, h, manilaSecretName, instance.Namespace) + if err != nil { + return err + } + customData[manila.DefaultsConfigFileName] = string(manilaSecret.Data[manila.DefaultsConfigFileName]) + customData[manila.CustomConfigFileName] = string(manilaSecret.Data[manila.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + + customData[manila.CustomServiceConfigSecretsFileName] = customSecrets - cms := []util.Template{ + configTemplates := []util.Template{ // Custom ConfigMap { Name: fmt.Sprintf("%s-config-data", instance.Name), @@ -595,11 +580,11 @@ func (r *ManilaShareReconciler) generateServiceConfigMaps( Type: util.TemplateTypeConfig, InstanceType: instance.Kind, CustomData: customData, - Labels: cmLabels, + Labels: labels, }, } - return configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars) + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) } // createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart diff --git a/pkg/manila/const.go b/pkg/manila/const.go index 44aa847a..b902a673 100644 --- a/pkg/manila/const.go +++ b/pkg/manila/const.go @@ -48,6 +48,15 @@ const ( // Manila is the global ServiceType that refers to all the components deployed // by the manila operator Manila storage.PropagationType = "Manila" + + // DefaultsConfigFileName - + DefaultsConfigFileName = "00-config.conf" + // CustomConfigFileName - + CustomConfigFileName = "01-config.conf" + // CustomServiceConfigFileName - + CustomServiceConfigFileName = "02-config.conf" + // CustomServiceConfigSecretsFileName - + CustomServiceConfigSecretsFileName = "03-config.conf" ) // DbsyncPropagation keeps track of the DBSync Service Propagation Type diff --git a/pkg/manila/dbsync.go b/pkg/manila/dbsync.go index 5452a006..fa644148 100644 --- a/pkg/manila/dbsync.go +++ b/pkg/manila/dbsync.go @@ -10,26 +10,60 @@ import ( ) const ( - // DBSyncCommand - - // FIXME?: The old CN-OSP use of bootstrap.sh does not work here, but not using it might be - // a problem as it has a few conditionals that should perhaps be considered (and they're not here) - DBSyncCommand = "/usr/local/bin/kolla_set_configs && su -s /bin/sh -c \"manila-manage db sync\"" + //DBSyncCommand - + DBSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" ) // DbSyncJob func func DbSyncJob(instance *manilav1.Manila, labels map[string]string, annotations map[string]string) *batchv1.Job { + var config0644AccessMode int32 = 0644 + + // Unlike the individual manila services, the DbSyncJob doesn't need a + // secret that contains all of the config snippets required by every + // service, The two snippet files that it does need (DefaultsConfigFileName + // and CustomConfigFileName) can be extracted from the top-level manila + // config-data secret. + dbSyncVolume := []corev1.Volume{ + { + Name: "db-sync-config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + Items: []corev1.KeyToPath{ + { + Key: DefaultsConfigFileName, + Path: DefaultsConfigFileName, + }, + }, + }, + }, + }, + { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + }, + }, + }, + } dbSyncMounts := []corev1.VolumeMount{ { - Name: "config-data-merged", + Name: "db-sync-config-data", + MountPath: "/etc/manila/manila.conf.d", + ReadOnly: true, + }, + { + Name: "config-data", MountPath: "/var/lib/kolla/config_files/config.json", SubPath: "db-sync-config.json", ReadOnly: true, }, } - dbSyncExtraMounts := []manilav1.ManilaExtraVolMounts{} - args := []string{"-c"} if instance.Spec.Debug.DBSync { args = append(args, common.DebugCommand) @@ -68,28 +102,14 @@ func DbSyncJob(instance *manilav1.Manila, labels map[string]string, annotations RunAsUser: &runAsUser, }, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: append(GetVolumeMounts(dbSyncExtraMounts, DbsyncPropagation), dbSyncMounts...), + VolumeMounts: dbSyncMounts, }, }, - Volumes: GetVolumes(instance.Name, dbSyncExtraMounts, DbsyncPropagation), + Volumes: dbSyncVolume, }, }, }, } - initContainerDetails := APIDetails{ - ContainerImage: instance.Spec.ManilaAPI.ContainerImage, - DatabaseHost: instance.Status.DatabaseHostname, - DatabaseUser: instance.Spec.DatabaseUser, - DatabaseName: DatabaseName, - OSPSecret: instance.Spec.Secret, - DBPasswordSelector: instance.Spec.PasswordSelectors.Database, - UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: GetInitVolumeMounts(dbSyncExtraMounts, DbsyncPropagation), - Debug: instance.Spec.Debug.DBInitContainer, - LoggingConf: false, - } - job.Spec.Template.Spec.InitContainers = InitContainer(initContainerDetails) - return job } diff --git a/pkg/manila/initcontainer.go b/pkg/manila/initcontainer.go deleted file mode 100644 index 1aebd94a..00000000 --- a/pkg/manila/initcontainer.go +++ /dev/null @@ -1,128 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "github.com/openstack-k8s-operators/lib-common/modules/common/env" - - corev1 "k8s.io/api/core/v1" - "strconv" -) - -// APIDetails information -type APIDetails struct { - ContainerImage string - DatabaseHost string - DatabaseUser string - DatabaseName string - OSPSecret string - TransportURLSecret string - DBPasswordSelector string - UserPasswordSelector string - VolumeMounts []corev1.VolumeMount - Privileged bool - Debug bool - LoggingConf bool -} - -const ( - // InitContainerCommand - - InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" -) - -// InitContainer - init container for Manila pods -func InitContainer(init APIDetails) []corev1.Container { - runAsUser := int64(0) - trueVar := true - - securityContext := &corev1.SecurityContext{ - RunAsUser: &runAsUser, - } - - if init.Privileged { - securityContext.Privileged = &trueVar - } - - args := []string{"-c"} - - if init.Debug { - args = append( - args, - "touch /tmp/stop-init-container && while [ -f /tmp/stop-init-container ]; do sleep 5; done", - ) - } else { - args = append(args, InitContainerCommand) - } - - envVars := map[string]env.Setter{} - envVars["DatabaseHost"] = env.SetValue(init.DatabaseHost) - envVars["DatabaseUser"] = env.SetValue(init.DatabaseUser) - envVars["DatabaseName"] = env.SetValue(init.DatabaseName) - envVars["LoggingConf"] = env.SetValue(strconv.FormatBool(init.LoggingConf)) - - envs := []corev1.EnvVar{ - { - Name: "DatabasePassword", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: init.OSPSecret, - }, - Key: init.DBPasswordSelector, - }, - }, - }, - { - Name: "ManilaPassword", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: init.OSPSecret, - }, - Key: init.UserPasswordSelector, - }, - }, - }, - } - - if init.TransportURLSecret != "" { - envTransport := corev1.EnvVar{ - Name: "TransportURL", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: init.TransportURLSecret, - }, - Key: "transport_url", - }, - }, - } - envs = append(envs, envTransport) - } - - envs = env.MergeEnvs(envs, envVars) - - return []corev1.Container{ - { - Name: "init", - Image: init.ContainerImage, - SecurityContext: securityContext, - Command: []string{ - "/bin/bash", - }, - Args: args, - Env: envs, - VolumeMounts: init.VolumeMounts, - }, - } -} diff --git a/pkg/manila/volumes.go b/pkg/manila/volumes.go index 37f8c2da..d766a64b 100644 --- a/pkg/manila/volumes.go +++ b/pkg/manila/volumes.go @@ -10,7 +10,7 @@ import ( // GetVolumes - func GetVolumes(name string, extraVol []manilav1.ManilaExtraVolMounts, svc []storage.PropagationType) []corev1.Volume { var scriptsVolumeDefaultMode int32 = 0755 - var config0640AccessMode int32 = 0640 + var config0644AccessMode int32 = 0644 res := []corev1.Volume{ { @@ -32,31 +32,27 @@ func GetVolumes(name string, extraVol []manilav1.ManilaExtraVolMounts, svc []sto { Name: "scripts", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ + Secret: &corev1.SecretVolumeSource{ DefaultMode: &scriptsVolumeDefaultMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-scripts", - }, + SecretName: name + "-scripts", }, }, }, { Name: "config-data", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &config0640AccessMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-config-data", - }, + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", }, }, }, - { + /*{ Name: "config-data-merged", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, }, - }, + },*/ } for _, exv := range extraVol { @@ -67,35 +63,6 @@ func GetVolumes(name string, extraVol []manilav1.ManilaExtraVolMounts, svc []sto return res } -// GetInitVolumeMounts - Nova Control Plane init task VolumeMounts -func GetInitVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts, svc []storage.PropagationType) []corev1.VolumeMount { - - vm := []corev1.VolumeMount{ - { - Name: "scripts", - MountPath: "/usr/local/bin/container-scripts", - ReadOnly: true, - }, - { - Name: "config-data", - MountPath: "/var/lib/config-data/default", - ReadOnly: true, - }, - { - Name: "config-data-merged", - MountPath: "/var/lib/config-data/merged", - ReadOnly: false, - }, - } - - for _, exv := range extraVol { - for _, vol := range exv.Propagate(svc) { - vm = append(vm, vol.Mounts...) - } - } - return vm -} - // GetVolumeMounts - Nova Control Plane VolumeMounts func GetVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts, svc []storage.PropagationType) []corev1.VolumeMount { res := []corev1.VolumeMount{ @@ -109,16 +76,21 @@ func GetVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts, svc []storage.Pro MountPath: "/etc/localtime", ReadOnly: true, }, + { + Name: "config-data", + MountPath: "/var/lib/config-data/default", + ReadOnly: true, + }, { Name: "scripts", MountPath: "/usr/local/bin/container-scripts", ReadOnly: true, }, - { + /*{ Name: "config-data-merged", MountPath: "/var/lib/config-data/merged", ReadOnly: false, - }, + },*/ } for _, exv := range extraVol { diff --git a/pkg/manilaapi/deployment.go b/pkg/manilaapi/deployment.go index 3bbeeefb..15d1f1cf 100644 --- a/pkg/manilaapi/deployment.go +++ b/pkg/manilaapi/deployment.go @@ -40,13 +40,11 @@ func Deployment( runAsUser := int64(0) livenessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, InitialDelaySeconds: 5, } readinessProbe := &corev1.Probe{ - // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 5, InitialDelaySeconds: 5, @@ -122,7 +120,6 @@ func Deployment( deployment.Spec.Template.Spec.Volumes = GetVolumes( manila.GetOwningManilaName(instance), instance.Name, - instance.Spec.CustomServiceConfigSecrets, instance.Spec.ExtraMounts, ) // If possible two pods of the same service should not @@ -139,29 +136,5 @@ func Deployment( deployment.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - initContainerDetails := manila.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - DatabaseHost: instance.Spec.DatabaseHostname, - DatabaseUser: instance.Spec.DatabaseUser, - DatabaseName: manila.DatabaseName, - OSPSecret: instance.Spec.Secret, - TransportURLSecret: instance.Spec.TransportURLSecret, - DBPasswordSelector: instance.Spec.PasswordSelectors.Database, - UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: GetInitVolumeMounts( - instance.Spec.CustomServiceConfigSecrets, - instance.Spec.ExtraMounts, - ), - Debug: instance.Spec.Debug.InitContainer, - LoggingConf: false, - } - deployment.Spec.Template.Spec.InitContainers = manila.InitContainer(initContainerDetails) - - // TODO: Clean up this hack - // Add custom config for the API Service - envVars = map[string]env.Setter{} - envVars["CustomConf"] = env.SetValue(common.CustomServiceConfigFileName) - deployment.Spec.Template.Spec.InitContainers[0].Env = env.MergeEnvs(deployment.Spec.Template.Spec.InitContainers[0].Env, envVars) - return deployment } diff --git a/pkg/manilaapi/volumes.go b/pkg/manilaapi/volumes.go index 47865288..85fb57ef 100644 --- a/pkg/manilaapi/volumes.go +++ b/pkg/manilaapi/volumes.go @@ -7,54 +7,32 @@ import ( ) // GetVolumes - -func GetVolumes(parentName string, name string, secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.Volume { - var config0640AccessMode int32 = 0640 +func GetVolumes(parentName string, name string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.Volume { + var config0644AccessMode int32 = 0644 apiVolumes := []corev1.Volume{ { Name: "config-data-custom", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &config0640AccessMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-config-data", - }, + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", }, }, }, } - // Mount secrets passed using the `customConfigServiceSecret` parameter - // and they will be rendered as part of the service config - secretConfig, _ := manila.GetConfigSecretVolumes(secretNames) - apiVolumes = append(apiVolumes, secretConfig...) - return append(manila.GetVolumes(parentName, extraVol, manila.ManilaAPIPropagation), apiVolumes...) } -// GetInitVolumeMounts - ManilaAPI init task VolumeMounts -func GetInitVolumeMounts(secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { - - initConfVolumeMount := []corev1.VolumeMount{ +// GetVolumeMounts - ManilaAPI VolumeMounts +func GetVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { + apiVolumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", - MountPath: "/var/lib/config-data/custom", + MountPath: "/etc/manila/manila.conf.d", ReadOnly: true, }, - } - - // Mount secrets passed using the `customConfigServiceSecret` parameter - // and they will be rendered as part of the service config - _, secretConfig := manila.GetConfigSecretVolumes(secretNames) - initConfVolumeMount = append(initConfVolumeMount, secretConfig...) - - return append(manila.GetInitVolumeMounts(extraVol, manila.ManilaAPIPropagation), initConfVolumeMount...) - -} - -// GetVolumeMounts - ManilaAPI VolumeMounts -func GetVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { - apiVolumeMounts := []corev1.VolumeMount{ { Name: "config-data", MountPath: "/var/lib/kolla/config_files/config.json", diff --git a/pkg/manilascheduler/statefulset.go b/pkg/manilascheduler/statefulset.go index 26549f8e..d3beb21f 100644 --- a/pkg/manilascheduler/statefulset.go +++ b/pkg/manilascheduler/statefulset.go @@ -42,7 +42,6 @@ func StatefulSet( manilaUser := int64(42429) manilaGroup := int64(42429) - // TODO until we determine how to properly query for these livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, @@ -145,7 +144,6 @@ func StatefulSet( statefulset.Spec.Template.Spec.Volumes = GetVolumes( manila.GetOwningManilaName(instance), instance.Name, - instance.Spec.CustomServiceConfigSecrets, instance.Spec.ExtraMounts, ) // If possible two pods of the same service should not @@ -162,30 +160,5 @@ func StatefulSet( statefulset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - initContainerDetails := manila.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - DatabaseHost: instance.Spec.DatabaseHostname, - DatabaseUser: instance.Spec.DatabaseUser, - DatabaseName: manila.DatabaseName, - OSPSecret: instance.Spec.Secret, - TransportURLSecret: instance.Spec.TransportURLSecret, - DBPasswordSelector: instance.Spec.PasswordSelectors.Database, - UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: GetInitVolumeMounts( - instance.Spec.CustomServiceConfigSecrets, - instance.Spec.ExtraMounts, - ), - Debug: instance.Spec.Debug.InitContainer, - LoggingConf: true, - } - - statefulset.Spec.Template.Spec.InitContainers = manila.InitContainer(initContainerDetails) - - // TODO: Clean up this hack - // Add custom config for the Scheduler Service - envVars = map[string]env.Setter{} - envVars["CustomConf"] = env.SetValue(common.CustomServiceConfigFileName) - statefulset.Spec.Template.Spec.InitContainers[0].Env = env.MergeEnvs(statefulset.Spec.Template.Spec.InitContainers[0].Env, envVars) - return statefulset } diff --git a/pkg/manilascheduler/volumes.go b/pkg/manilascheduler/volumes.go index 8839b6a0..311afe78 100644 --- a/pkg/manilascheduler/volumes.go +++ b/pkg/manilascheduler/volumes.go @@ -7,53 +7,32 @@ import ( ) // GetVolumes - -func GetVolumes(parentName string, name string, secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.Volume { - var config0640AccessMode int32 = 0640 +func GetVolumes(parentName string, name string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.Volume { + var config0644AccessMode int32 = 0644 schedulerVolumes := []corev1.Volume{ { Name: "config-data-custom", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &config0640AccessMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-config-data", - }, + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", }, }, }, } - // Mount secrets passed using the `customConfigServiceSecret` parameter - // and they will be rendered as part of the service config - secretConfig, _ := manila.GetConfigSecretVolumes(secretNames) - schedulerVolumes = append(schedulerVolumes, secretConfig...) - return append(manila.GetVolumes(parentName, extraVol, manila.ManilaSchedulerPropagation), schedulerVolumes...) } -// GetInitVolumeMounts - ManilaScheduler init task VolumeMounts -func GetInitVolumeMounts(secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { - - initVolumeMount := []corev1.VolumeMount{ +// GetVolumeMounts - ManilaScheduler VolumeMounts +func GetVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { + schedulerVolumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", - MountPath: "/var/lib/config-data/custom", + MountPath: "/etc/manila/manila.conf.d", ReadOnly: true, }, - } - - // Mount secrets passed using the `customConfigServiceSecret` parameter - // and they will be rendered as part of the service config - _, secretConfig := manila.GetConfigSecretVolumes(secretNames) - initVolumeMount = append(initVolumeMount, secretConfig...) - - return append(manila.GetInitVolumeMounts(extraVol, manila.ManilaSchedulerPropagation), initVolumeMount...) -} - -// GetVolumeMounts - ManilaScheduler VolumeMounts -func GetVolumeMounts(extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { - schedulerVolumeMounts := []corev1.VolumeMount{ { Name: "config-data", MountPath: "/var/lib/kolla/config_files/config.json", diff --git a/pkg/manilashare/statefulset.go b/pkg/manilashare/statefulset.go index 8298892c..983b8265 100644 --- a/pkg/manilashare/statefulset.go +++ b/pkg/manilashare/statefulset.go @@ -102,7 +102,6 @@ func StatefulSet( volumeMounts := GetVolumeMounts( instance.Name, - instance.Spec.CustomServiceConfigSecrets, instance.Spec.ExtraMounts, ) @@ -161,7 +160,6 @@ func StatefulSet( statefulset.Spec.Template.Spec.Volumes = GetVolumes( manila.GetOwningManilaName(instance), instance.Name, - instance.Spec.CustomServiceConfigSecrets, instance.Spec.ExtraMounts, ) // If possible two pods of the same service should not @@ -178,31 +176,5 @@ func StatefulSet( statefulset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - initContainerDetails := manila.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - DatabaseHost: instance.Spec.DatabaseHostname, - DatabaseUser: instance.Spec.DatabaseUser, - DatabaseName: manila.DatabaseName, - OSPSecret: instance.Spec.Secret, - TransportURLSecret: instance.Spec.TransportURLSecret, - DBPasswordSelector: instance.Spec.PasswordSelectors.Database, - UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: GetInitVolumeMounts( - instance.Name, - instance.Spec.CustomServiceConfigSecrets, - instance.Spec.ExtraMounts, - ), - Debug: instance.Spec.Debug.InitContainer, - LoggingConf: true, - } - - statefulset.Spec.Template.Spec.InitContainers = manila.InitContainer(initContainerDetails) - - // TODO: Clean up this hack - // Add custom config for the Share Service - envVars = map[string]env.Setter{} - envVars["CustomConf"] = env.SetValue(common.CustomServiceConfigFileName) - statefulset.Spec.Template.Spec.InitContainers[0].Env = env.MergeEnvs(statefulset.Spec.Template.Spec.InitContainers[0].Env, envVars) - return statefulset } diff --git a/pkg/manilashare/volumes.go b/pkg/manilashare/volumes.go index 0755fe52..0fc264be 100644 --- a/pkg/manilashare/volumes.go +++ b/pkg/manilashare/volumes.go @@ -9,8 +9,8 @@ import ( ) // GetVolumes - -func GetVolumes(parentName string, name string, secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.Volume { - var config0640AccessMode int32 = 0640 +func GetVolumes(parentName string, name string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.Volume { + var config0644AccessMode int32 = 0644 var dirOrCreate = corev1.HostPathDirectoryOrCreate shareVolumes := []corev1.Volume{ @@ -26,51 +26,27 @@ func GetVolumes(parentName string, name string, secretNames []string, extraVol [ { Name: "config-data-custom", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &config0640AccessMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: name + "-config-data", - }, + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", }, }, }, } - // Mount secrets passed using the `customConfigServiceSecret` parameter - // and they will be rendered as part of the service config - secretConfig, _ := manila.GetConfigSecretVolumes(secretNames) - shareVolumes = append(shareVolumes, secretConfig...) - // Set the propagation levels for ManilaShare, including the backend name propagation := append(manila.ManilaSharePropagation, storage.PropagationType(strings.TrimPrefix(name, "manila-share-"))) return append(manila.GetVolumes(parentName, extraVol, propagation), shareVolumes...) } -// GetInitVolumeMounts - Manila Share init task -func GetInitVolumeMounts(name string, secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { - - initVolumeMount := []corev1.VolumeMount{ - { - Name: "config-data-custom", - MountPath: "/var/lib/config-data/custom", - ReadOnly: true, - }, - } - - // Mount secrets passed using the `customConfigServiceSecret` parameter - // and they will be rendered as part of the service config - _, secretConfig := manila.GetConfigSecretVolumes(secretNames) - initVolumeMount = append(initVolumeMount, secretConfig...) - - // Set the propagation levels for ManilaShare, including the backend name - propagation := append(manila.ManilaSharePropagation, storage.PropagationType(strings.TrimPrefix(name, "manila-share-"))) - return append(manila.GetInitVolumeMounts(extraVol, propagation), initVolumeMount...) -} - // GetVolumeMounts - Manila Share VolumeMounts -func GetVolumeMounts(name string, secretNames []string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { +func GetVolumeMounts(name string, extraVol []manilav1.ManilaExtraVolMounts) []corev1.VolumeMount { shareVolumeMounts := []corev1.VolumeMount{ { + Name: "config-data-custom", + MountPath: "/etc/manila/manila.conf.d", + ReadOnly: true, + }, { Name: "var-lib-manila", MountPath: "/var/lib/manila", }, diff --git a/templates/common/common.sh b/templates/common/common.sh deleted file mode 100755 index 0e2515eb..00000000 --- a/templates/common/common.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# -# Copyright 2022 Red Hat Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -e - -function merge_config_dir { - echo merge config dir $1 - for conf in $(find $1 -type f);do - conf_base=$(basename $conf) - - # If CFG already exist in ../merged and is not a json file, - # we expect for now it can be merged using crudini. - # Else, just copy the full file. - if [[ -f /var/lib/config-data/merged/${conf_base} && ${conf_base} != *.json && ${conf_base} != nfs_shares ]]; then - echo merging ${conf} into /var/lib/config-data/merged/${conf_base} - crudini --merge /var/lib/config-data/merged/${conf_base} < ${conf} - else - echo copy ${conf} to /var/lib/config-data/merged/ - cp -f ${conf} /var/lib/config-data/merged/ - fi - done -} diff --git a/templates/manila/bin/healthcheck.sh b/templates/manila/bin/healthcheck.sh new file mode 100755 index 00000000..773d1e49 --- /dev/null +++ b/templates/manila/bin/healthcheck.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2020 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +echo "Do nothing atm" diff --git a/templates/manila/bin/init.sh b/templates/manila/bin/init.sh deleted file mode 100755 index 916b00ce..00000000 --- a/templates/manila/bin/init.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# -# Copyright 2020 Red Hat Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -ex - -# This script generates the nova.conf file and copies the result to the -# ephemeral /var/lib/config-data/merged volume. -# -# Secrets are obtained from ENV variables. -export DB=${DatabaseName:-"manila"} -export DBHOST=${DatabaseHost:?"Please specify a DatabaseHost variable."} -export DBUSER=${DatabaseUser:-"manila"} -export DBPASSWORD=${DatabasePassword:?"Please specify a DatabasePassword variable."} -export PASSWORD=${ManilaPassword:?"Please specify a ManilaPassword variable."} -export TRANSPORTURL=${TransportURL:-""} -export LOGGINGCONF=${LoggingConf:-"false"} - -export CUSTOMCONF=${CustomConf:-""} - -DEFAULT_DIR=/var/lib/config-data/default -CUSTOM_DIR=/var/lib/config-data/custom -MERGED_DIR=/var/lib/config-data/merged -SVC_CFG=/etc/manila/manila.conf -SVC_CFG_MERGED=/var/lib/config-data/merged/manila.conf -SVC_CFG_MERGED_DIR=${MERGED_DIR}/manila.conf.d -SVC_CFG_LOGGING=/etc/manila/logging.conf - -mkdir -p ${SVC_CFG_MERGED_DIR} - -cp ${DEFAULT_DIR}/* ${MERGED_DIR} - -# Save the default service config from container image as manila.conf.sample, -# and create a small manila.conf file that directs people to files in -# manila.conf.d. -cp -a ${SVC_CFG} ${SVC_CFG_MERGED}.sample -cat < ${SVC_CFG_MERGED} -# Service configuration snippets are stored in the manila.conf.d subdirectory. -EOF - -cp ${DEFAULT_DIR}/manila.conf ${SVC_CFG_MERGED_DIR}/00-default.conf - -# Generate 01-deployment-secrets.conf -DEPLOYMENT_SECRETS=${SVC_CFG_MERGED_DIR}/01-deployment-secrets.conf -if [ -n "$TRANSPORTURL" ]; then - cat < ${DEPLOYMENT_SECRETS} -[DEFAULT] -transport_url = ${TRANSPORTURL} - -EOF -fi - -# TODO: service token -cat <> ${DEPLOYMENT_SECRETS} -[database] -connection = mysql+pymysql://${DBUSER}:${DBPASSWORD}@${DBHOST}/${DB} - -[keystone_authtoken] -password = ${PASSWORD} - -[nova] -password = ${PASSWORD} - -[service_user] -password = ${PASSWORD} -EOF - -if [ -f ${DEFAULT_DIR}/custom.conf ]; then - cp ${DEFAULT_DIR}/custom.conf ${SVC_CFG_MERGED_DIR}/02-global.conf -fi - -if [ -f ${CUSTOM_DIR}/custom.conf ]; then - cp ${CUSTOM_DIR}/custom.conf ${SVC_CFG_MERGED_DIR}/03-service.conf -fi - -if [ "$LOGGINGCONF" == "true" ]; then -cat <> ${SVC_CFG_MERGED_DIR}/03-service.conf - -[DEFAULT] -log_config_append=${SVC_CFG_LOGGING} -EOF -fi - -SECRET_FILES="$(ls /var/lib/config-data/secret-*/* 2>/dev/null || true)" -if [ -n "${SECRET_FILES}" ]; then - cat ${SECRET_FILES} > ${SVC_CFG_MERGED_DIR}/04-secrets.conf -fi - -# Probes cannot run kolla_set_configs because it uses the 'manila' uid -# and gid and doesn't have permission to make files be owned by root. -# This means the probe must use files in the "merged" location, and the -# files must be readable by 'manila'. -chown -R :manila ${SVC_CFG_MERGED_DIR} diff --git a/templates/manila/config/manila.conf b/templates/manila/config/00-config.conf similarity index 86% rename from templates/manila/config/manila.conf rename to templates/manila/config/00-config.conf index d72b04ec..1db24f9b 100644 --- a/templates/manila/config/manila.conf +++ b/templates/manila/config/00-config.conf @@ -1,4 +1,5 @@ [DEFAULT] +transport_url = {{ .TransportURL }} state_path=/var/lib/manila enabled_share_backends = alpha host=hostgroup @@ -6,7 +7,6 @@ storage_availability_zone=nova default_share_type=default rootwrap_config=/etc/manila/rootwrap.conf auth_strategy=keystone -log_dir=/var/log/manila control_exchange=openstack api_paste_config=/etc/manila/api-paste.ini @@ -15,6 +15,9 @@ api_paste_config=/etc/manila/api-paste.ini [database] max_retries=-1 +connection = {{ .DatabaseConnection }} +max_retries = -1 +db_max_retries = -1 [glance] [healthcheck] @@ -27,6 +30,7 @@ project_domain_name = Default user_domain_name = Default project_name = service username = {{ .ServiceUser }} +password = {{ .ServicePassword }} interface = internal [neutron] @@ -36,12 +40,14 @@ project_domain_name=Default project_name=service user_domain_name=Default username = {{ .ServiceUser }} +password = {{ .ServicePassword }} [nova] interface = internal auth_type = password auth_url = {{ .KeystoneInternalURL }} username = {{ .ServiceUser }} +password = {{ .ServicePassword }} user_domain_name = Default project_name = service project_domain_name = Default diff --git a/templates/manila/config/db-sync-config.json b/templates/manila/config/db-sync-config.json index 29dc0100..8eb37b50 100644 --- a/templates/manila/config/db-sync-config.json +++ b/templates/manila/config/db-sync-config.json @@ -1,11 +1,3 @@ { - "command": "/usr/local/bin/container-scripts/bootstrap.sh", - "config_files": [ - { - "source": "/var/lib/config-data/merged/manila.conf.d", - "dest": "/etc/manila/manila.conf.d", - "owner": "manila", - "perm": "0700" - } - ] + "command": "/usr/bin/manila-manage --config-dir /etc/manila/manila.conf.d db sync" } diff --git a/templates/manila/config/logging.conf b/templates/manila/config/logging.conf deleted file mode 100644 index 75cd16df..00000000 --- a/templates/manila/config/logging.conf +++ /dev/null @@ -1,34 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=stdout - -[formatters] -keys=normal - - -########### -# Loggers # -########### - -[logger_root] -level=WARNING -handlers=stdout - -################ -# Log Handlers # -################ - -[handler_stdout] -class=StreamHandler -level=WARNING -formatter=normal -args=(sys.stdout,) - -################## -# Log Formatters # -################## - -[formatter_normal] -format=(%(name)s): %(asctime)s %(levelname)s %(message)s diff --git a/templates/manila/config/manila-api-config.json b/templates/manila/config/manila-api-config.json index 3f741e38..6632b578 100644 --- a/templates/manila/config/manila-api-config.json +++ b/templates/manila/config/manila-api-config.json @@ -2,19 +2,13 @@ "command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [ { - "source": "/var/lib/config-data/merged/manila.conf.d", - "dest": "/etc/manila/manila.conf.d", - "owner": "root:manila", - "perm": "0755" - }, - { - "source": "/var/lib/config-data/merged/httpd.conf", + "source": "/var/lib/config-data/default/httpd.conf", "dest": "/etc/httpd/conf/httpd.conf", "owner": "root", "perm": "0644" }, { - "source": "/var/lib/config-data/merged/10-manila_wsgi.conf", + "source": "/var/lib/config-data/default/10-manila_wsgi.conf", "dest": "/etc/httpd/conf.d/10-manila_wsgi.conf", "owner": "root", "perm": "0644" diff --git a/templates/manila/config/manila-scheduler-config.json b/templates/manila/config/manila-scheduler-config.json index e2d3239e..2be62daf 100644 --- a/templates/manila/config/manila-scheduler-config.json +++ b/templates/manila/config/manila-scheduler-config.json @@ -1,17 +1,3 @@ { - "command": "/usr/bin/manila-scheduler --config-dir /etc/manila/manila.conf.d", - "config_files": [ - { - "source": "/var/lib/config-data/merged/manila.conf.d", - "dest": "/etc/manila/manila.conf.d", - "owner": "root:manila", - "perm": "0750" - }, - { - "source": "/var/lib/config-data/merged/logging.conf", - "dest": "/etc/manila/logging.conf", - "owner": "root:manila", - "perm": "0600" - } - ] + "command": "/usr/bin/manila-scheduler --config-dir /etc/manila/manila.conf.d" } diff --git a/templates/manila/config/manila-share-config.json b/templates/manila/config/manila-share-config.json index c3906927..c1f0fb7c 100644 --- a/templates/manila/config/manila-share-config.json +++ b/templates/manila/config/manila-share-config.json @@ -1,17 +1,3 @@ { - "command": "/usr/bin/manila-share --config-dir /etc/manila/manila.conf.d", - "config_files": [ - { - "source": "/var/lib/config-data/merged/manila.conf.d", - "dest": "/etc/manila/manila.conf.d", - "owner": "root:manila", - "perm": "0750" - }, - { - "source": "/var/lib/config-data/merged/logging.conf", - "dest": "/etc/manila/logging.conf", - "owner": "root:manila", - "perm": "0600" - } - ] + "command": "/usr/bin/manila-share --config-dir /etc/manila/manila.conf.d" } diff --git a/tests/functional/manila_controller_test.go b/tests/functional/manila_controller_test.go index c6221530..8c75a0d1 100644 --- a/tests/functional/manila_controller_test.go +++ b/tests/functional/manila_controller_test.go @@ -71,11 +71,6 @@ var _ = Describe("Manila controller", func() { return GetManila(manilaTest.Instance).Finalizers }, timeout, interval).Should(ContainElement("Manila")) }) - It("should not create a config map", func() { - Eventually(func() []corev1.ConfigMap { - return th.ListConfigMaps(manilaTest.ManilaConfigMapData.Name).Items - }, timeout, interval).Should(BeEmpty()) - }) It("creates service account, role and rolebindig", func() { th.ExpectCondition( @@ -117,7 +112,6 @@ var _ = Describe("Manila controller", func() { corev1.ConditionUnknown, ) }) - // should create 01-deployment.conf secret }) When("Manila DB is created", func() { BeforeEach(func() { @@ -197,11 +191,11 @@ var _ = Describe("Manila controller", func() { It("should create config-data and scripts ConfigMaps", func() { keystoneAPI := th.CreateKeystoneAPI(manilaTest.Instance.Namespace) DeferCleanup(th.DeleteKeystoneAPI, keystoneAPI) - Eventually(func() corev1.ConfigMap { - return *th.GetConfigMap(manilaTest.ManilaConfigMapData) + Eventually(func() corev1.Secret { + return th.GetSecret(manilaTest.ManilaConfigSecret) }, timeout, interval).ShouldNot(BeNil()) - Eventually(func() corev1.ConfigMap { - return *th.GetConfigMap(manilaTest.ManilaConfigMapScripts) + Eventually(func() corev1.Secret { + return th.GetSecret(manilaTest.ManilaConfigScripts) }, timeout, interval).ShouldNot(BeNil()) }) }) @@ -291,25 +285,8 @@ var _ = Describe("Manila controller", func() { mDB = th.GetMariaDBDatabase(manilaTest.Instance) Expect(mDB.Finalizers).NotTo(ContainElement("Manila")) }) - It("removes the ConfigMaps", func() { - keystoneAPI := th.CreateKeystoneAPI(manilaTest.Instance.Namespace) - DeferCleanup(th.DeleteKeystoneAPI, keystoneAPI) - - Eventually(func() corev1.ConfigMap { - return *th.GetConfigMap(manilaTest.ManilaConfigMapData) - }, timeout, interval).ShouldNot(BeNil()) - Eventually(func() corev1.ConfigMap { - return *th.GetConfigMap(manilaTest.ManilaConfigMapScripts) - }, timeout, interval).ShouldNot(BeNil()) - Eventually(func() []corev1.ConfigMap { - return th.ListConfigMaps(manilaTest.ManilaConfigMapData.Name).Items - }, timeout, interval).Should(BeEmpty()) - Eventually(func() []corev1.ConfigMap { - return th.ListConfigMaps(manilaTest.ManilaConfigMapScripts.Name).Items - }, timeout, interval).Should(BeEmpty()) - }) }) - When("Manila CR instance is built w/ NAD", func() { + When("Manila CR instance is built with NAD", func() { BeforeEach(func() { nad := th.CreateNetworkAttachmentDefinition(manilaTest.InternalAPINAD) DeferCleanup(th.DeleteInstance, nad) diff --git a/tests/functional/manila_test_data.go b/tests/functional/manila_test_data.go index 18ac2d08..cdedf836 100644 --- a/tests/functional/manila_test_data.go +++ b/tests/functional/manila_test_data.go @@ -38,8 +38,8 @@ type ManilaTestData struct { ManilaKeystoneEndpoint types.NamespacedName ManilaServicePublic types.NamespacedName ManilaServiceInternal types.NamespacedName - ManilaConfigMapData types.NamespacedName - ManilaConfigMapScripts types.NamespacedName + ManilaConfigSecret types.NamespacedName + ManilaConfigScripts types.NamespacedName ManilaAPI types.NamespacedName ManilaScheduler types.NamespacedName ManilaShares []types.NamespacedName @@ -93,11 +93,11 @@ func GetManilaTestData(manilaName types.NamespacedName) ManilaTestData { Namespace: manilaName.Namespace, Name: fmt.Sprintf("manila-%s-transport", manilaName.Name), }, - ManilaConfigMapData: types.NamespacedName{ + ManilaConfigSecret: types.NamespacedName{ Namespace: manilaName.Namespace, Name: fmt.Sprintf("%s-%s", manilaName.Name, "config-data"), }, - ManilaConfigMapScripts: types.NamespacedName{ + ManilaConfigScripts: types.NamespacedName{ Namespace: manilaName.Namespace, Name: fmt.Sprintf("%s-%s", manilaName.Name, "scripts"), }, diff --git a/tests/kuttl/common/assert_sample_deployment.yaml b/tests/kuttl/common/assert_sample_deployment.yaml index c8fddc6f..3c7ba713 100644 --- a/tests/kuttl/common/assert_sample_deployment.yaml +++ b/tests/kuttl/common/assert_sample_deployment.yaml @@ -17,21 +17,18 @@ spec: databaseInstance: openstack databaseUser: manila debug: - dbInitContainer: false dbSync: false manilaAPI: customServiceConfig: | [DEFAULT] enabled_share_protocols = cephfs debug: - initContainer: false service: false replicas: 1 resources: {} manilaScheduler: customServiceConfig: '# add your customization here' debug: - initContainer: false service: false replicas: 1 resources: {} @@ -51,7 +48,6 @@ spec: cephfs_cluster_name=ceph cephfs_protocol_helper_type=CEPHFS debug: - initContainer: false service: false passwordSelectors: database: ManilaDatabasePassword