Rework glance config process

This patch represents an implementation of the proposal [1] and aligns
the glance-operator with the work already done in Cinder and the other
storage operators. There are a few relevant changes in the bootstrap
process of Glance, in particular:

- It stops using an init container to generate the snippet files that
  configure each glanceAPI service. The logic that was previously
  implemented in the Init Container has been moved to the Controllers,
  where config files are generated and stored in k8s Secrets. Init
  Containers are fully removed from the bootstrap process;
- It uses Kolla to copy the generated content to the target directory
  /etc/glance/glance.conf.d, which is used by each API service to run;
- The relevant content, including scripts, previously stored in a
  ConfigMap, are now stored in a corresponding k8s Secret, which is
  mounted to the GlanceAPI deployment Pods (hence no additional calls
  to GetVolume/GetVolumeMounts are required);
- A total of 4 config snippet files are generated:
  - 00-config.conf contains global settings that are common to every
    GlanceAPI Pod, including ones that are derived from deployment
    secrets (e.g. database password, etc.)
  - 01-config.conf contains the global customServiceConfig settings
    that apply to every GlanceAPI service.
  - 02-config.conf contains the customServiceConfig settings that
    are specific to each service.
  - 03-config.conf contains secrets specified by each service's
    customServiceConfigSecrets.
- glance-image-import.conf has been removed, and the default config has
  been added to 00-config.conf, which is the common file generated by
  the umbrella Glance controller;
- logging.conf has been removed as it's no longer required in the switch
  to a side container approach for logging purposes;
- functional tests are aligned to the use of k8s Secrets instead of the
  old pattern based on ConfigMaps;
- kuttl tests are updated and the 'initContainer' has been removed.
- dbsync now mounts only the required files (a minimal 00-config.conf)
  and a db-sync-config.json containing the command run through kolla.

[1] openstack-k8s-operators/dev-docs#31

Signed-off-by: Francesco Pantano <[email protected]>

config/rbac/role.yaml

@@ -51,18 +51,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - ""
   resources:

controllers/glance_common.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2022.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func GenerateConfigsGeneric(
+	ctx context.Context, h *helper.Helper,
+	instance client.Object,
+	envVars *map[string]env.Setter,
+	templateParameters map[string]interface{},
+	customData map[string]string,
+	cmLabels map[string]string,
+	scripts bool,
+) error {
+
+	cms := []util.Template{
+		// Templates where the GlanceAPI config is stored
+		{
+			Name:          fmt.Sprintf("%s-config-data", instance.GetName()),
+			Namespace:     instance.GetNamespace(),
+			Type:          util.TemplateTypeConfig,
+			InstanceType:  instance.GetObjectKind().GroupVersionKind().Kind,
+			ConfigOptions: templateParameters,
+			CustomData:    customData,
+			Labels:        cmLabels,
+		},
+	}
+	if scripts {
+		cms = append(cms, util.Template{
+			Name:         fmt.Sprintf("%s-scripts", instance.GetName()),
+			Namespace:    instance.GetNamespace(),
+			Type:         util.TemplateTypeScripts,
+			InstanceType: instance.GetObjectKind().GroupVersionKind().Kind,
+			Labels:       cmLabels,
+		})
+	}
+	return secret.EnsureSecrets(ctx, h, instance, cms, envVars)
+}

controllers/glance_controller.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	rbacv1 "k8s.io/api/rbac/v1"
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -37,7 +38,6 @@ import (
 	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
-	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/endpoint"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -191,7 +191,6 @@ func (r *GlanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&keystonev1.KeystoneService{}).
 		Owns(&corev1.PersistentVolumeClaim{}).
 		Owns(&batchv1.Job{}).
-		Owns(&corev1.ConfigMap{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.Role{}).
@@ -491,8 +490,12 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
 		return rbacResult, nil
 	}
 
-	// ConfigMap
-	configMapVars := make(map[string]env.Setter)
+	configVars := make(map[string]env.Setter)
+
+	// ServiceLabels
+	serviceLabels := map[string]string{
+		common.AppSelector: glance.ServiceName,
+	}
 
 	//
 	// check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map
@@ -515,21 +518,16 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
 			err.Error()))
 		return ctrl.Result{}, err
 	}
-	configMapVars[ospSecret.Name] = env.SetValue(hash)
+	configVars[ospSecret.Name] = env.SetValue(hash)
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
 	// run check OpenStack secret - end
 
 	//
-	// Create ConfigMaps and Secrets required as input for the Service and calculate an overall hash of hashes
+	// Create Secrets required as input for the Service and calculate an overall hash of hashes
 	//
 
 	//
-	// create Configmap required for glance input
-	// - %-scripts configmap holding scripts to e.g. bootstrap the service
-	// - %-config configmap holding minimal glance config required to get the service up, user can add additional files to be added to the service
-	// - parameters which has passwords gets added from the OpenStack secret via the init container
-	//
-	err = r.generateServiceConfigMaps(ctx, helper, instance, &configMapVars)
+	err = r.generateServiceConfig(ctx, helper, instance, &configVars)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -540,16 +538,12 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
 		return ctrl.Result{}, err
 	}
 	instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
-	// Create ConfigMaps and Secrets - end
+	// Create Secrets - end
 
 	//
 	// TODO check when/if Init, Update, or Upgrade should/could be skipped
 	//
 
-	serviceLabels := map[string]string{
-		common.AppSelector: glance.ServiceName,
-	}
-
 	// networks to attach to
 	for _, netAtt := range instance.Spec.GlanceAPIInternal.NetworkAttachments {
 		_, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace)
@@ -739,61 +733,38 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst
 	return deployment, op, err
 }
 
-// generateServiceConfigMaps - create create configmaps which hold scripts and service configuration (*used for DBSync only*)
-// TODO add DefaultConfigOverwrite
-func (r *GlanceReconciler) generateServiceConfigMaps(
+// generateServiceConfig - create secrets which hold scripts and service configuration (*used for DBSync only*)
+func (r *GlanceReconciler) generateServiceConfig(
 	ctx context.Context,
 	h *helper.Helper,
 	instance *glancev1.Glance,
 	envVars *map[string]env.Setter,
 ) error {
-	// TODO: Since these CMs are only used for the DBSync, is there further pruning that
-	//       could be done here?
+	labels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), map[string]string{})
 
-	//
-	// create Configmap/Secret required for glance input
-	// - %-scripts configmap holding scripts to e.g. bootstrap the service
-	// - %-config configmap holding minimal glance config required to get the service up, user can add additional files to be added to the service
-	// - parameters which has passwords gets added from the ospSecret via the init container
-	//
+	ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Secret, instance.Namespace)
+	if err != nil {
+		return err
+	}
 
-	cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), map[string]string{})
+	// We only need a minimal 00-config.conf that is only used by db-sync job,
+	// hence only passing the database related parameters
+	templateParameters := map[string]interface{}{
+		"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
+			instance.Spec.DatabaseUser,
+			string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
+			instance.Status.DatabaseHostname,
+			glance.DatabaseName,
+		),
+	}
+	customData := map[string]string{glance.CustomConfigFileName: instance.Spec.CustomServiceConfig}
 
-	// customData hold any customization for the service.
-	// custom.conf is going to /etc/<service>/<service>.conf.d
-	// all other files get placed into /etc/<service> to allow overwrite of e.g. logging.conf or policy.json
-	// TODO: make sure custom.conf can not be overwritten
-	customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
 	for key, data := range instance.Spec.DefaultConfigOverwrite {
 		customData[key] = data
 	}
 
-	cms := []util.Template{
-		// ScriptsConfigMap
-		{
-			Name:         fmt.Sprintf("%s-scripts", instance.Name),
-			Namespace:    instance.Namespace,
-			Type:         util.TemplateTypeScripts,
-			InstanceType: instance.Kind,
-			Labels:       cmLabels,
-		},
-		// ConfigMap
-		{
-			Name:         fmt.Sprintf("%s-config-data", instance.Name),
-			Namespace:    instance.Namespace,
-			Type:         util.TemplateTypeConfig,
-			InstanceType: instance.Kind,
-			CustomData:   customData,
-			Labels:       cmLabels,
-		},
-	}
-	err := configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars)
-
-	if err != nil {
-		return nil
-	}
+	return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, false)
 
-	return nil
 }
 
 // ensureRegisteredLimits - create registered limits in keystone that will be