Skip to content

Commit

Permalink
Rework glance config process
Browse files Browse the repository at this point in the history
This patch represents an implementation of the proposal [1] and aligns
the glance-operator with the work already done in Cinder and the other
storage operators. There are a few relevant changes in the bootstrap
process of Glance, in particular:

- It stops using an init container to generate the snippet files that
  configure each glanceAPI service. The logic that was previously
  implemented in the Init Container has been moved to the Controllers,
  where config files are generated and stored in k8s Secrets. Init
  Containers are fully removed from the bootstrap process;
- It uses Kolla to copy the generated content to the target directory
  /etc/glance/glance.conf.d, which is used by each API service to run;
- The relevant content, including scripts, previously stored in a
  ConfigMap, are now stored in a corresponding k8s Secret, which is
  mounted to the GlanceAPI deployment Pods (hence no additional calls
  to GetVolume/GetVolumeMounts are required);
- A total of 4 config snippet files are generated:
  - 00-config.conf contains global settings that are common to every
    GlanceAPI Pod, including ones that are derived from deployment
    secrets (e.g. database password, etc.)
  - 01-config.conf contains the global customServiceConfig settings
    that apply to every GlanceAPI service.
  - 02-config.conf contains the customServiceConfig settings that
    are specific to each service.
  - 03-config.conf contains secrets specified by each service's
    customServiceConfigSecrets.
- glance-image-import.conf has been removed, and the default config has
  been added to 00-config.conf, which is the common file generated by
  the umbrella Glance controller;
- logging.conf has been removed as it's no longer required in the switch
  to a side container approach for logging purposes;
- functional tests are aligned to the use of k8s Secrets instead of the
  old pattern based on ConfigMaps;
- kuttl tests are updated and the 'initContainer' has been removed.
- dbsync now mounts only the required files (a minimal 00-config.conf)
  and a db-sync-config.json containing the command run through kolla.

[1] openstack-k8s-operators/dev-docs#31

Signed-off-by: Francesco Pantano <[email protected]>
  • Loading branch information
fmount committed Aug 20, 2023
1 parent 36ff067 commit ce4f0a8
Show file tree
Hide file tree
Showing 19 changed files with 273 additions and 595 deletions.
12 changes: 0 additions & 12 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,6 @@ rules:
- patch
- update
- watch
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down
61 changes: 61 additions & 0 deletions controllers/glance_common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
Copyright 2022.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
"context"
"fmt"
"github.com/openstack-k8s-operators/lib-common/modules/common/env"
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
"github.com/openstack-k8s-operators/lib-common/modules/common/util"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func GenerateConfigsGeneric(
ctx context.Context, h *helper.Helper,
instance client.Object,
envVars *map[string]env.Setter,
templateParameters map[string]interface{},
customData map[string]string,
cmLabels map[string]string,
scripts bool,
) error {

cms := []util.Template{
// Templates where the GlanceAPI config is stored
{
Name: fmt.Sprintf("%s-config-data", instance.GetName()),
Namespace: instance.GetNamespace(),
Type: util.TemplateTypeConfig,
InstanceType: instance.GetObjectKind().GroupVersionKind().Kind,
ConfigOptions: templateParameters,
CustomData: customData,
Labels: cmLabels,
},
}
if scripts {
cms = append(cms, util.Template{
Name: fmt.Sprintf("%s-scripts", instance.GetName()),
Namespace: instance.GetNamespace(),
Type: util.TemplateTypeScripts,
InstanceType: instance.GetObjectKind().GroupVersionKind().Kind,
Labels: cmLabels,
})
}
return secret.EnsureSecrets(ctx, h, instance, cms, envVars)
}
89 changes: 30 additions & 59 deletions controllers/glance_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"
"time"

"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
rbacv1 "k8s.io/api/rbac/v1"
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -37,7 +38,6 @@ import (
keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
"github.com/openstack-k8s-operators/lib-common/modules/common"
"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
"github.com/openstack-k8s-operators/lib-common/modules/common/endpoint"
"github.com/openstack-k8s-operators/lib-common/modules/common/env"
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
Expand Down Expand Up @@ -191,7 +191,6 @@ func (r *GlanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
Owns(&keystonev1.KeystoneService{}).
Owns(&corev1.PersistentVolumeClaim{}).
Owns(&batchv1.Job{}).
Owns(&corev1.ConfigMap{}).
Owns(&corev1.Secret{}).
Owns(&corev1.ServiceAccount{}).
Owns(&rbacv1.Role{}).
Expand Down Expand Up @@ -491,8 +490,12 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
return rbacResult, nil
}

// ConfigMap
configMapVars := make(map[string]env.Setter)
configVars := make(map[string]env.Setter)

// ServiceLabels
serviceLabels := map[string]string{
common.AppSelector: glance.ServiceName,
}

//
// check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map
Expand All @@ -515,21 +518,16 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
err.Error()))
return ctrl.Result{}, err
}
configMapVars[ospSecret.Name] = env.SetValue(hash)
configVars[ospSecret.Name] = env.SetValue(hash)
instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
// run check OpenStack secret - end

//
// Create ConfigMaps and Secrets required as input for the Service and calculate an overall hash of hashes
// Create Secrets required as input for the Service and calculate an overall hash of hashes
//

//
// create Configmap required for glance input
// - %-scripts configmap holding scripts to e.g. bootstrap the service
// - %-config configmap holding minimal glance config required to get the service up, user can add additional files to be added to the service
// - parameters which has passwords gets added from the OpenStack secret via the init container
//
err = r.generateServiceConfigMaps(ctx, helper, instance, &configMapVars)
err = r.generateServiceConfig(ctx, helper, instance, &configVars)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.ServiceConfigReadyCondition,
Expand All @@ -540,16 +538,12 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance
return ctrl.Result{}, err
}
instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
// Create ConfigMaps and Secrets - end
// Create Secrets - end

//
// TODO check when/if Init, Update, or Upgrade should/could be skipped
//

serviceLabels := map[string]string{
common.AppSelector: glance.ServiceName,
}

// networks to attach to
for _, netAtt := range instance.Spec.GlanceAPIInternal.NetworkAttachments {
_, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace)
Expand Down Expand Up @@ -739,61 +733,38 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst
return deployment, op, err
}

// generateServiceConfigMaps - create create configmaps which hold scripts and service configuration (*used for DBSync only*)
// TODO add DefaultConfigOverwrite
func (r *GlanceReconciler) generateServiceConfigMaps(
// generateServiceConfig - create secrets which hold scripts and service configuration (*used for DBSync only*)
func (r *GlanceReconciler) generateServiceConfig(
ctx context.Context,
h *helper.Helper,
instance *glancev1.Glance,
envVars *map[string]env.Setter,
) error {
// TODO: Since these CMs are only used for the DBSync, is there further pruning that
// could be done here?
labels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), map[string]string{})

//
// create Configmap/Secret required for glance input
// - %-scripts configmap holding scripts to e.g. bootstrap the service
// - %-config configmap holding minimal glance config required to get the service up, user can add additional files to be added to the service
// - parameters which has passwords gets added from the ospSecret via the init container
//
ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Secret, instance.Namespace)
if err != nil {
return err
}

cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(glance.ServiceName), map[string]string{})
// We only need a minimal 00-config.conf that is only used by db-sync job,
// hence only passing the database related parameters
templateParameters := map[string]interface{}{
"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s",
instance.Spec.DatabaseUser,
string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
instance.Status.DatabaseHostname,
glance.DatabaseName,
),
}
customData := map[string]string{glance.CustomConfigFileName: instance.Spec.CustomServiceConfig}

// customData hold any customization for the service.
// custom.conf is going to /etc/<service>/<service>.conf.d
// all other files get placed into /etc/<service> to allow overwrite of e.g. logging.conf or policy.json
// TODO: make sure custom.conf can not be overwritten
customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
for key, data := range instance.Spec.DefaultConfigOverwrite {
customData[key] = data
}

cms := []util.Template{
// ScriptsConfigMap
{
Name: fmt.Sprintf("%s-scripts", instance.Name),
Namespace: instance.Namespace,
Type: util.TemplateTypeScripts,
InstanceType: instance.Kind,
Labels: cmLabels,
},
// ConfigMap
{
Name: fmt.Sprintf("%s-config-data", instance.Name),
Namespace: instance.Namespace,
Type: util.TemplateTypeConfig,
InstanceType: instance.Kind,
CustomData: customData,
Labels: cmLabels,
},
}
err := configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars)

if err != nil {
return nil
}
return GenerateConfigsGeneric(ctx, h, instance, envVars, templateParameters, customData, labels, false)

return nil
}

// ensureRegisteredLimits - create registered limits in keystone that will be
Expand Down
Loading

0 comments on commit ce4f0a8

Please sign in to comment.