Skip to content

Commit

Permalink
Pass through Annotations and Labels to all Child resources (#193)
Browse files Browse the repository at this point in the history 
* Initial working for issue #95

Signed-off-by: David Collom <[email protected]>

* Fix up annotations continously changing

Signed-off-by: David Collom <[email protected]>
  • Loading branch information
@davidcollom
davidcollom authored Jan 10, 2022
1 parent e3b1879 commit ab7bd65
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 47 deletions.
42 changes: 30 additions & 12 deletions k8sutils/labels.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,32 +66,50 @@ func redisClusterAsOwner(cr *redisv1beta1.RedisCluster) metav1.OwnerReference {
}

// generateStatefulSetsAnots generates and returns statefulsets annotations
func generateStatefulSetsAnots() map[string]string {
return map[string]string{
"redis.opstreelabs.in": "true",
"prometheus.io/scrape": "true",
"prometheus.io/port": "9121",
func generateStatefulSetsAnots(stsMeta metav1.ObjectMeta) map[string]string {
anots := map[string]string{
"redis.opstreelabs.in": "true",
"redis.opstreelabs.instance": stsMeta.GetName(),
}
for k, v := range stsMeta.GetAnnotations() {
anots[k] = v
}
return filterAnnotations(anots)
}

// filterAnnotations Remove autogenerated annotations which pose no use to downstream objects (Services,Pods,etc)
func filterAnnotations(anots map[string]string) map[string]string {
// Filter out some problematic annotations we don't want in the template.
delete(anots, "kubectl.kubernetes.io/last-applied-configuration")
delete(anots, "banzaicloud.com/last-applied")
return anots
}

// generateServiceAnots generates and returns service annotations
func generateServiceAnots() map[string]string {
return map[string]string{
"redis.opstreelabs.in": "true",
"prometheus.io/scrape": "true",
"prometheus.io/port": "9121",
func generateServiceAnots(stsMeta metav1.ObjectMeta) map[string]string {
anots := map[string]string{
"redis.opstreelabs.in": "true",
"redis.opstreelabs.instance": stsMeta.GetName(),
}
for k, v := range stsMeta.GetAnnotations() {
anots[k] = v
}
return filterAnnotations(anots)
}

// LabelSelectors generates object for label selection
func LabelSelectors(labels map[string]string) *metav1.LabelSelector {
return &metav1.LabelSelector{MatchLabels: labels}
}

func getRedisLabels(name, setupType, role string) map[string]string {
return map[string]string{
func getRedisLabels(name, setupType, role string, labels map[string]string) map[string]string {
lbls := map[string]string{
"app": name,
"redis_setup_type": setupType,
"role": role,
}
for k, v := range labels {
lbls[k] = v
}
return lbls
}
5 changes: 3 additions & 2 deletions k8sutils/poddisruption.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ import (
func ReconcileRedisPodDisruptionBudget(cr *redisv1beta1.RedisCluster, role string) error {
pdbName := cr.ObjectMeta.Name + "-" + role
if cr.Spec.RedisLeader.PodDisruptionBudget != nil && cr.Spec.RedisLeader.PodDisruptionBudget.Enabled {
labels := getRedisLabels(cr.ObjectMeta.Name, "cluster", role)
pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, generateStatefulSetsAnots())
labels := getRedisLabels(cr.ObjectMeta.Name, "cluster", role, cr.ObjectMeta.GetLabels())
annotations := generateStatefulSetsAnots(cr.ObjectMeta)
pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations)
pdbDef := generatePodDisruptionBudgetDef(cr, role, pdbMeta, cr.Spec.RedisLeader.PodDisruptionBudget)
return CreateOrUpdatePodDisruptionBudget(pdbDef)
} else {
Expand Down
18 changes: 10 additions & 8 deletions k8sutils/redis-cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type RedisClusterService struct {
// generateRedisStandalone generates Redis standalone information
func generateRedisClusterParams(cr *redisv1beta1.RedisCluster, replicas *int32, externalConfig *string, affinity *corev1.Affinity) statefulSetParameters {
res := statefulSetParameters{
Metadata: cr.ObjectMeta,
Replicas: replicas,
NodeSelector: cr.Spec.NodeSelector,
SecurityContext: cr.Spec.SecurityContext,
Expand Down Expand Up @@ -150,12 +151,12 @@ func (service RedisClusterSTS) getReplicaCount(cr *redisv1beta1.RedisCluster) *i
func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta1.RedisCluster) error {
stateFulName := cr.ObjectMeta.Name + "-" + service.RedisStateFulType
logger := stateFulSetLogger(cr.Namespace, stateFulName)
labels := getRedisLabels(stateFulName, "cluster", service.RedisStateFulType)
objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, generateStatefulSetsAnots())
labels := getRedisLabels(stateFulName, "cluster", service.RedisStateFulType, cr.ObjectMeta.Labels)
annotations := generateStatefulSetsAnots(cr.ObjectMeta)
objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, annotations)
err := CreateOrUpdateStateFul(
cr.Namespace,
objectMetaInfo,
labels,
generateRedisClusterParams(cr, service.getReplicaCount(cr), service.ExternalConfig, service.Affinity),
redisClusterAsOwner(cr),
generateRedisClusterContainerParams(cr, service.ReadinessProbe, service.LivenessProbe),
Expand All @@ -172,18 +173,19 @@ func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta1.RedisClu
func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta1.RedisCluster) error {
serviceName := cr.ObjectMeta.Name + "-" + service.RedisServiceRole
logger := serviceLogger(cr.Namespace, serviceName)
labels := getRedisLabels(serviceName, "cluster", service.RedisServiceRole)
labels := getRedisLabels(serviceName, "cluster", service.RedisServiceRole, cr.ObjectMeta.Labels)
annotations := generateServiceAnots(cr.ObjectMeta)
if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled {
enableMetrics = true
}
objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, generateServiceAnots())
headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, generateServiceAnots())
err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, labels, redisClusterAsOwner(cr))
objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations)
headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations)
err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr))
if err != nil {
logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole)
return err
}
err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, labels, redisClusterAsOwner(cr), enableMetrics)
err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics)
if err != nil {
logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole)
return err
Expand Down
17 changes: 9 additions & 8 deletions k8sutils/redis-standalone.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,19 @@ var (
// CreateStandAloneService method will create standalone service for Redis
func CreateStandAloneService(cr *redisv1beta1.Redis) error {
logger := serviceLogger(cr.Namespace, cr.ObjectMeta.Name)
labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone")
labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone", cr.ObjectMeta.Labels)
annotations := generateServiceAnots(cr.ObjectMeta)
if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled {
enableMetrics = true
}
objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, generateServiceAnots())
headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, generateServiceAnots())
err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, labels, redisAsOwner(cr))
objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations)
err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr))
if err != nil {
logger.Error(err, "Cannot create standalone headless service for Redis")
return err
}
err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, labels, redisAsOwner(cr), enableMetrics)
err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics)
if err != nil {
logger.Error(err, "Cannot create standalone service for Redis")
return err
Expand All @@ -33,11 +34,11 @@ func CreateStandAloneService(cr *redisv1beta1.Redis) error {
// CreateStandAloneRedis will create a standalone redis setup
func CreateStandAloneRedis(cr *redisv1beta1.Redis) error {
logger := stateFulSetLogger(cr.Namespace, cr.ObjectMeta.Name)
labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone")
objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, generateStatefulSetsAnots())
labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone", cr.ObjectMeta.Labels)
annotations := generateStatefulSetsAnots(cr.ObjectMeta)
objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
err := CreateOrUpdateStateFul(cr.Namespace,
objectMetaInfo,
labels,
generateRedisStandaloneParams(cr),
redisAsOwner(cr),
generateRedisStandaloneContainerParams(cr),
Expand Down
12 changes: 6 additions & 6 deletions k8sutils/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ var (
)

// generateHeadlessServiceDef generates service definition for headless service
func generateHeadlessServiceDef(serviceMeta metav1.ObjectMeta, labels map[string]string, ownerDef metav1.OwnerReference) *corev1.Service {
func generateHeadlessServiceDef(serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference) *corev1.Service {
service := &corev1.Service{
TypeMeta: generateMetaInformation("Service", "core/v1"),
ObjectMeta: serviceMeta,
Spec: corev1.ServiceSpec{
ClusterIP: "None",
Selector: labels,
Selector: serviceMeta.Labels,
Ports: []corev1.ServicePort{
{
Name: "redis-client",
Expand Down Expand Up @@ -135,10 +135,10 @@ func serviceLogger(namespace string, name string) logr.Logger {
}

// CreateOrUpdateHeadlessService method will create or update Redis headless service
func CreateOrUpdateHeadlessService(namespace string, serviceMeta metav1.ObjectMeta, labels map[string]string, ownerDef metav1.OwnerReference) error {
func CreateOrUpdateHeadlessService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference) error {
logger := serviceLogger(namespace, serviceMeta.Name)
storedService, err := getService(namespace, serviceMeta.Name)
serviceDef := generateHeadlessServiceDef(serviceMeta, labels, ownerDef)
serviceDef := generateHeadlessServiceDef(serviceMeta, ownerDef)
if err != nil {
if errors.IsNotFound(err) {
if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(serviceDef); err != nil {
Expand All @@ -153,9 +153,9 @@ func CreateOrUpdateHeadlessService(namespace string, serviceMeta metav1.ObjectMe
}

// CreateOrUpdateService method will create or update Redis service
func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, labels map[string]string, ownerDef metav1.OwnerReference, enableMetrics bool) error {
func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, enableMetrics bool) error {
logger := serviceLogger(namespace, serviceMeta.Name)
serviceDef := generateServiceDef(serviceMeta, labels, enableMetrics, ownerDef)
serviceDef := generateServiceDef(serviceMeta, serviceMeta.Labels, enableMetrics, ownerDef)
storedService, err := getService(namespace, serviceMeta.Name)
if err != nil {
if errors.IsNotFound(err) {
Expand Down
32 changes: 21 additions & 11 deletions k8sutils/statefulset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const (
// statefulSetParameters will define statefulsets input params
type statefulSetParameters struct {
Replicas *int32
Metadata metav1.ObjectMeta
NodeSelector map[string]string
SecurityContext *corev1.PodSecurityContext
PriorityClassName string
Expand Down Expand Up @@ -53,10 +54,10 @@ type containerParameters struct {
}

// CreateOrUpdateStateFul method will create or update Redis service
func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, labels map[string]string, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars *[]redisv1beta1.Sidecar) error {
func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars *[]redisv1beta1.Sidecar) error {
logger := stateFulSetLogger(namespace, stsMeta.Name)
storedStateful, err := GetStateFulSet(namespace, stsMeta.Name)
statefulSetDef := generateStateFulSetsDef(stsMeta, labels, params, ownerDef, containerParams, getSidecars(sidecars))
statefulSetDef := generateStateFulSetsDef(stsMeta, params, ownerDef, containerParams, getSidecars(sidecars))
if err != nil {
if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(statefulSetDef); err != nil {
logger.Error(err, "Unable to patch redis statefulset with comparison object")
Expand All @@ -79,9 +80,12 @@ func patchStateFulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St
return err
}
if !patchResult.IsEmpty() {
logger.Info("Changes in statefulset Detected, Updating...")
newStateful.ResourceVersion = storedStateful.ResourceVersion
newStateful.CreationTimestamp = storedStateful.CreationTimestamp
newStateful.ManagedFields = storedStateful.ManagedFields
// Field is immutable therefore we MUST keep it as is.
newStateful.Spec.VolumeClaimTemplates = storedStateful.Spec.VolumeClaimTemplates
for key, value := range storedStateful.Annotations {
if _, present := newStateful.Annotations[key]; !present {
newStateful.Annotations[key] = value
Expand All @@ -97,20 +101,22 @@ func patchStateFulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St
}

// generateStateFulSetsDef generates the statefulsets definition of Redis
func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, labels map[string]string, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars []redisv1beta1.Sidecar) *appsv1.StatefulSet {
func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars []redisv1beta1.Sidecar) *appsv1.StatefulSet {
statefulset := &appsv1.StatefulSet{
TypeMeta: generateMetaInformation("StatefulSet", "apps/v1"),
ObjectMeta: stsMeta,
Spec: appsv1.StatefulSetSpec{
Selector: LabelSelectors(labels),
Selector: LabelSelectors(stsMeta.GetLabels()),
ServiceName: stsMeta.Name,
Replicas: params.Replicas,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
Labels: stsMeta.GetLabels(),
Annotations: generateStatefulSetsAnots(stsMeta),
// Annotations: stsMeta.Annotations,
},
Spec: corev1.PodSpec{
Containers: generateContainerDef(stsMeta.Name, containerParams, params.EnableMetrics, params.ExternalConfig, sidecars),
Containers: generateContainerDef(stsMeta.GetName(), containerParams, params.EnableMetrics, params.ExternalConfig, sidecars),
NodeSelector: params.NodeSelector,
SecurityContext: params.SecurityContext,
PriorityClassName: params.PriorityClassName,
Expand All @@ -119,14 +125,15 @@ func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, labels map[string]string
},
},
}
// fmt.Printf("STS: %s", statefulset)
if params.Tolerations != nil {
statefulset.Spec.Template.Spec.Tolerations = *params.Tolerations
}
if params.ImagePullSecrets != nil {
statefulset.Spec.Template.Spec.ImagePullSecrets = *params.ImagePullSecrets
}
if containerParams.PersistenceEnabled != nil && *containerParams.PersistenceEnabled {
statefulset.Spec.VolumeClaimTemplates = append(statefulset.Spec.VolumeClaimTemplates, createPVCTemplate(stsMeta.Name, params.PersistentVolumeClaim))
statefulset.Spec.VolumeClaimTemplates = append(statefulset.Spec.VolumeClaimTemplates, createPVCTemplate(stsMeta, params.PersistentVolumeClaim))
}
if params.ExternalConfig != nil {
statefulset.Spec.Template.Spec.Volumes = getExternalConfig(*params.ExternalConfig)
Expand All @@ -152,18 +159,20 @@ func getExternalConfig(configMapName string) []corev1.Volume {
}

// createPVCTemplate will create the persistent volume claim template
func createPVCTemplate(name string, storageSpec corev1.PersistentVolumeClaim) corev1.PersistentVolumeClaim {
func createPVCTemplate(stsMeta metav1.ObjectMeta, storageSpec corev1.PersistentVolumeClaim) corev1.PersistentVolumeClaim {
pvcTemplate := storageSpec
pvcTemplate.CreationTimestamp = metav1.Time{}
pvcTemplate.Name = name
pvcTemplate.Name = stsMeta.GetName()
pvcTemplate.Labels = stsMeta.GetLabels()
// We want the same annoations as the StatefulSet here
pvcTemplate.Annotations = generateStatefulSetsAnots(stsMeta)
if storageSpec.Spec.AccessModes == nil {
pvcTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}
} else {
pvcTemplate.Spec.AccessModes = storageSpec.Spec.AccessModes
}
pvcTemplate.Spec.Resources = storageSpec.Spec.Resources
pvcTemplate.Spec.Selector = storageSpec.Spec.Selector
pvcTemplate.Spec.Selector = storageSpec.Spec.Selector
return pvcTemplate
}

Expand Down Expand Up @@ -406,12 +415,13 @@ func createStateFulSet(namespace string, stateful *appsv1.StatefulSet) error {
// updateStateFulSet is a method to update statefulset in Kubernetes
func updateStateFulSet(namespace string, stateful *appsv1.StatefulSet) error {
logger := stateFulSetLogger(namespace, stateful.Name)
// logger.Info(fmt.Sprintf("Setting Statefulset to the following: %s", stateful))
_, err := generateK8sClient().AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{})
if err != nil {
logger.Error(err, "Redis stateful update failed")
return err
}
logger.Info("Redis stateful ")
logger.Info("Redis stateful successfully updated ")
return nil
}

Expand Down

0 comments on commit ab7bd65

Please sign in to comment.