diff --git a/k8sutils/labels.go b/k8sutils/labels.go index 062d7ce7b..06d906138 100644 --- a/k8sutils/labels.go +++ b/k8sutils/labels.go @@ -66,21 +66,35 @@ func redisClusterAsOwner(cr *redisv1beta1.RedisCluster) metav1.OwnerReference { } // generateStatefulSetsAnots generates and returns statefulsets annotations -func generateStatefulSetsAnots() map[string]string { - return map[string]string{ - "redis.opstreelabs.in": "true", - "prometheus.io/scrape": "true", - "prometheus.io/port": "9121", +func generateStatefulSetsAnots(stsMeta metav1.ObjectMeta) map[string]string { + anots := map[string]string{ + "redis.opstreelabs.in": "true", + "redis.opstreelabs.instance": stsMeta.GetName(), } + for k, v := range stsMeta.GetAnnotations() { + anots[k] = v + } + return filterAnnotations(anots) +} + +// filterAnnotations Remove autogenerated annotations which pose no use to downstream objects (Services,Pods,etc) +func filterAnnotations(anots map[string]string) map[string]string { + // Filter out some problematic annotations we don't want in the template. + delete(anots, "kubectl.kubernetes.io/last-applied-configuration") + delete(anots, "banzaicloud.com/last-applied") + return anots } // generateServiceAnots generates and returns service annotations -func generateServiceAnots() map[string]string { - return map[string]string{ - "redis.opstreelabs.in": "true", - "prometheus.io/scrape": "true", - "prometheus.io/port": "9121", +func generateServiceAnots(stsMeta metav1.ObjectMeta) map[string]string { + anots := map[string]string{ + "redis.opstreelabs.in": "true", + "redis.opstreelabs.instance": stsMeta.GetName(), } + for k, v := range stsMeta.GetAnnotations() { + anots[k] = v + } + return filterAnnotations(anots) } // LabelSelectors generates object for label selection @@ -88,10 +102,14 @@ func LabelSelectors(labels map[string]string) *metav1.LabelSelector { return &metav1.LabelSelector{MatchLabels: labels} } -func getRedisLabels(name, setupType, role string) map[string]string { - return map[string]string{ +func getRedisLabels(name, setupType, role string, labels map[string]string) map[string]string { + lbls := map[string]string{ "app": name, "redis_setup_type": setupType, "role": role, } + for k, v := range labels { + lbls[k] = v + } + return lbls } diff --git a/k8sutils/poddisruption.go b/k8sutils/poddisruption.go index 6f4660658..697e5da85 100644 --- a/k8sutils/poddisruption.go +++ b/k8sutils/poddisruption.go @@ -17,8 +17,9 @@ import ( func ReconcileRedisPodDisruptionBudget(cr *redisv1beta1.RedisCluster, role string) error { pdbName := cr.ObjectMeta.Name + "-" + role if cr.Spec.RedisLeader.PodDisruptionBudget != nil && cr.Spec.RedisLeader.PodDisruptionBudget.Enabled { - labels := getRedisLabels(cr.ObjectMeta.Name, "cluster", role) - pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, generateStatefulSetsAnots()) + labels := getRedisLabels(cr.ObjectMeta.Name, "cluster", role, cr.ObjectMeta.GetLabels()) + annotations := generateStatefulSetsAnots(cr.ObjectMeta) + pdbMeta := generateObjectMetaInformation(pdbName, cr.Namespace, labels, annotations) pdbDef := generatePodDisruptionBudgetDef(cr, role, pdbMeta, cr.Spec.RedisLeader.PodDisruptionBudget) return CreateOrUpdatePodDisruptionBudget(pdbDef) } else { diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index b24a0772d..813c703cc 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -23,6 +23,7 @@ type RedisClusterService struct { // generateRedisStandalone generates Redis standalone information func generateRedisClusterParams(cr *redisv1beta1.RedisCluster, replicas *int32, externalConfig *string, affinity *corev1.Affinity) statefulSetParameters { res := statefulSetParameters{ + Metadata: cr.ObjectMeta, Replicas: replicas, NodeSelector: cr.Spec.NodeSelector, SecurityContext: cr.Spec.SecurityContext, @@ -150,12 +151,12 @@ func (service RedisClusterSTS) getReplicaCount(cr *redisv1beta1.RedisCluster) *i func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta1.RedisCluster) error { stateFulName := cr.ObjectMeta.Name + "-" + service.RedisStateFulType logger := stateFulSetLogger(cr.Namespace, stateFulName) - labels := getRedisLabels(stateFulName, "cluster", service.RedisStateFulType) - objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, generateStatefulSetsAnots()) + labels := getRedisLabels(stateFulName, "cluster", service.RedisStateFulType, cr.ObjectMeta.Labels) + annotations := generateStatefulSetsAnots(cr.ObjectMeta) + objectMetaInfo := generateObjectMetaInformation(stateFulName, cr.Namespace, labels, annotations) err := CreateOrUpdateStateFul( cr.Namespace, objectMetaInfo, - labels, generateRedisClusterParams(cr, service.getReplicaCount(cr), service.ExternalConfig, service.Affinity), redisClusterAsOwner(cr), generateRedisClusterContainerParams(cr, service.ReadinessProbe, service.LivenessProbe), @@ -172,18 +173,19 @@ func (service RedisClusterSTS) CreateRedisClusterSetup(cr *redisv1beta1.RedisClu func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta1.RedisCluster) error { serviceName := cr.ObjectMeta.Name + "-" + service.RedisServiceRole logger := serviceLogger(cr.Namespace, serviceName) - labels := getRedisLabels(serviceName, "cluster", service.RedisServiceRole) + labels := getRedisLabels(serviceName, "cluster", service.RedisServiceRole, cr.ObjectMeta.Labels) + annotations := generateServiceAnots(cr.ObjectMeta) if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled { enableMetrics = true } - objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, generateServiceAnots()) - headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, generateServiceAnots()) - err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, labels, redisClusterAsOwner(cr)) + objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations) + headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations) + err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr)) if err != nil { logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole) return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, labels, redisClusterAsOwner(cr), enableMetrics) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics) if err != nil { logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole) return err diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go index 37c1fc956..83e2dc4c9 100644 --- a/k8sutils/redis-standalone.go +++ b/k8sutils/redis-standalone.go @@ -11,18 +11,19 @@ var ( // CreateStandAloneService method will create standalone service for Redis func CreateStandAloneService(cr *redisv1beta1.Redis) error { logger := serviceLogger(cr.Namespace, cr.ObjectMeta.Name) - labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone") + labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone", cr.ObjectMeta.Labels) + annotations := generateServiceAnots(cr.ObjectMeta) if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled { enableMetrics = true } - objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, generateServiceAnots()) - headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, generateServiceAnots()) - err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, labels, redisAsOwner(cr)) + objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations) + headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations) + err := CreateOrUpdateHeadlessService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr)) if err != nil { logger.Error(err, "Cannot create standalone headless service for Redis") return err } - err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, labels, redisAsOwner(cr), enableMetrics) + err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics) if err != nil { logger.Error(err, "Cannot create standalone service for Redis") return err @@ -33,11 +34,11 @@ func CreateStandAloneService(cr *redisv1beta1.Redis) error { // CreateStandAloneRedis will create a standalone redis setup func CreateStandAloneRedis(cr *redisv1beta1.Redis) error { logger := stateFulSetLogger(cr.Namespace, cr.ObjectMeta.Name) - labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone") - objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, generateStatefulSetsAnots()) + labels := getRedisLabels(cr.ObjectMeta.Name, "standalone", "standalone", cr.ObjectMeta.Labels) + annotations := generateStatefulSetsAnots(cr.ObjectMeta) + objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations) err := CreateOrUpdateStateFul(cr.Namespace, objectMetaInfo, - labels, generateRedisStandaloneParams(cr), redisAsOwner(cr), generateRedisStandaloneContainerParams(cr), diff --git a/k8sutils/services.go b/k8sutils/services.go index a9b9925ce..f0711776f 100644 --- a/k8sutils/services.go +++ b/k8sutils/services.go @@ -21,13 +21,13 @@ var ( ) // generateHeadlessServiceDef generates service definition for headless service -func generateHeadlessServiceDef(serviceMeta metav1.ObjectMeta, labels map[string]string, ownerDef metav1.OwnerReference) *corev1.Service { +func generateHeadlessServiceDef(serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference) *corev1.Service { service := &corev1.Service{ TypeMeta: generateMetaInformation("Service", "core/v1"), ObjectMeta: serviceMeta, Spec: corev1.ServiceSpec{ ClusterIP: "None", - Selector: labels, + Selector: serviceMeta.Labels, Ports: []corev1.ServicePort{ { Name: "redis-client", @@ -135,10 +135,10 @@ func serviceLogger(namespace string, name string) logr.Logger { } // CreateOrUpdateHeadlessService method will create or update Redis headless service -func CreateOrUpdateHeadlessService(namespace string, serviceMeta metav1.ObjectMeta, labels map[string]string, ownerDef metav1.OwnerReference) error { +func CreateOrUpdateHeadlessService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference) error { logger := serviceLogger(namespace, serviceMeta.Name) storedService, err := getService(namespace, serviceMeta.Name) - serviceDef := generateHeadlessServiceDef(serviceMeta, labels, ownerDef) + serviceDef := generateHeadlessServiceDef(serviceMeta, ownerDef) if err != nil { if errors.IsNotFound(err) { if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(serviceDef); err != nil { @@ -153,9 +153,9 @@ func CreateOrUpdateHeadlessService(namespace string, serviceMeta metav1.ObjectMe } // CreateOrUpdateService method will create or update Redis service -func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, labels map[string]string, ownerDef metav1.OwnerReference, enableMetrics bool) error { +func CreateOrUpdateService(namespace string, serviceMeta metav1.ObjectMeta, ownerDef metav1.OwnerReference, enableMetrics bool) error { logger := serviceLogger(namespace, serviceMeta.Name) - serviceDef := generateServiceDef(serviceMeta, labels, enableMetrics, ownerDef) + serviceDef := generateServiceDef(serviceMeta, serviceMeta.Labels, enableMetrics, ownerDef) storedService, err := getService(namespace, serviceMeta.Name) if err != nil { if errors.IsNotFound(err) { diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 74d6b4688..3ba81040d 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -22,6 +22,7 @@ const ( // statefulSetParameters will define statefulsets input params type statefulSetParameters struct { Replicas *int32 + Metadata metav1.ObjectMeta NodeSelector map[string]string SecurityContext *corev1.PodSecurityContext PriorityClassName string @@ -53,10 +54,10 @@ type containerParameters struct { } // CreateOrUpdateStateFul method will create or update Redis service -func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, labels map[string]string, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars *[]redisv1beta1.Sidecar) error { +func CreateOrUpdateStateFul(namespace string, stsMeta metav1.ObjectMeta, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars *[]redisv1beta1.Sidecar) error { logger := stateFulSetLogger(namespace, stsMeta.Name) storedStateful, err := GetStateFulSet(namespace, stsMeta.Name) - statefulSetDef := generateStateFulSetsDef(stsMeta, labels, params, ownerDef, containerParams, getSidecars(sidecars)) + statefulSetDef := generateStateFulSetsDef(stsMeta, params, ownerDef, containerParams, getSidecars(sidecars)) if err != nil { if err := patch.DefaultAnnotator.SetLastAppliedAnnotation(statefulSetDef); err != nil { logger.Error(err, "Unable to patch redis statefulset with comparison object") @@ -79,9 +80,12 @@ func patchStateFulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St return err } if !patchResult.IsEmpty() { + logger.Info("Changes in statefulset Detected, Updating...") newStateful.ResourceVersion = storedStateful.ResourceVersion newStateful.CreationTimestamp = storedStateful.CreationTimestamp newStateful.ManagedFields = storedStateful.ManagedFields + // Field is immutable therefore we MUST keep it as is. + newStateful.Spec.VolumeClaimTemplates = storedStateful.Spec.VolumeClaimTemplates for key, value := range storedStateful.Annotations { if _, present := newStateful.Annotations[key]; !present { newStateful.Annotations[key] = value @@ -97,20 +101,22 @@ func patchStateFulSet(storedStateful *appsv1.StatefulSet, newStateful *appsv1.St } // generateStateFulSetsDef generates the statefulsets definition of Redis -func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, labels map[string]string, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars []redisv1beta1.Sidecar) *appsv1.StatefulSet { +func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, params statefulSetParameters, ownerDef metav1.OwnerReference, containerParams containerParameters, sidecars []redisv1beta1.Sidecar) *appsv1.StatefulSet { statefulset := &appsv1.StatefulSet{ TypeMeta: generateMetaInformation("StatefulSet", "apps/v1"), ObjectMeta: stsMeta, Spec: appsv1.StatefulSetSpec{ - Selector: LabelSelectors(labels), + Selector: LabelSelectors(stsMeta.GetLabels()), ServiceName: stsMeta.Name, Replicas: params.Replicas, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Labels: stsMeta.GetLabels(), + Annotations: generateStatefulSetsAnots(stsMeta), + // Annotations: stsMeta.Annotations, }, Spec: corev1.PodSpec{ - Containers: generateContainerDef(stsMeta.Name, containerParams, params.EnableMetrics, params.ExternalConfig, sidecars), + Containers: generateContainerDef(stsMeta.GetName(), containerParams, params.EnableMetrics, params.ExternalConfig, sidecars), NodeSelector: params.NodeSelector, SecurityContext: params.SecurityContext, PriorityClassName: params.PriorityClassName, @@ -119,6 +125,7 @@ func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, labels map[string]string }, }, } + // fmt.Printf("STS: %s", statefulset) if params.Tolerations != nil { statefulset.Spec.Template.Spec.Tolerations = *params.Tolerations } @@ -126,7 +133,7 @@ func generateStateFulSetsDef(stsMeta metav1.ObjectMeta, labels map[string]string statefulset.Spec.Template.Spec.ImagePullSecrets = *params.ImagePullSecrets } if containerParams.PersistenceEnabled != nil && *containerParams.PersistenceEnabled { - statefulset.Spec.VolumeClaimTemplates = append(statefulset.Spec.VolumeClaimTemplates, createPVCTemplate(stsMeta.Name, params.PersistentVolumeClaim)) + statefulset.Spec.VolumeClaimTemplates = append(statefulset.Spec.VolumeClaimTemplates, createPVCTemplate(stsMeta, params.PersistentVolumeClaim)) } if params.ExternalConfig != nil { statefulset.Spec.Template.Spec.Volumes = getExternalConfig(*params.ExternalConfig) @@ -152,10 +159,13 @@ func getExternalConfig(configMapName string) []corev1.Volume { } // createPVCTemplate will create the persistent volume claim template -func createPVCTemplate(name string, storageSpec corev1.PersistentVolumeClaim) corev1.PersistentVolumeClaim { +func createPVCTemplate(stsMeta metav1.ObjectMeta, storageSpec corev1.PersistentVolumeClaim) corev1.PersistentVolumeClaim { pvcTemplate := storageSpec pvcTemplate.CreationTimestamp = metav1.Time{} - pvcTemplate.Name = name + pvcTemplate.Name = stsMeta.GetName() + pvcTemplate.Labels = stsMeta.GetLabels() + // We want the same annoations as the StatefulSet here + pvcTemplate.Annotations = generateStatefulSetsAnots(stsMeta) if storageSpec.Spec.AccessModes == nil { pvcTemplate.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} } else { @@ -163,7 +173,6 @@ func createPVCTemplate(name string, storageSpec corev1.PersistentVolumeClaim) co } pvcTemplate.Spec.Resources = storageSpec.Spec.Resources pvcTemplate.Spec.Selector = storageSpec.Spec.Selector - pvcTemplate.Spec.Selector = storageSpec.Spec.Selector return pvcTemplate } @@ -406,12 +415,13 @@ func createStateFulSet(namespace string, stateful *appsv1.StatefulSet) error { // updateStateFulSet is a method to update statefulset in Kubernetes func updateStateFulSet(namespace string, stateful *appsv1.StatefulSet) error { logger := stateFulSetLogger(namespace, stateful.Name) + // logger.Info(fmt.Sprintf("Setting Statefulset to the following: %s", stateful)) _, err := generateK8sClient().AppsV1().StatefulSets(namespace).Update(context.TODO(), stateful, metav1.UpdateOptions{}) if err != nil { logger.Error(err, "Redis stateful update failed") return err } - logger.Info("Redis stateful ") + logger.Info("Redis stateful successfully updated ") return nil }