From a0bc0f414d9e3e3f69fae7f5ffe099237e31bf0f Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Tue, 2 Jan 2018 15:19:16 +0100 Subject: [PATCH 1/5] [DEVOPS-649] Add AntiAffinity to redis and sentinel pods --- pkg/failover/client.go | 104 ++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index 539528704..b0a148ddb 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -31,13 +31,15 @@ const ( ) const ( - description = "Manage a Redis Failover deployment" - baseName = "rf" - bootstrapName = "b" - sentinelName = "s" - sentinelRoleName = "sentinel" - redisName = "r" - redisRoleName = "redis" + description = "Manage a Redis Failover deployment" + baseName = "rf" + bootstrapName = "b" + sentinelName = "s" + sentinelRoleName = "sentinel" + redisName = "r" + redisRoleName = "redis" + bootstrapRoleName = "bootstrap" + appLabel = "redis-failover" ) const ( @@ -119,6 +121,14 @@ func getRedisImage(rf *RedisFailover) string { return fmt.Sprintf("%s:%s", config.RedisImage, rf.Spec.Redis.Version) } +func generateLabels(component, role string) map[string]string { + return map[string]string{ + "app": appLabel, + "component": component, + component: role, + } +} + // GetAllRedisfailovers connects to k8s and returns all RF deployed on cluster func (r *RedisFailoverKubeClient) GetAllRedisfailovers() (*RedisFailoverList, error) { uri := fmt.Sprintf("/apis/%s/%s/%s/", config.Domain, config.Version, config.APIName) @@ -260,15 +270,13 @@ func (r *RedisFailoverKubeClient) CreateBootstrapPod(rf *RedisFailover) error { redisResources := getRedisResources(spec) sentinelResources := getSentinelResources(spec) + labels := generateLabels(rf.Metadata.Name, bootstrapRoleName) + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, - Labels: map[string]string{ - "app": "redis-failover", - "component": "sentinel", - "sentinel": rf.Metadata.Name, - }, + Labels: labels, }, Spec: v1.PodSpec{ Containers: []v1.Container{ @@ -362,22 +370,16 @@ func (r *RedisFailoverKubeClient) CreateSentinelService(rf *RedisFailover) error sentinelTargetPort := intstr.FromInt(26379) + labels := generateLabels(rf.Metadata.Name, sentinelRoleName) + sentinelSvc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, - Labels: map[string]string{ - "app": "redis-failover", - "component": "sentinel", - "sentinel": rf.Metadata.Name, - }, + Labels: labels, }, Spec: v1.ServiceSpec{ - Selector: map[string]string{ - "app": "redis-failover", - "component": "sentinel", - "sentinel": rf.Metadata.Name, - }, + Selector: labels, Ports: []v1.ServicePort{ v1.ServicePort{ Name: "sentinel", @@ -413,6 +415,8 @@ func (r *RedisFailoverKubeClient) CreateSentinelDeployment(rf *RedisFailover) er resources := getSentinelResources(spec) + labels := generateLabels(rf.Metadata.Name, sentinelRoleName) + sentinelDeployment := &v1beta1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -422,13 +426,20 @@ func (r *RedisFailoverKubeClient) CreateSentinelDeployment(rf *RedisFailover) er Replicas: &spec.Sentinel.Replicas, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "redis-failover", - "component": "sentinel", - "sentinel": rf.Metadata.Name, - }, + Labels: labels, }, Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + }, + }, + }, + }, InitContainers: []v1.Container{ v1.Container{ Name: "sentinel-config", @@ -549,6 +560,8 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro resources := getRedisResources(spec) + labels := generateLabels(rf.Metadata.Name, redisRoleName) + redisStatefulset := &v1beta1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -562,13 +575,20 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "redis-failover", - "component": "redis", - "redis": rf.Metadata.Name, - }, + Labels: labels, }, Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + }, + }, + }, + }, InitContainers: []v1.Container{ v1.Container{ Name: "redis-config", @@ -682,6 +702,9 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro func (r *RedisFailoverKubeClient) CreateRedisService(rf *RedisFailover) error { name := r.GetRedisName(rf) namespace := rf.Metadata.Namespace + + labels := generateLabels(rf.Metadata.Name, redisRoleName) + srv := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -697,11 +720,7 @@ func (r *RedisFailoverKubeClient) CreateRedisService(rf *RedisFailover) error { Name: exporterPortName, }, }, - Selector: map[string]string{ - "app": "redis-failover", - "component": "redis", - "redis": rf.Metadata.Name, - }, + Selector: labels, }, } @@ -710,11 +729,14 @@ func (r *RedisFailoverKubeClient) CreateRedisService(rf *RedisFailover) error { } // createPodDisruptionBudget creates a PodDisruptionBudget for redis or sentinel -func (r *RedisFailoverKubeClient) createPodDisruptionBudget(rf *RedisFailover, name string, role string) error { +func (r *RedisFailoverKubeClient) createPodDisruptionBudget(rf *RedisFailover, name string, component string) error { name = generateName(name, rf.Metadata.Name) namespace := rf.Metadata.Namespace if _, err := r.Client.PolicyV1beta1().PodDisruptionBudgets(namespace).Get(name, metav1.GetOptions{}); err != nil { minAvailable := intstr.FromInt(2) + + labels := generateLabels(component, rf.Metadata.Name) + pdb := &policy.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -723,11 +745,7 @@ func (r *RedisFailoverKubeClient) createPodDisruptionBudget(rf *RedisFailover, n Spec: policy.PodDisruptionBudgetSpec{ MinAvailable: &minAvailable, Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "redis-failover", - "component": role, - role: rf.Metadata.Name, - }, + MatchLabels: labels, }, }, } From 21deda99d518a1b5834412953359b156768ad049 Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Tue, 2 Jan 2018 15:20:30 +0100 Subject: [PATCH 2/5] [DEVOPS-FIX] Lowercase errors --- pkg/failover/client.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index b0a148ddb..f79fa6825 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -175,7 +175,7 @@ func (r *RedisFailoverKubeClient) GetBootstrapPod(rf *RedisFailover) (*v1.Pod, e namespace := rf.Metadata.Namespace pod, err := r.Client.Core().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { - return nil, errors.New("Could not get pod") + return nil, errors.New("could not get pod") } return pod, nil } @@ -186,7 +186,7 @@ func (r *RedisFailoverKubeClient) GetSentinelService(rf *RedisFailover) (*v1.Ser namespace := rf.Metadata.Namespace service, err := r.Client.Core().Services(namespace).Get(name, metav1.GetOptions{}) if err != nil { - return nil, errors.New("Could not get service") + return nil, errors.New("could not get service") } return service, nil } @@ -197,7 +197,7 @@ func (r *RedisFailoverKubeClient) GetSentinelDeployment(rf *RedisFailover) (*v1b namespace := rf.Metadata.Namespace deployment, err := r.Client.AppsV1beta1().Deployments(namespace).Get(name, metav1.GetOptions{}) if err != nil { - return nil, errors.New("Could not get deployment") + return nil, errors.New("could not get deployment") } return deployment, nil } @@ -208,7 +208,7 @@ func (r *RedisFailoverKubeClient) GetRedisService(rf *RedisFailover) (*v1.Servic namespace := rf.Metadata.Namespace service, err := r.Client.Core().Services(namespace).Get(name, metav1.GetOptions{}) if err != nil { - return nil, errors.New("Could not get service") + return nil, errors.New("could not get service") } return service, nil } @@ -219,7 +219,7 @@ func (r *RedisFailoverKubeClient) GetRedisStatefulset(rf *RedisFailover) (*v1bet namespace := rf.Metadata.Namespace statefulset, err := r.Client.AppsV1beta1().StatefulSets(namespace).Get(name, metav1.GetOptions{}) if err != nil { - return nil, errors.New("Could not get statefulset") + return nil, errors.New("could not get statefulset") } return statefulset, nil } @@ -233,7 +233,7 @@ func (r *RedisFailoverKubeClient) GetSentinelPodsIPs(rf *RedisFailover) ([]strin return nil, err } if len(endpoints.Subsets) != 1 { - return nil, errors.New("The Sentinel Service has different endpoints than expected") + return nil, errors.New("the Sentinel Service has different endpoints than expected") } pods := []string{} for _, e := range endpoints.Subsets[0].Addresses { From ca1b90a7dc4dae537b00c9dbb26c11bab6df83d3 Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Tue, 2 Jan 2018 15:33:34 +0100 Subject: [PATCH 3/5] [DEVOPS-649] Add topology key --- pkg/failover/client.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index f79fa6825..20faa356d 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -31,15 +31,16 @@ const ( ) const ( - description = "Manage a Redis Failover deployment" - baseName = "rf" - bootstrapName = "b" - sentinelName = "s" - sentinelRoleName = "sentinel" - redisName = "r" - redisRoleName = "redis" - bootstrapRoleName = "bootstrap" - appLabel = "redis-failover" + description = "Manage a Redis Failover deployment" + baseName = "rf" + bootstrapName = "b" + sentinelName = "s" + sentinelRoleName = "sentinel" + redisName = "r" + redisRoleName = "redis" + bootstrapRoleName = "bootstrap" + appLabel = "redis-failover" + hostnameTopologyKey = "kubernetes.io/hostname" ) const ( @@ -433,6 +434,7 @@ func (r *RedisFailoverKubeClient) CreateSentinelDeployment(rf *RedisFailover) er PodAntiAffinity: &v1.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ v1.PodAffinityTerm{ + TopologyKey: hostnameTopologyKey, LabelSelector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -582,6 +584,7 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro PodAntiAffinity: &v1.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ v1.PodAffinityTerm{ + TopologyKey: hostnameTopologyKey, LabelSelector: &metav1.LabelSelector{ MatchLabels: labels, }, From c541ecd15600aac3e690e6bfba762f5e30d6d504 Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Tue, 2 Jan 2018 15:41:28 +0100 Subject: [PATCH 4/5] [DEVOPS-649] Fix generate labels order --- pkg/failover/client.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index 20faa356d..1961c659d 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -271,7 +271,7 @@ func (r *RedisFailoverKubeClient) CreateBootstrapPod(rf *RedisFailover) error { redisResources := getRedisResources(spec) sentinelResources := getSentinelResources(spec) - labels := generateLabels(rf.Metadata.Name, bootstrapRoleName) + labels := generateLabels(bootstrapRoleName, rf.Metadata.Name) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -371,7 +371,7 @@ func (r *RedisFailoverKubeClient) CreateSentinelService(rf *RedisFailover) error sentinelTargetPort := intstr.FromInt(26379) - labels := generateLabels(rf.Metadata.Name, sentinelRoleName) + labels := generateLabels(sentinelRoleName, rf.Metadata.Name) sentinelSvc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -416,7 +416,7 @@ func (r *RedisFailoverKubeClient) CreateSentinelDeployment(rf *RedisFailover) er resources := getSentinelResources(spec) - labels := generateLabels(rf.Metadata.Name, sentinelRoleName) + labels := generateLabels(sentinelRoleName, rf.Metadata.Name) sentinelDeployment := &v1beta1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -562,7 +562,7 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro resources := getRedisResources(spec) - labels := generateLabels(rf.Metadata.Name, redisRoleName) + labels := generateLabels(redisRoleName, rf.Metadata.Name) redisStatefulset := &v1beta1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -706,7 +706,7 @@ func (r *RedisFailoverKubeClient) CreateRedisService(rf *RedisFailover) error { name := r.GetRedisName(rf) namespace := rf.Metadata.Namespace - labels := generateLabels(rf.Metadata.Name, redisRoleName) + labels := generateLabels(redisRoleName, rf.Metadata.Name) srv := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ From e1be6cce23bc3f9eed845de0a60a44f21511dabf Mon Sep 17 00:00:00 2001 From: Julio Chana Date: Tue, 2 Jan 2018 15:50:04 +0100 Subject: [PATCH 5/5] [DEVOPS-649] Bootstrap has to be treated as a sentinel so it can be found by service --- pkg/failover/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/failover/client.go b/pkg/failover/client.go index 1961c659d..9ab9b9132 100644 --- a/pkg/failover/client.go +++ b/pkg/failover/client.go @@ -38,7 +38,6 @@ const ( sentinelRoleName = "sentinel" redisName = "r" redisRoleName = "redis" - bootstrapRoleName = "bootstrap" appLabel = "redis-failover" hostnameTopologyKey = "kubernetes.io/hostname" ) @@ -271,7 +270,7 @@ func (r *RedisFailoverKubeClient) CreateBootstrapPod(rf *RedisFailover) error { redisResources := getRedisResources(spec) sentinelResources := getSentinelResources(spec) - labels := generateLabels(bootstrapRoleName, rf.Metadata.Name) + labels := generateLabels(sentinelRoleName, rf.Metadata.Name) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{