From 57563f6ff0ab428082d957c14228879b09733878 Mon Sep 17 00:00:00 2001 From: Sergio Ballesteros Date: Fri, 18 Aug 2023 12:53:53 +0200 Subject: [PATCH 1/4] Expect surge in running pods when rolling update deployment --- operator/redisfailover/service/check.go | 8 ++++++-- operator/redisfailover/util/pod.go | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 operator/redisfailover/util/pod.go diff --git a/operator/redisfailover/service/check.go b/operator/redisfailover/service/check.go index def501825..86401f0f9 100644 --- a/operator/redisfailover/service/check.go +++ b/operator/redisfailover/service/check.go @@ -12,6 +12,7 @@ import ( redisfailoverv1 "github.com/spotahome/redis-operator/api/redisfailover/v1" "github.com/spotahome/redis-operator/log" "github.com/spotahome/redis-operator/metrics" + "github.com/spotahome/redis-operator/operator/redisfailover/util" "github.com/spotahome/redis-operator/service/k8s" "github.com/spotahome/redis-operator/service/redis" ) @@ -498,10 +499,13 @@ func getRedisPort(p int32) string { func AreAllRunning(pods *corev1.PodList, expectedRunningPods int) bool { var runningPods int for _, pod := range pods.Items { - if pod.Status.Phase != corev1.PodRunning || pod.DeletionTimestamp != nil { + if util.PodIsScheduling(&pod) { + return false + } + if util.PodIsTerminal(&pod) { continue } runningPods++ } - return runningPods == expectedRunningPods + return runningPods >= expectedRunningPods } diff --git a/operator/redisfailover/util/pod.go b/operator/redisfailover/util/pod.go new file mode 100644 index 000000000..d0e4b0137 --- /dev/null +++ b/operator/redisfailover/util/pod.go @@ -0,0 +1,11 @@ +package util + +import v1 "k8s.io/api/core/v1" + +func PodIsTerminal(pod *v1.Pod) bool { + return pod.Status.Phase == v1.PodFailed || pod.Status.Phase == v1.PodSucceeded +} + +func PodIsScheduling(pod *v1.Pod) bool { + return pod.DeletionTimestamp != nil || pod.Status.Phase == v1.PodPending +} From 82f46b6f47b836d96544f5bf80d5902c0daeee29 Mon Sep 17 00:00:00 2001 From: Sergio Ballesteros Date: Fri, 18 Aug 2023 12:54:24 +0200 Subject: [PATCH 2/4] Use internal service for master/slave services --- operator/redisfailover/service/generator.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index d16fbaa49..079b350db 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -130,7 +130,8 @@ func generateRedisMasterService(rf *redisfailoverv1.RedisFailover, labels map[st Annotations: rf.Spec.Redis.ServiceAnnotations, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, Ports: []corev1.ServicePort{ { Name: "redis", @@ -163,7 +164,8 @@ func generateRedisSlaveService(rf *redisfailoverv1.RedisFailover, labels map[str Annotations: rf.Spec.Redis.ServiceAnnotations, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, Ports: []corev1.ServicePort{ { Name: "redis", From 1fdf21c83afe5834b7ad1a4d67ff96c4f141e6f6 Mon Sep 17 00:00:00 2001 From: Sergio Ballesteros Date: Sat, 19 Aug 2023 23:59:00 +0200 Subject: [PATCH 3/4] Test for ClusterIP services --- .../redisfailover/service/generator_test.go | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/operator/redisfailover/service/generator_test.go b/operator/redisfailover/service/generator_test.go index 54794571e..116e0e8b1 100644 --- a/operator/redisfailover/service/generator_test.go +++ b/operator/redisfailover/service/generator_test.go @@ -1322,7 +1322,8 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1361,7 +1362,8 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": "custom-name", @@ -1400,7 +1402,8 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1440,7 +1443,8 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1481,7 +1485,8 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1562,7 +1567,8 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1601,7 +1607,8 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": "custom-name", @@ -1640,7 +1647,8 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1680,7 +1688,8 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1721,7 +1730,8 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, From b2686ac5ec4ec3440e653eec780376a17819c96f Mon Sep 17 00:00:00 2001 From: Sergio Ballesteros Date: Sun, 20 Aug 2023 00:15:21 +0200 Subject: [PATCH 4/4] Use cluster IP for master and slave services --- operator/redisfailover/service/generator.go | 6 ++-- .../redisfailover/service/generator_test.go | 30 +++++++------------ 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 079b350db..f82eaab9a 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -130,8 +130,7 @@ func generateRedisMasterService(rf *redisfailoverv1.RedisFailover, labels map[st Annotations: rf.Spec.Redis.ServiceAnnotations, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, + Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { Name: "redis", @@ -164,8 +163,7 @@ func generateRedisSlaveService(rf *redisfailoverv1.RedisFailover, labels map[str Annotations: rf.Spec.Redis.ServiceAnnotations, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, + Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { Name: "redis", diff --git a/operator/redisfailover/service/generator_test.go b/operator/redisfailover/service/generator_test.go index 116e0e8b1..b14317072 100644 --- a/operator/redisfailover/service/generator_test.go +++ b/operator/redisfailover/service/generator_test.go @@ -1322,8 +1322,7 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1362,8 +1361,7 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": "custom-name", @@ -1402,8 +1400,7 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1443,8 +1440,7 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1485,8 +1481,7 @@ func TestRedisMasterService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1567,8 +1562,7 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1607,8 +1601,7 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": "custom-name", @@ -1647,8 +1640,7 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1688,8 +1680,7 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name, @@ -1730,8 +1721,7 @@ func TestRedisSlaveService(t *testing.T) { }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{ "app.kubernetes.io/component": "redis", "app.kubernetes.io/name": name,