Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Devops 632 add or delete exporter on update #12

Merged
merged 3 commits into from
Dec 29, 2017
Merged

Devops 632 add or delete exporter on update #12

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
55fbab6
[DEVOPS-632] When updating the redis statefulset, add or delete exporter
Dec 28, 2017
33d422b
[DEVOPS-632] Bump version to 0.1.3
Dec 29, 2017
b0e5469
[DEVOPS-632] Change default exporter resources/limits
Dec 29, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# The following are targers that do not exist in the filesystem as real files and should be always executed by make
.PHONY: default build deps-development docker-build shell run image unit-test test generate go-generate get-deps update-deps
VERSION := 0.1.2
VERSION := 0.1.3

# Name of this service/application
SERVICE_NAME := redis-operator
Expand Down
102 changes: 59 additions & 43 deletions pkg/failover/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ import (

// variables refering to the redis exporter port
const (
exporterPort = 9121
exporterPortName = "http-metrics"
exporterPort = 9121
exporterPortName = "http-metrics"
exporterContainerName = "redis-exporter"
)

const (
Expand Down Expand Up @@ -645,47 +646,7 @@ func (r *RedisFailoverKubeClient) CreateRedisStatefulset(rf *RedisFailover) erro
}

if rf.Spec.Redis.Exporter {
exporter := v1.Container{
Name: "redis-exporter",
Image: exporterImage,
ImagePullPolicy: "Always",
Ports: []v1.ContainerPort{
v1.ContainerPort{
Name: "metrics",
ContainerPort: exporterPort,
Protocol: v1.ProtocolTCP,
},
},
ReadinessProbe: &v1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/",
Port: intstr.FromString("metrics"),
},
},
},
LivenessProbe: &v1.Probe{
TimeoutSeconds: 3,
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/",
Port: intstr.FromString("metrics"),
},
},
},
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("300m"),
v1.ResourceMemory: resource.MustParse("300Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("200m"),
v1.ResourceMemory: resource.MustParse("150Mi"),
},
},
}
exporter := createRedisExporterContainer()
redisStatefulset.Spec.Template.Spec.Containers = append(redisStatefulset.Spec.Template.Spec.Containers, exporter)
}

Expand Down Expand Up @@ -818,6 +779,17 @@ func (r *RedisFailoverKubeClient) UpdateRedisStatefulset(rf *RedisFailover) erro
oldSS.Spec.Template.Spec.Containers[0].Resources = getRedisResources(rf.Spec)
oldSS.Spec.Template.Spec.Containers[0].Image = getRedisImage(rf)

if rf.Spec.Redis.Exporter {
exporter := createRedisExporterContainer()
oldSS.Spec.Template.Spec.Containers = append(oldSS.Spec.Template.Spec.Containers, exporter)
} else {
for pos, container := range oldSS.Spec.Template.Spec.Containers {
if container.Name == exporterContainerName {
oldSS.Spec.Template.Spec.Containers = append(oldSS.Spec.Template.Spec.Containers[:pos], oldSS.Spec.Template.Spec.Containers[pos+1:]...)
}
}
}

if _, err := r.Client.AppsV1beta1().StatefulSets(namespace).Update(oldSS); err != nil {
return err
}
Expand Down Expand Up @@ -955,3 +927,47 @@ func generateResourceList(cpu string, memory string) v1.ResourceList {
}
return resources
}

func createRedisExporterContainer() v1.Container {
return v1.Container{
Name: exporterContainerName,
Image: exporterImage,
ImagePullPolicy: "Always",
Ports: []v1.ContainerPort{
v1.ContainerPort{
Name: "metrics",
ContainerPort: exporterPort,
Protocol: v1.ProtocolTCP,
},
},
ReadinessProbe: &v1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 3,
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/",
Port: intstr.FromString("metrics"),
},
},
},
LivenessProbe: &v1.Probe{
TimeoutSeconds: 3,
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/",
Port: intstr.FromString("metrics"),
},
},
},
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("300m"),
v1.ResourceMemory: resource.MustParse("300Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("200m"),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Too much default requests, I think 25m cpu and 50Mi RAM is enough

v1.ResourceMemory: resource.MustParse("150Mi"),
},
},
}
}
241 changes: 241 additions & 0 deletions pkg/failover/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1659,6 +1659,247 @@ func TestUpdateRedisStatefulsetError(t *testing.T) {
assert.Error(err)
}

func TestUpdateRedisStatefulsetWithUpdate(t *testing.T) {
assert := assert.New(t)

replicas := int32(3)
replicasUpdated := int32(4)
called := false
cpu := "200m"
memory := "200Mi"
cpuQuantityOriginal, _ := resource.ParseQuantity("100m")
memoryQuantityOriginal, _ := resource.ParseQuantity("100Mi")
cpuQuantityRequired, _ := resource.ParseQuantity(cpu)
memoryQuantityRequired, _ := resource.ParseQuantity(memory)
var updatedRequests v1.ResourceRequirements

requiredRequests := v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: cpuQuantityRequired,
v1.ResourceMemory: memoryQuantityRequired,
},
Requests: v1.ResourceList{
v1.ResourceCPU: cpuQuantityRequired,
v1.ResourceMemory: memoryQuantityRequired,
},
}

exporterExists := false

// Create a faked K8S client
client := &fake.Clientset{}
client.Fake.AddReactor("get", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) {
r := replicas
if called {
r = replicasUpdated
}
statefulset := &v1beta1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: redisName,
Namespace: namespace,
},
Status: v1beta1.StatefulSetStatus{
ReadyReplicas: r,
UpdatedReplicas: r,
},
Spec: v1beta1.StatefulSetSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Name: redisName,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: cpuQuantityOriginal,
v1.ResourceMemory: memoryQuantityOriginal,
},
Requests: v1.ResourceList{
v1.ResourceCPU: cpuQuantityOriginal,
v1.ResourceMemory: memoryQuantityOriginal,
},
},
},
},
},
},
},
}
called = true
return true, statefulset, nil
})
client.Fake.AddReactor("update", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) {
updateAction := action.(k8stesting.UpdateAction)
statefulset := updateAction.GetObject().(*v1beta1.StatefulSet)
for _, container := range statefulset.Spec.Template.Spec.Containers {
if container.Name == redisName {
updatedRequests = container.Resources
}
if container.Name == "redis-exporter" {
exporterExists = true
}
}
return true, nil, nil
})

mc := &mocks.Clock{}
mc.On("NewTicker", mock.Anything).
Once().Return(time.NewTicker(1))
r := failover.NewRedisFailoverKubeClient(client, mc, log.Nil)

redisFailover := &failover.RedisFailover{
Metadata: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: failover.RedisFailoverSpec{
Redis: failover.RedisSettings{
Replicas: replicasUpdated,
Resources: failover.RedisFailoverResources{
Limits: failover.CPUAndMem{
CPU: cpu,
Memory: memory,
},
Requests: failover.CPUAndMem{
CPU: cpu,
Memory: memory,
},
},
Exporter: true,
},
Sentinel: failover.SentinelSettings{
Replicas: int32(3),
},
},
}

err := r.UpdateRedisStatefulset(redisFailover)
assert.NoError(err)
assert.Equal(requiredRequests, updatedRequests, "Requests are not equal as updated")
assert.True(exporterExists, "Redis-exporter should exist")
}

func TestUpdateRedisStatefulsetWithoutUpdate(t *testing.T) {
assert := assert.New(t)

replicas := int32(3)
replicasUpdated := int32(4)
called := false
cpu := "200m"
memory := "200Mi"
cpuQuantityOriginal, _ := resource.ParseQuantity("100m")
memoryQuantityOriginal, _ := resource.ParseQuantity("100Mi")
cpuQuantityRequired, _ := resource.ParseQuantity(cpu)
memoryQuantityRequired, _ := resource.ParseQuantity(memory)
var updatedRequests v1.ResourceRequirements

requiredRequests := v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: cpuQuantityRequired,
v1.ResourceMemory: memoryQuantityRequired,
},
Requests: v1.ResourceList{
v1.ResourceCPU: cpuQuantityRequired,
v1.ResourceMemory: memoryQuantityRequired,
},
}

exporterExists := false

// Create a faked K8S client
client := &fake.Clientset{}
client.Fake.AddReactor("get", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) {
r := replicas
if called {
r = replicasUpdated
}
statefulset := &v1beta1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: redisName,
Namespace: namespace,
},
Status: v1beta1.StatefulSetStatus{
ReadyReplicas: r,
UpdatedReplicas: r,
},
Spec: v1beta1.StatefulSetSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Name: redisName,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: cpuQuantityOriginal,
v1.ResourceMemory: memoryQuantityOriginal,
},
Requests: v1.ResourceList{
v1.ResourceCPU: cpuQuantityOriginal,
v1.ResourceMemory: memoryQuantityOriginal,
},
},
},
v1.Container{
Name: "redis-exporter",
},
},
},
},
},
}
called = true
return true, statefulset, nil
})
client.Fake.AddReactor("update", "statefulsets", func(action k8stesting.Action) (bool, runtime.Object, error) {
updateAction := action.(k8stesting.UpdateAction)
statefulset := updateAction.GetObject().(*v1beta1.StatefulSet)
for _, container := range statefulset.Spec.Template.Spec.Containers {
if container.Name == redisName {
updatedRequests = container.Resources
}
if container.Name == "redis-exporter" {
exporterExists = true
}
}
return true, nil, nil
})

mc := &mocks.Clock{}
mc.On("NewTicker", mock.Anything).
Once().Return(time.NewTicker(1))
r := failover.NewRedisFailoverKubeClient(client, mc, log.Nil)

redisFailover := &failover.RedisFailover{
Metadata: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: failover.RedisFailoverSpec{
Redis: failover.RedisSettings{
Replicas: replicasUpdated,
Resources: failover.RedisFailoverResources{
Limits: failover.CPUAndMem{
CPU: cpu,
Memory: memory,
},
Requests: failover.CPUAndMem{
CPU: cpu,
Memory: memory,
},
},
Exporter: false,
},
Sentinel: failover.SentinelSettings{
Replicas: int32(3),
},
},
}

err := r.UpdateRedisStatefulset(redisFailover)
assert.NoError(err)
assert.Equal(requiredRequests, updatedRequests, "Requests are not equal as updated")
assert.False(exporterExists, "Redis-exporter should not exist")
}

func TestUpdateRedisStatefulset(t *testing.T) {
assert := assert.New(t)

Expand Down