From 55924c34519a95a610a6b434c2aea3ec63b7a2f3 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Thu, 22 Sep 2022 17:28:29 +0800 Subject: [PATCH 1/7] [feature]resize pvc // 1.Get the data already stored internally // 2.Get the desired data // 3.Start querying the pvc list when you find data inconsistencies // 3.1 Comparison using real pvc capacity and desired data // 3.1.1 Update if you find inconsistencies // 3.2 Writing successful updates to internal // 4. Set to old VolumeClaimTemplates to update.Prevent update error reporting // 5. Set to old annotations to update --- service/k8s/statefulset.go | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/service/k8s/statefulset.go b/service/k8s/statefulset.go index 0bbbee003..db02b3b8a 100644 --- a/service/k8s/statefulset.go +++ b/service/k8s/statefulset.go @@ -3,6 +3,8 @@ package k8s import ( "context" "fmt" + "k8s.io/apimachinery/pkg/labels" + "strconv" "strings" appsv1 "k8s.io/api/apps/v1" @@ -99,6 +101,69 @@ func (s *StatefulSetService) CreateOrUpdateStatefulSet(namespace string, statefu // namespace is our spec(https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#concurrency-control-and-consistency), // we will replace the current namespace state. statefulSet.ResourceVersion = storedStatefulSet.ResourceVersion + // resize pvc + // 1.Get the data already stored internally + // 2.Get the desired data + // 3.Start querying the pvc list when you find data inconsistencies + // 3.1 Comparison using real pvc capacity and desired data + // 3.1.1 Update if you find inconsistencies + // 3.2 Writing successful updates to internal + // 4. Set to old VolumeClaimTemplates to update.Prevent update error reporting + // 5. Set to old annotations to update + annotations := storedStatefulSet.Annotations + if annotations == nil { + annotations = map[string]string{ + "storageCapacity": "0", + } + } + storedCapacity, _ := strconv.ParseInt(annotations["storageCapacity"], 0, 64) + if len(statefulSet.Spec.VolumeClaimTemplates) != 0 { + stateCapacity := statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Value() + if storedCapacity != stateCapacity { + rfName := strings.TrimPrefix(storedStatefulSet.Name, "rfr-") + listOpt := metav1.ListOptions{ + LabelSelector: labels.FormatLabels( + map[string]string{ + "app.kubernetes.io/component": "redis", + "app.kubernetes.io/name": strings.TrimPrefix(storedStatefulSet.Name, "rfr-"), + "app.kubernetes.io/part-of": "redis-failover", + }, + ), + } + pvcs, err := s.kubeClient.CoreV1().PersistentVolumeClaims(storedStatefulSet.Namespace).List(context.Background(), listOpt) + if err != nil { + return err + } + updateFailed := false + realUpdate := false + for _, pvc := range pvcs.Items { + realCapacity := pvc.Spec.Resources.Requests.Storage().Value() + if realCapacity != stateCapacity { + realUpdate = true + pvc.Spec.Resources.Requests = statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests + _, err = s.kubeClient.CoreV1().PersistentVolumeClaims(storedStatefulSet.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) + if err != nil { + if !updateFailed { + updateFailed = true + } + s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Warningf("resize pvc failed:%s", err.Error()) + } + } + } + if !updateFailed && len(pvcs.Items) != 0 { + annotations["storageCapacity"] = fmt.Sprintf("%d", stateCapacity) + storedStatefulSet.Annotations = annotations + if realUpdate { + s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Infof("resize pvc will resize from %d to %d Success", storedCapacity, stateCapacity) + } else { + s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Warningf("set annotations,resize nothing") + } + } + } + } + // set stored.volumeClaimTemplates + statefulSet.Spec.VolumeClaimTemplates = storedStatefulSet.Spec.VolumeClaimTemplates + statefulSet.Annotations = storedStatefulSet.Annotations return s.UpdateStatefulSet(namespace, statefulSet) } From 45c16cf30f1ed70d43723047841241f35b02eec2 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Thu, 22 Sep 2022 17:48:59 +0800 Subject: [PATCH 2/7] goimport --- service/k8s/statefulset.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/k8s/statefulset.go b/service/k8s/statefulset.go index db02b3b8a..244f582b8 100644 --- a/service/k8s/statefulset.go +++ b/service/k8s/statefulset.go @@ -3,10 +3,11 @@ package k8s import ( "context" "fmt" - "k8s.io/apimachinery/pkg/labels" "strconv" "strings" + "k8s.io/apimachinery/pkg/labels" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" From 6d89b9ad62ed2bee1807a616ad27a7ed95e1fa70 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Fri, 25 Nov 2022 11:02:57 +0800 Subject: [PATCH 3/7] resize pvc --- service/k8s/statefulset.go | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/service/k8s/statefulset.go b/service/k8s/statefulset.go index 8de666ba4..e35186d47 100644 --- a/service/k8s/statefulset.go +++ b/service/k8s/statefulset.go @@ -3,8 +3,11 @@ package k8s import ( "context" "fmt" + "strconv" "strings" + "k8s.io/apimachinery/pkg/labels" + "github.com/spotahome/redis-operator/operator/redisfailover/util" appsv1 "k8s.io/api/apps/v1" @@ -107,6 +110,68 @@ func (s *StatefulSetService) CreateOrUpdateStatefulSet(namespace string, statefu // namespace is our spec(https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#concurrency-control-and-consistency), // we will replace the current namespace state. statefulSet.ResourceVersion = storedStatefulSet.ResourceVersion + // resize pvc + // 1.Get the data already stored internally + // 2.Get the desired data + // 3.Start querying the pvc list when you find data inconsistencies + // 3.1 Comparison using real pvc capacity and desired data + // 3.1.1 Update if you find inconsistencies + // 3.2 Writing successful updates to internal + // 4. Set to old VolumeClaimTemplates to update.Prevent update error reporting + // 5. Set to old annotations to update + annotations := storedStatefulSet.Annotations + if annotations == nil { + annotations = map[string]string{ + "storageCapacity": "0", + } + } + storedCapacity, _ := strconv.ParseInt(annotations["storageCapacity"], 0, 64) + if len(statefulSet.Spec.VolumeClaimTemplates) != 0 { + stateCapacity := statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Value() + if storedCapacity != stateCapacity { + rfName := strings.TrimPrefix(storedStatefulSet.Name, "rfr-") + listOpt := metav1.ListOptions{ + LabelSelector: labels.FormatLabels( + map[string]string{ + "app.kubernetes.io/component": "redis", + "app.kubernetes.io/name": strings.TrimPrefix(storedStatefulSet.Name, "rfr-"), + "app.kubernetes.io/part-of": "redis-failover", + }, + ), + } + pvcs, err := s.kubeClient.CoreV1().PersistentVolumeClaims(storedStatefulSet.Namespace).List(context.Background(), listOpt) + if err != nil { + return err + } + updateFailed := false + realUpdate := false + for _, pvc := range pvcs.Items { + realCapacity := pvc.Spec.Resources.Requests.Storage().Value() + if realCapacity != stateCapacity { + realUpdate = true + pvc.Spec.Resources.Requests = statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests + _, err = s.kubeClient.CoreV1().PersistentVolumeClaims(storedStatefulSet.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) + if err != nil { + if !updateFailed { + updateFailed = true + } + s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Warningf("resize pvc failed:%s", err.Error()) + } + } + } + if !updateFailed && len(pvcs.Items) != 0 { + annotations["storageCapacity"] = fmt.Sprintf("%d", stateCapacity) + storedStatefulSet.Annotations = annotations + if realUpdate { + s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Infof("resize pvc will resize from %d to %d Success", storedCapacity, stateCapacity) + } else { + s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Warningf("set annotations,resize nothing") + } + } + } + } + // set stored.volumeClaimTemplates + statefulSet.Spec.VolumeClaimTemplates = storedStatefulSet.Spec.VolumeClaimTemplates statefulSet.Annotations = util.MergeAnnotations(statefulSet.Annotations, storedStatefulSet.Annotations) return s.UpdateStatefulSet(namespace, statefulSet) } From 555d20a24ef04355d460458729aa63bd0def8d16 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Tue, 6 Dec 2022 11:13:34 +0800 Subject: [PATCH 4/7] resize pvc --- service/k8s/statefulset_test.go | 104 ++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/service/k8s/statefulset_test.go b/service/k8s/statefulset_test.go index 57b653710..1d6781767 100644 --- a/service/k8s/statefulset_test.go +++ b/service/k8s/statefulset_test.go @@ -4,6 +4,10 @@ import ( "errors" "testing" + "k8s.io/apimachinery/pkg/api/resource" + + v1 "k8s.io/api/core/v1" + "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" kubeerrors "k8s.io/apimachinery/pkg/api/errors" @@ -116,4 +120,104 @@ func TestStatefulSetServiceGetCreateOrUpdate(t *testing.T) { } }) } + // test resize pvc + { + t.Run("test_Resize_Pvc", func(t *testing.T) { + assert := assert.New(t) + beforeSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "teststatefulSet1", + ResourceVersion: "10", + }, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("0.5Gi"), + }, + }, + }, + }, + }, + }, + } + afterSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "teststatefulSet1", + ResourceVersion: "10", + }, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } + pvcList := &v1.PersistentVolumeClaimList{ + Items: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/component": "redis", + "app.kubernetes.io/name": "teststatefulSet1", + "app.kubernetes.io/part-of": "redis-failover", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "vol-1", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("0.5Gi"), + }, + }, + }, + }, + // resized already + { + Spec: v1.PersistentVolumeClaimSpec{ + VolumeName: "vol-2", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + } + // Mock. + mcli := &kubernetes.Clientset{} + mcli.AddReactor("get", "statefulsets", func(action kubetesting.Action) (bool, runtime.Object, error) { + return true, beforeSts, nil + }) + mcli.AddReactor("list", "persistentvolumeclaims", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + return true, pvcList, nil + }) + mcli.AddReactor("update", "persistentvolumeclaims", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + // update pvc[0] + pvcList.Items[0] = *action.(kubetesting.UpdateActionImpl).Object.(*v1.PersistentVolumeClaim) + return true, action.(kubetesting.UpdateActionImpl).Object, nil + }) + service := k8s.NewStatefulSetService(mcli, log.Dummy, metrics.Dummy) + err := service.CreateOrUpdateStatefulSet(testns, afterSts) + assert.NoError(err) + assert.Equal(pvcList.Items[0].Spec.Resources, pvcList.Items[1].Spec.Resources) + // should not call update + mcli.AddReactor("update", "persistentvolumeclaims", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + panic("shouldn't call update") + }) + service = k8s.NewStatefulSetService(mcli, log.Dummy, metrics.Dummy) + err = service.CreateOrUpdateStatefulSet(testns, afterSts) + assert.NoError(err) + }) + } } From 817c07a245a23a072af533e6b34ab3e69942238d Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Sat, 10 Dec 2022 07:44:00 +0800 Subject: [PATCH 5/7] Update service/k8s/statefulset.go Co-authored-by: Sergio Ballesteros --- service/k8s/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/k8s/statefulset.go b/service/k8s/statefulset.go index e202af649..8b80c8308 100644 --- a/service/k8s/statefulset.go +++ b/service/k8s/statefulset.go @@ -163,7 +163,7 @@ func (s *StatefulSetService) CreateOrUpdateStatefulSet(namespace string, statefu annotations["storageCapacity"] = fmt.Sprintf("%d", stateCapacity) storedStatefulSet.Annotations = annotations if realUpdate { - s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Infof("resize pvc will resize from %d to %d Success", storedCapacity, stateCapacity) + s.logger.WithField("namespace", namespace).WithField("statefulSet", statefulSet.Name).Infof("resize statefulset pvcs from %d to %d Success", storedCapacity, stateCapacity) } else { s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Warningf("set annotations,resize nothing") } From b3d4bcdf812420a21188f1f26173bd0b24932bfe Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Sat, 10 Dec 2022 07:44:55 +0800 Subject: [PATCH 6/7] Update service/k8s/statefulset.go Co-authored-by: Sergio Ballesteros --- service/k8s/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/k8s/statefulset.go b/service/k8s/statefulset.go index 8b80c8308..cd243c6a2 100644 --- a/service/k8s/statefulset.go +++ b/service/k8s/statefulset.go @@ -155,7 +155,7 @@ func (s *StatefulSetService) CreateOrUpdateStatefulSet(namespace string, statefu if !updateFailed { updateFailed = true } - s.logger.WithField("namespace", namespace).WithField("pvc", rfName).Warningf("resize pvc failed:%s", err.Error()) + s.logger.WithField("namespace", namespace).WithField("pvc", pvc.Name).Warningf("resize pvc failed:%s", err.Error()) } } } From e11885ea7f3d2ad664e0ed6168572a3d7086d2a4 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Mon, 12 Dec 2022 14:16:51 +0000 Subject: [PATCH 7/7] Update statefulset.go --- service/k8s/statefulset.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/service/k8s/statefulset.go b/service/k8s/statefulset.go index cd243c6a2..38cc95ff2 100644 --- a/service/k8s/statefulset.go +++ b/service/k8s/statefulset.go @@ -152,9 +152,7 @@ func (s *StatefulSetService) CreateOrUpdateStatefulSet(namespace string, statefu pvc.Spec.Resources.Requests = statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests _, err = s.kubeClient.CoreV1().PersistentVolumeClaims(storedStatefulSet.Namespace).Update(context.Background(), &pvc, metav1.UpdateOptions{}) if err != nil { - if !updateFailed { - updateFailed = true - } + updateFailed = true s.logger.WithField("namespace", namespace).WithField("pvc", pvc.Name).Warningf("resize pvc failed:%s", err.Error()) } }