Merge pull request #403 from cybozu-go/d-kuro/resize-pvc

Resizing PVC

api/v1beta2/mysqlcluster_types.go

@@ -8,6 +8,7 @@ import (
 	"github.com/robfig/cron/v3"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
@@ -125,6 +126,12 @@ func (s MySQLClusterSpec) validateCreate() field.ErrorList {
 		allErrs = append(allErrs, field.Required(pp, fmt.Sprintf("required volume claim template %s is missing", constants.MySQLDataVolumeName)))
 	}
 
+	for _, vc := range s.VolumeClaimTemplates {
+		if vc.Spec.Resources == nil || vc.Spec.Resources.Requests == nil || vc.Spec.Resources.Requests.Storage() == nil {
+			allErrs = append(allErrs, field.Required(pp, fmt.Sprintf("required volume claim template %s storage requests is missing", vc.Name)))
+		}
+	}
+
 	pp = p.Child("serverIDBase")
 	if s.ServerIDBase <= 0 {
 		allErrs = append(allErrs, field.Invalid(pp, s.ServerIDBase, "serverIDBase must be a positive integer"))
@@ -242,6 +249,24 @@ func (s MySQLClusterSpec) validateUpdate(old MySQLClusterSpec) field.ErrorList {
 		allErrs = append(allErrs, field.Forbidden(p, "not editable"))
 	}
 
+	oldPVCSet := make(map[string]PersistentVolumeClaim)
+	for _, oldPVC := range old.VolumeClaimTemplates {
+		oldPVCSet[oldPVC.Name] = oldPVC
+	}
+
+	for i, pvc := range s.VolumeClaimTemplates {
+		if old, ok := oldPVCSet[pvc.Name]; ok {
+			newSize := pvc.StorageSize()
+			oldSize := old.StorageSize()
+
+			if newSize.Cmp(oldSize) == -1 {
+				p := p.Child("volumeClaimTemplates").Index(i).
+					Child("spec").Child("resources").Child("requests").Key("storage")
+				allErrs = append(allErrs, field.Forbidden(p, "storage size cannot be reduced"))
+			}
+		}
+	}
+
 	return append(allErrs, s.validateCreate()...)
 }
 
@@ -365,6 +390,15 @@ type PersistentVolumeClaim struct {
 	Spec PersistentVolumeClaimSpecApplyConfiguration `json:"spec"`
 }
 
+func (in PersistentVolumeClaim) StorageSize() resource.Quantity {
+	if in.Spec.Resources != nil && in.Spec.Resources.Requests != nil {
+		requests := *in.Spec.Resources.Requests
+		return requests[corev1.ResourceStorage]
+	}
+
+	return resource.Quantity{}
+}
+
 // ToCoreV1 converts the PersistentVolumeClaim to a PersistentVolumeClaimApplyConfiguration.
 func (in PersistentVolumeClaim) ToCoreV1() *corev1ac.PersistentVolumeClaimApplyConfiguration {
 	// If you use this, the namespace will not be nil and will not match for "equality.Semantic.DeepEqual".

api/v1beta2/mysqlcluster_webhook_test.go

@@ -7,7 +7,9 @@ import (
 	"github.com/cybozu-go/moco/pkg/constants"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
 	"k8s.io/utils/pointer"
@@ -30,6 +32,13 @@ func makeMySQLCluster() *mocov1beta2.MySQLCluster {
 					ObjectMeta: mocov1beta2.ObjectMeta{
 						Name: "mysql-data",
 					},
+					Spec: mocov1beta2.PersistentVolumeClaimSpecApplyConfiguration(*corev1ac.PersistentVolumeClaimSpec().
+						WithStorageClassName("default").
+						WithResources(corev1ac.ResourceRequirements().
+							WithRequests(corev1.ResourceList{
+								corev1.ResourceStorage: resource.MustParse("1Gi"),
+							})),
+					),
 				},
 			},
 		},
@@ -82,6 +91,13 @@ var _ = Describe("MySQLCluster Webhook", func() {
 		Expect(err).To(HaveOccurred())
 	})
 
+	It("should deny without storage size in volume claim template", func() {
+		r := makeMySQLCluster()
+		r.Spec.VolumeClaimTemplates[0].Spec.Resources = nil
+		err := k8sClient.Create(ctx, r)
+		Expect(err).To(HaveOccurred())
+	})
+
 	It("should set a valid serverIDBase", func() {
 		r := makeMySQLCluster()
 		err := k8sClient.Create(ctx, r)
@@ -400,4 +416,42 @@ var _ = Describe("MySQLCluster Webhook", func() {
 		err = k8sClient.Update(ctx, r)
 		Expect(err).To(HaveOccurred())
 	})
+
+	It("should allow storage size expansion", func() {
+		r := makeMySQLCluster()
+		err := k8sClient.Create(ctx, r)
+		Expect(err).NotTo(HaveOccurred())
+
+		r.Spec.VolumeClaimTemplates[0].Spec = mocov1beta2.PersistentVolumeClaimSpecApplyConfiguration(
+			*corev1ac.PersistentVolumeClaimSpec().
+				WithStorageClassName("default").
+				WithResources(corev1ac.ResourceRequirements().
+					WithRequests(corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("10Gi"),
+					}),
+				),
+		)
+
+		err = k8sClient.Update(ctx, r)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should deny reduced storage size", func() {
+		r := makeMySQLCluster()
+		err := k8sClient.Create(ctx, r)
+		Expect(err).NotTo(HaveOccurred())
+
+		r.Spec.VolumeClaimTemplates[0].Spec = mocov1beta2.PersistentVolumeClaimSpecApplyConfiguration(
+			*corev1ac.PersistentVolumeClaimSpec().
+				WithStorageClassName("default").
+				WithResources(corev1ac.ResourceRequirements().
+					WithRequests(corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("1Mi"),
+					}),
+				),
+		)
+
+		err = k8sClient.Update(ctx, r)
+		Expect(err).To(HaveOccurred())
+	})
 })

charts/moco/templates/generated/generated.yaml

@@ -136,6 +136,16 @@ rules:
       - create
       - patch
       - update
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -298,6 +308,14 @@ rules:
       - patch
       - update
       - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

config/rbac/role.yaml

@@ -31,6 +31,16 @@ rules:
   - create
   - patch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -193,3 +203,11 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch

controllers/mysqlcluster_controller.go

@@ -9,10 +9,12 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	mocov1beta2 "github.com/cybozu-go/moco/api/v1beta2"
 	"github.com/cybozu-go/moco/clustering"
 	"github.com/cybozu-go/moco/pkg/constants"
+	"github.com/cybozu-go/moco/pkg/metrics"
 	"github.com/cybozu-go/moco/pkg/mycnf"
 	"github.com/cybozu-go/moco/pkg/password"
 	"github.com/google/go-cmp/cmp"
@@ -25,10 +27,12 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
 	appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1"
 	batchv1ac "k8s.io/client-go/applyconfigurations/batch/v1"
 	batchv1beta1ac "k8s.io/client-go/applyconfigurations/batch/v1beta1"
@@ -124,6 +128,8 @@ type MySQLClusterReconciler struct {
 //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups="",resources=configmaps/status,verbs=get
 //+kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch
+//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update;patch
+//+kubebuilder:rbac:groups="storage.k8s.io",resources=storageclasses,verbs=get;list;watch
 //+kubebuilder:rbac:groups="policy",resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups="cert-manager.io",resources=certificates,verbs=get;list;watch;create;delete
 //+kubebuilder:rbac:groups="batch",resources=cronjobs;jobs,verbs=get;list;watch;create;update;patch;delete
@@ -226,6 +232,10 @@ func (r *MySQLClusterReconciler) reconcileV1(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{}, err
 	}
 
+	if err := r.reconcilePVC(ctx, req, cluster); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	if err := r.reconcileV1StatefulSet(ctx, req, cluster, mycnf); err != nil {
 		log.Error(err, "failed to reconcile stateful set")
 		return ctrl.Result{}, err
@@ -913,14 +923,58 @@ func (r *MySQLClusterReconciler) reconcileV1StatefulSet(ctx context.Context, req
 		return nil
 	}
 
+	needRecreate := false
+
+	// Recreate StatefulSet if VolumeClaimTemplates has differences.
+	// sts will never be nil.
+	if origApplyConfig != nil && origApplyConfig.Spec != nil && origApplyConfig.Spec.VolumeClaimTemplates != nil {
+		if !equality.Semantic.DeepEqual(sts.Spec.VolumeClaimTemplates, origApplyConfig.Spec.VolumeClaimTemplates) {
+			needRecreate = true
+
+			// Don’t delete the Pod, only delete the StatefulSet.
+			// Same behavior as `kubectl delete sts moco-xxx --cascade=orphan`
+			opt := metav1.DeletePropagationOrphan
+			if err := r.Delete(ctx, &orig, &client.DeleteOptions{
+				PropagationPolicy: &opt,
+			}); err != nil {
+				metrics.StatefulSetRecreateErrorTotal.WithLabelValues(cluster.Name, cluster.Namespace).Inc()
+				return err
+			}
+
+			log.Info("volumeClaimTemplates has changed, delete StatefulSet and try to recreate it", "statefulSetName", cluster.PrefixedName())
+
+			// When DeletePropagationOrphan is used to delete, it waits because it is not deleted immediately.
+			if err := wait.PollImmediate(time.Millisecond*500, time.Second*5, func() (bool, error) {
+				err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.PrefixedName()}, &appsv1.StatefulSet{})
+				if err != nil {
+					if apierrors.IsNotFound(err) {
+						return true, nil
+					}
+					return false, err
+				}
+
+				return false, fmt.Errorf("re-creation failed the StatefulSet %s/%s has not been deleted", cluster.Namespace, cluster.PrefixedName())
+			}); err != nil {
+				return err
+			}
+		}
+	}
+
 	err = r.Patch(ctx, patch, client.Apply, &client.PatchOptions{
 		FieldManager: fieldManager,
 		Force:        pointer.Bool(true),
 	})
 	if err != nil {
+		if needRecreate {
+			metrics.StatefulSetRecreateErrorTotal.WithLabelValues(cluster.Name, cluster.Namespace).Inc()
+		}
 		return fmt.Errorf("failed to reconcile stateful set: %w", err)
 	}
 
+	if needRecreate {
+		metrics.StatefulSetRecreateTotal.WithLabelValues(cluster.Name, cluster.Namespace).Inc()
+	}
+
 	if debugController {
 		var updated appsv1.StatefulSet
 		if err := r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.PrefixedName()}, &updated); err != nil && !apierrors.IsNotFound(err) {

controllers/mysqlcluster_controller_test.go

@@ -686,6 +686,7 @@ var _ = Describe("MySQLCluster reconciler", func() {
 			if cluster.Status.ReconcileInfo.Generation != cluster.Generation {
 				return fmt.Errorf("status is not updated")
 			}
+
 			return nil
 		}).Should(Succeed())