diff --git a/api/v1beta2/mysqlcluster_types.go b/api/v1beta2/mysqlcluster_types.go index a0eb52254..143c87870 100644 --- a/api/v1beta2/mysqlcluster_types.go +++ b/api/v1beta2/mysqlcluster_types.go @@ -241,7 +241,7 @@ func (s MySQLClusterSpec) validateUpdate(old MySQLClusterSpec) field.ErrorList { p := p.Child("restore") allErrs = append(allErrs, field.Forbidden(p, "not editable")) } - + return append(allErrs, s.validateCreate()...) } @@ -502,9 +502,10 @@ type MySQLClusterConditionType string // Valid values for MySQLClusterConditionType const ( - ConditionInitialized MySQLClusterConditionType = "Initialized" - ConditionAvailable MySQLClusterConditionType = "Available" - ConditionHealthy MySQLClusterConditionType = "Healthy" + ConditionInitialized MySQLClusterConditionType = "Initialized" + ConditionAvailable MySQLClusterConditionType = "Available" + ConditionHealthy MySQLClusterConditionType = "Healthy" + ConditionVolumeResized MySQLClusterConditionType = "VolumeResized" ) // BackupStatus represents the status of the last successful backup. diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go index 9bafe5c04..0ea690a6e 100644 --- a/controllers/mysqlcluster_controller_test.go +++ b/controllers/mysqlcluster_controller_test.go @@ -686,6 +686,7 @@ var _ = Describe("MySQLCluster reconciler", func() { if cluster.Status.ReconcileInfo.Generation != cluster.Generation { return fmt.Errorf("status is not updated") } + return nil }).Should(Succeed()) diff --git a/controllers/resize_pvc.go b/controllers/resize_pvc.go index ae31b8844..26e201b67 100644 --- a/controllers/resize_pvc.go +++ b/controllers/resize_pvc.go @@ -9,19 +9,21 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - corev1ac "k8s.io/client-go/applyconfigurations/core/v1" - "k8s.io/utils/pointer" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" crlog "sigs.k8s.io/controller-runtime/pkg/log" ) +var ( + ErrReduceVolumeSize = errors.New("cannot reduce volume size") +) + // reconcilePVC resizes the PVC as needed. // Since the PVC template of the StatefulSet is unchangeable, the following steps are required to resize the PVC // @@ -45,30 +47,44 @@ func (r *MySQLClusterReconciler) reconcilePVC(ctx context.Context, req ctrl.Requ return nil } - resizeTarget, ok := r.needResizePVC(cluster, &sts) + resizeTarget, ok, err := r.needResizePVC(cluster, &sts) if !ok { return nil } + if err != nil { + if condErr := r.updateResizeCondition(ctx, req, corev1.ConditionFalse, err.Error()); condErr != nil { + return fmt.Errorf("failed to update resize condition status fields in MySQLCluster: %s: %w", condErr, err) + } + + return err + } log.Info("Starting PVC resize") - patches, err := r.findPVCs(ctx, cluster, &sts, resizeTarget) - if err != nil { - return fmt.Errorf("failed to resize PVC: %w", err) - } + if err := r.resizePVCs(ctx, cluster, &sts, resizeTarget); err != nil { + if condErr := r.updateResizeCondition(ctx, req, corev1.ConditionFalse, err.Error()); condErr != nil { + return fmt.Errorf("failed to update resize condition status: %s: %w", condErr, err) + } - if err := r.resizePVCs(ctx, patches); err != nil { - return fmt.Errorf("failed to resize PVCs: %w", err) + return err } if err := r.deleteStatefulSet(ctx, &sts); err != nil { - return fmt.Errorf("failed to delete StatefulSet: %w", err) + if condErr := r.updateResizeCondition(ctx, req, corev1.ConditionFalse, err.Error()); condErr != nil { + return fmt.Errorf("failed to update resize condition status: %s: %w", condErr, err) + } + + return err + } + + if err := r.updateResizeCondition(ctx, req, corev1.ConditionTrue, "successfully resized pvc"); err != nil { + return fmt.Errorf("failed to update resize condition status: %w", err) } return nil } -func (r *MySQLClusterReconciler) findPVCs(ctx context.Context, cluster *mocov1beta2.MySQLCluster, sts *appsv1.StatefulSet, resizeTarget map[string]corev1.PersistentVolumeClaim) (map[types.NamespacedName]*unstructured.Unstructured, error) { +func (r *MySQLClusterReconciler) resizePVCs(ctx context.Context, cluster *mocov1beta2.MySQLCluster, sts *appsv1.StatefulSet, resizeTarget map[string]corev1.PersistentVolumeClaim) error { log := crlog.FromContext(ctx) newSizes := make(map[string]*resource.Quantity) @@ -80,9 +96,15 @@ func (r *MySQLClusterReconciler) findPVCs(ctx context.Context, cluster *mocov1be newSizes[pvc.Name] = newSize } - pvcsToKeep := make(map[string]*resource.Quantity, int(*sts.Spec.Replicas)*len(resizeTarget)) + var replicas int32 + if sts.Spec.Replicas == nil { + replicas = 1 + } else { + replicas = *sts.Spec.Replicas + } + pvcsToKeep := make(map[string]*resource.Quantity, replicas*int32(len(resizeTarget))) for _, pvc := range resizeTarget { - for i := int32(0); i < *sts.Spec.Replicas; i++ { + for i := int32(0); i < replicas; i++ { name := fmt.Sprintf("%s-%s-%d", pvc.Name, sts.Name, i) newSize := newSizes[pvc.Name] pvcsToKeep[name] = newSize @@ -91,16 +113,14 @@ func (r *MySQLClusterReconciler) findPVCs(ctx context.Context, cluster *mocov1be selector, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector) if err != nil { - return nil, fmt.Errorf("failed to parse selector: %w", err) + return fmt.Errorf("failed to parse selector: %w", err) } var pvcs corev1.PersistentVolumeClaimList if err := r.Client.List(ctx, &pvcs, client.MatchingLabelsSelector{Selector: selector}); err != nil { - return nil, fmt.Errorf("failed to list PVCs: %w", err) + return fmt.Errorf("failed to list PVCs: %w", err) } - patches := make(map[types.NamespacedName]*unstructured.Unstructured) - for _, pvc := range pvcs.Items { newSize, ok := pvcsToKeep[pvc.Name] if !ok { @@ -109,48 +129,25 @@ func (r *MySQLClusterReconciler) findPVCs(ctx context.Context, cluster *mocov1be supported, err := r.isVolumeExpansionSupported(ctx, &pvc) if err != nil { - return nil, fmt.Errorf("failed to check if volume expansion is supported: %w", err) + return fmt.Errorf("failed to check if volume expansion is supported: %w", err) } if !supported { log.Info("StorageClass used by PVC does not support volume expansion, skipped", "storageClassName", *pvc.Spec.StorageClassName, "pvcName", pvc.Name) continue } - pvcac := corev1ac.PersistentVolumeClaim(pvc.Name, pvc.Namespace). - WithSpec(corev1ac.PersistentVolumeClaimSpec(). - WithResources(corev1ac.ResourceRequirements(). - WithRequests(corev1.ResourceList{corev1.ResourceStorage: *newSize}), - ), - ) - - obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvcac) - if err != nil { - return nil, fmt.Errorf("failed to convert PVC %s/%s to unstructured: %w", pvc.Namespace, pvc.Name, err) - } - - patches[types.NamespacedName{Namespace: pvc.Namespace, Name: pvc.Name}] = &unstructured.Unstructured{Object: obj} - } - - if len(patches) == 0 { - return nil, errors.New("could not find resizable PVCs") - } - - return patches, nil -} - -func (r *MySQLClusterReconciler) resizePVCs(ctx context.Context, patches map[types.NamespacedName]*unstructured.Unstructured) error { - log := crlog.FromContext(ctx) - - for key, patch := range patches { - err := r.Patch(ctx, patch, client.Apply, &client.PatchOptions{ - FieldManager: fieldManager, - Force: pointer.Bool(true), - }) - if err != nil { - return fmt.Errorf("failed to patch PVC %s/%s: %w", key.Namespace, key.Name, err) + switch i := pvc.Spec.Resources.Requests.Storage().Cmp(*newSize); { + case i == 0: // volume size is equal + continue + case i == 1: // current volume size is greater than new size + return fmt.Errorf("failed to resize pvc %q, want size: %s, deployed size: %s: %w", pvc.Name, newSize.String(), pvc.Spec.Resources.Requests.Storage().String(), ErrReduceVolumeSize) + case i == -1: // current volume size is smaller than new size + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = *newSize + + if err := r.Client.Update(ctx, &pvc); err != nil { + return fmt.Errorf("failed to update PVC: %w", err) + } } - - log.Info("Resized PVC", "pvcName", key.Name) } return nil @@ -169,9 +166,9 @@ func (r *MySQLClusterReconciler) deleteStatefulSet(ctx context.Context, sts *app return nil } -func (r *MySQLClusterReconciler) needResizePVC(cluster *mocov1beta2.MySQLCluster, sts *appsv1.StatefulSet) (map[string]corev1.PersistentVolumeClaim, bool) { +func (r *MySQLClusterReconciler) needResizePVC(cluster *mocov1beta2.MySQLCluster, sts *appsv1.StatefulSet) (map[string]corev1.PersistentVolumeClaim, bool, error) { if len(sts.Spec.VolumeClaimTemplates) == 0 { - return nil, false + return nil, false, nil } pvcSet := make(map[string]corev1.PersistentVolumeClaim, len(sts.Spec.VolumeClaimTemplates)) @@ -190,17 +187,22 @@ func (r *MySQLClusterReconciler) needResizePVC(cluster *mocov1beta2.MySQLCluster deployedSize := current.Spec.Resources.Requests.Storage() wantSize := pvc.Spec.Resources.Requests.Storage() - if deployedSize.Equal(wantSize.DeepCopy()) { + switch i := deployedSize.Cmp(wantSize.DeepCopy()); { + case i == 0: // volume size is equal delete(pvcSet, pvc.Name) continue + case i == 1: // volume size is greater + return nil, false, fmt.Errorf("failed to resize pvc %q, want size: %s, deployed size: %s: %w", pvc.Name, wantSize, deployedSize, ErrReduceVolumeSize) + case i == -1: // volume size is smaller + continue } } if len(pvcSet) == 0 { - return nil, false + return nil, false, nil } - return pvcSet, true + return pvcSet, true, nil } func (r *MySQLClusterReconciler) isVolumeExpansionSupported(ctx context.Context, pvc *corev1.PersistentVolumeClaim) (bool, error) { @@ -235,3 +237,46 @@ func (MySQLClusterReconciler) isUpdatingStatefulSet(sts *appsv1.StatefulSet) boo return false } + +func (r *MySQLClusterReconciler) updateResizeCondition(ctx context.Context, req ctrl.Request, status corev1.ConditionStatus, msg string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + now := metav1.Now() + + var cluster mocov1beta2.MySQLCluster + if err := r.Client.Get(ctx, types.NamespacedName{Name: req.Name, Namespace: req.Namespace}, &cluster); err != nil { + return fmt.Errorf("failed to get MySQLCluster %s/%s: %w", req.Namespace, req.Name, err) + } + orig := cluster.DeepCopy() + + newCond := mocov1beta2.MySQLClusterCondition{ + Type: mocov1beta2.ConditionVolumeResized, + Status: status, + Message: msg, + LastTransitionTime: now, + } + + find := false + + for i, cond := range cluster.Status.Conditions { + if cond.Type != mocov1beta2.ConditionVolumeResized { + continue + } + find = true + if cond.Status == status { + newCond.LastTransitionTime = cond.LastTransitionTime + } + cluster.Status.Conditions[i] = newCond + } + + if !find { + cluster.Status.Conditions = append(cluster.Status.Conditions, newCond) + } + + // if nothing has changed, skip updating. + if equality.Semantic.DeepEqual(orig, cluster) { + return nil + } + + return r.Client.Status().Update(ctx, &cluster) + }) +} diff --git a/controllers/resize_pvc_test.go b/controllers/resize_pvc_test.go new file mode 100644 index 000000000..5f9388b78 --- /dev/null +++ b/controllers/resize_pvc_test.go @@ -0,0 +1,246 @@ +package controllers + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/cybozu-go/moco/pkg/constants" + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + mocov1beta2 "github.com/cybozu-go/moco/api/v1beta2" +) + +func TestReconcilePVC(t *testing.T) { + tests := []struct { + name string + cluster *mocov1beta2.MySQLCluster + setupClient func(*testing.T) client.Client + wantSize resource.Quantity + }{ + { + name: "resize succeeded", + cluster: newMySQLClusterWithVolumeSize(resource.MustParse("2Gi")), + setupClient: func(t *testing.T) client.Client { + cluster := newMySQLClusterWithVolumeSize(resource.MustParse("2Gi")) + sts := newStatefulSetWithVolumeSize(resource.MustParse("1Gi")) + return setupMockClient(t, cluster, sts) + }, + wantSize: resource.MustParse("2Gi"), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + r := &MySQLClusterReconciler{Client: tt.setupClient(t)} + + err := r.reconcilePVC(ctx, ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: tt.cluster.Namespace, + Name: tt.cluster.Name, + }}, tt.cluster) + if err != nil { + t.Fatalf("reconcilePVC() error = %v", err) + } + + var pvc corev1.PersistentVolumeClaim + if err := r.Get(ctx, types.NamespacedName{Name: "mysql-data-moco-mysql-cluster-0", Namespace: tt.cluster.Namespace}, &pvc); err != nil { + t.Fatalf("failed to get PVC: %v", err) + } + if !pvc.Spec.Resources.Requests.Storage().Equal(tt.wantSize) { + t.Errorf("unexpected PVC size: got: %s, want: %s", pvc.Spec.Resources.Requests.Storage().String(), tt.wantSize.String()) + } + + var cluster mocov1beta2.MySQLCluster + if err := r.Get(ctx, types.NamespacedName{Name: tt.cluster.Name, Namespace: tt.cluster.Namespace}, &cluster); err != nil { + t.Fatalf("failed to get MySQLCluster: %v", err) + } + if len(cluster.Status.Conditions) == 0 { + t.Fatal("MySQLCluster should have conditions") + } + + found := false + + for _, cond := range cluster.Status.Conditions { + if cond.Type == mocov1beta2.ConditionVolumeResized && + cond.Status == corev1.ConditionTrue { + found = true + break + } + } + + if !found { + t.Error("MySQLCluster should have VolumeResized condition") + } + }) + } +} + +func setupMockClient(t *testing.T, cluster *mocov1beta2.MySQLCluster, sts *appsv1.StatefulSet) client.Client { + t.Helper() + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add scheme: %v", err) + } + if err := mocov1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add scheme: %v", err) + } + + var pvcs []client.Object + + for _, pvc := range sts.Spec.VolumeClaimTemplates { + pvc := pvc + for i := int32(0); i < *sts.Spec.Replicas; i++ { + pvc.Name = fmt.Sprintf("%s-%s-%d", pvc.Name, cluster.PrefixedName(), i) + pvc.Namespace = cluster.Namespace + pvc.Labels = sts.Spec.Selector.MatchLabels + pvc.Spec.StorageClassName = pointer.String("default") + pvcs = append(pvcs, &pvc) + } + } + + storageClass := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Provisioner: "kubernetes.io/no-provisioner", + AllowVolumeExpansion: pointer.Bool(true), + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cluster, sts, storageClass). + WithObjects(pvcs...). + Build() + + return client +} + +func TestNeedResizePVC(t *testing.T) { + tests := []struct { + name string + cluster *mocov1beta2.MySQLCluster + sts *appsv1.StatefulSet + wantResizeTarget map[string]corev1.PersistentVolumeClaim + wantResize bool + wantError error + }{ + { + name: "no resizing", + cluster: newMySQLClusterWithVolumeSize(resource.MustParse("1Gi")), + sts: newStatefulSetWithVolumeSize(resource.MustParse("1Gi")), + wantResize: false, + }, + { + name: "need resizing", + cluster: newMySQLClusterWithVolumeSize(resource.MustParse("2Gi")), + sts: newStatefulSetWithVolumeSize(resource.MustParse("1Gi")), + wantResizeTarget: func() map[string]corev1.PersistentVolumeClaim { + sts := newStatefulSetWithVolumeSize(resource.MustParse("1Gi")) + pvc := sts.Spec.VolumeClaimTemplates[0] + m := make(map[string]corev1.PersistentVolumeClaim) + m[pvc.Name] = pvc + return m + }(), + wantResize: true, + }, + { + name: "reduce volume size error", + cluster: newMySQLClusterWithVolumeSize(resource.MustParse("1Gi")), + sts: newStatefulSetWithVolumeSize(resource.MustParse("2Gi")), + wantResize: false, + wantError: ErrReduceVolumeSize, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + r := &MySQLClusterReconciler{} + resizeTarget, resize, err := r.needResizePVC(tt.cluster, tt.sts) + if err != nil { + if !errors.Is(err, tt.wantError) { + t.Fatalf("want error %v, got %v", tt.wantError, err) + } + } + + if tt.wantResize != resize { + t.Fatalf("want resize %v, got %v", tt.wantResize, resize) + } + + for key, value := range tt.wantResizeTarget { + if diff := cmp.Diff(value, resizeTarget[key]); len(diff) != 0 { + t.Fatalf("want resize target %v, got %v", value, resizeTarget[key]) + } + } + }) + } +} + +func newMySQLClusterWithVolumeSize(size resource.Quantity) *mocov1beta2.MySQLCluster { + return &mocov1beta2.MySQLCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mysql-cluster", + Namespace: "default", + }, + Spec: mocov1beta2.MySQLClusterSpec{ + VolumeClaimTemplates: []mocov1beta2.PersistentVolumeClaim{ + { + ObjectMeta: mocov1beta2.ObjectMeta{Name: "mysql-data"}, + Spec: mocov1beta2.PersistentVolumeClaimSpecApplyConfiguration(*corev1ac.PersistentVolumeClaimSpec(). + WithStorageClassName("default").WithResources(corev1ac.ResourceRequirements(). + WithRequests(corev1.ResourceList{corev1.ResourceStorage: size}), + )), + }, + }, + }, + } +} + +func newStatefulSetWithVolumeSize(size resource.Quantity) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moco-mysql-cluster", + Namespace: "default", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: pointer.Int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + constants.LabelAppName: constants.AppNameMySQL, + constants.LabelAppInstance: "mysql-cluster", + constants.LabelAppCreatedBy: constants.AppCreator, + }, + }, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "mysql-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: pointer.String("default"), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: size}, + }, + }, + }, + }, + }, + } +}