Skip to content

Commit

Permalink
Merge pull request #266 from raaizik/RHSTOR-5759-VolGSnapClass
Browse files Browse the repository at this point in the history
Labels `VolumeSnapshotClass`
  • Loading branch information
openshift-merge-bot[bot] authored Nov 11, 2024
2 parents 64a0317 + 4a2bcb5 commit 13fdfb2
Show file tree
Hide file tree
Showing 12 changed files with 109 additions and 119 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ metadata:
categories: Storage
console.openshift.io/plugins: '["odf-client-console"]'
containerImage: quay.io/ocs-dev/ocs-client-operator:latest
createdAt: "2024-10-29T07:06:57Z"
createdAt: "2024-11-10T14:44:36Z"
description: OpenShift Data Foundation client operator enables consumption of
storage services from a remote centralized OpenShift Data Foundation provider
cluster.
Expand Down
2 changes: 1 addition & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis"
configv1 "github.com/openshift/api/config/v1"
consolev1 "github.com/openshift/api/console/v1"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ exclude (
require (
github.com/ceph/ceph-csi-operator/api v0.0.0-20240812072523-4d50cf3a32a0
github.com/go-logr/logr v1.4.2
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.34.1
github.com/openshift/api v0.0.0-20240828125535-01b3675ba7b3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 h1:dQEHhTfi+bSIOSViQrKY9PqJvZenD6tFz+3lPzux58o=
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1/go.mod h1:my+EVjOJLeQ9lUR9uVkxRvNNkhO2saSGIgzV8GZT9HY=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0 h1:qS4r4ljINLWKJ9m9Ge3Q3sGZ/eIoDVDT2RhAdQFHb1k=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0/go.mod h1:oGXx2XTEzs9ikW2V6IC1dD8trgjRsS/Mvc2JRiC618Y=
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
Expand Down
132 changes: 27 additions & 105 deletions internal/controller/storageclaim_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"encoding/json"
"fmt"
"reflect"
"slices"
"strings"
"time"
Expand All @@ -31,7 +30,8 @@ import (

csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
"github.com/go-logr/logr"
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"

snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
providerclient "github.com/red-hat-storage/ocs-operator/services/provider/api/v4/client"
corev1 "k8s.io/api/core/v1"
Expand All @@ -40,7 +40,6 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/cache"
Expand All @@ -57,9 +56,8 @@ const (
storageClaimAnnotation = "ocs.openshift.io/storageclaim"
keyRotationAnnotation = "keyrotation.csiaddons.openshift.io/schedule"

pvClusterIDIndexName = "index:persistentVolumeClusterID"
vscClusterIDIndexName = "index:volumeSnapshotContentCSIDriver"

pvClusterIDIndexName = "index:persistentVolumeClusterID"
vscClusterIDIndexName = "index:volumeSnapshotContentCSIDriver"
drClusterConfigCRDName = "drclusterconfigs.ramendr.openshift.io"
)

Expand Down Expand Up @@ -117,6 +115,7 @@ func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
For(&v1alpha1.StorageClaim{}, builder.WithPredicates(generationChangePredicate)).
Owns(&storagev1.StorageClass{}).
Owns(&snapapi.VolumeSnapshotClass{}).
Owns(&csiopv1a1.ClientProfile{}, builder.WithPredicates(generationChangePredicate)).
Watches(
&extv1.CustomResourceDefinition{},
&handler.EnqueueRequestForObject{},
Expand All @@ -125,8 +124,7 @@ func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
utils.CrdCreateAndDeletePredicate(&r.log, drClusterConfigCRDName, r.AvailableCrds[drClusterConfigCRDName]),
),
builder.OnlyMetadata,
).
Owns(&csiopv1a1.ClientProfile{}, builder.WithPredicates(generationChangePredicate))
)

if r.AvailableCrds[drClusterConfigCRDName] {
bldr = bldr.Owns(&ramenv1alpha1.DRClusterConfig{}, builder.WithPredicates(generationChangePredicate))
Expand Down Expand Up @@ -387,12 +385,16 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) {
data["clusterID"] = r.storageClaimHash

if resource.Name == "cephfs" {
storageClass = r.getCephFSStorageClass(data)
storageClass = r.getCephFSStorageClass()
} else if resource.Name == "ceph-rbd" {
storageClass = r.getCephRBDStorageClass(data)
storageClass = r.getCephRBDStorageClass()
}
utils.AddAnnotation(storageClass, storageClaimAnnotation, r.storageClaim.Name)
err = r.createOrReplaceStorageClass(storageClass)
err = utils.CreateOrReplace(r.ctx, r.Client, storageClass, func() error {
utils.AddLabels(storageClass, resource.Labels)
utils.AddAnnotation(storageClass, storageClaimAnnotation, r.storageClaim.Name)
storageClass.Parameters = data
return nil
})
if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to create or update StorageClass: %s", err)
}
Expand All @@ -409,12 +411,17 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) {
// hash as the clusterID
data["clusterID"] = r.storageClaimHash
if resource.Name == "cephfs" {
volumeSnapshotClass = r.getCephFSVolumeSnapshotClass(data)
volumeSnapshotClass = r.getCephFSVolumeSnapshotClass()
} else if resource.Name == "ceph-rbd" {
volumeSnapshotClass = r.getCephRBDVolumeSnapshotClass(data)
volumeSnapshotClass = r.getCephRBDVolumeSnapshotClass()
}
utils.AddAnnotation(volumeSnapshotClass, storageClaimAnnotation, r.storageClaim.Name)
if err := r.createOrReplaceVolumeSnapshotClass(volumeSnapshotClass); err != nil {
err = utils.CreateOrReplace(r.ctx, r.Client, volumeSnapshotClass, func() error {
utils.AddLabels(volumeSnapshotClass, resource.Labels)
utils.AddAnnotation(volumeSnapshotClass, storageClaimAnnotation, r.storageClaim.Name)
volumeSnapshotClass.Parameters = data
return nil
})
if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to create or update VolumeSnapshotClass: %s", err)
}
case "ClientProfile":
Expand Down Expand Up @@ -482,7 +489,7 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) {
return reconcile.Result{}, nil
}

func (r *StorageClaimReconciler) getCephFSStorageClass(data map[string]string) *storagev1.StorageClass {
func (r *StorageClaimReconciler) getCephFSStorageClass() *storagev1.StorageClass {
pvReclaimPolicy := corev1.PersistentVolumeReclaimDelete
allowVolumeExpansion := true
storageClass := &storagev1.StorageClass{
Expand All @@ -495,12 +502,11 @@ func (r *StorageClaimReconciler) getCephFSStorageClass(data map[string]string) *
ReclaimPolicy: &pvReclaimPolicy,
AllowVolumeExpansion: &allowVolumeExpansion,
Provisioner: templates.CephFsDriverName,
Parameters: data,
}
return storageClass
}

func (r *StorageClaimReconciler) getCephRBDStorageClass(data map[string]string) *storagev1.StorageClass {
func (r *StorageClaimReconciler) getCephRBDStorageClass() *storagev1.StorageClass {
pvReclaimPolicy := corev1.PersistentVolumeReclaimDelete
allowVolumeExpansion := true
storageClass := &storagev1.StorageClass{
Expand All @@ -514,7 +520,6 @@ func (r *StorageClaimReconciler) getCephRBDStorageClass(data map[string]string)
ReclaimPolicy: &pvReclaimPolicy,
AllowVolumeExpansion: &allowVolumeExpansion,
Provisioner: templates.RBDDriverName,
Parameters: data,
}

if r.storageClaim.Spec.EncryptionMethod != "" {
Expand All @@ -523,68 +528,28 @@ func (r *StorageClaimReconciler) getCephRBDStorageClass(data map[string]string)
return storageClass
}

func (r *StorageClaimReconciler) getCephFSVolumeSnapshotClass(data map[string]string) *snapapi.VolumeSnapshotClass {
func (r *StorageClaimReconciler) getCephFSVolumeSnapshotClass() *snapapi.VolumeSnapshotClass {
volumesnapshotclass := &snapapi.VolumeSnapshotClass{
ObjectMeta: metav1.ObjectMeta{
Name: r.storageClaim.Name,
},
Driver: templates.CephFsDriverName,
DeletionPolicy: snapapi.VolumeSnapshotContentDelete,
Parameters: data,
}
return volumesnapshotclass
}

func (r *StorageClaimReconciler) getCephRBDVolumeSnapshotClass(data map[string]string) *snapapi.VolumeSnapshotClass {
func (r *StorageClaimReconciler) getCephRBDVolumeSnapshotClass() *snapapi.VolumeSnapshotClass {
volumesnapshotclass := &snapapi.VolumeSnapshotClass{
ObjectMeta: metav1.ObjectMeta{
Name: r.storageClaim.Name,
},
Driver: templates.RBDDriverName,
DeletionPolicy: snapapi.VolumeSnapshotContentDelete,
Parameters: data,
}
return volumesnapshotclass
}

func (r *StorageClaimReconciler) createOrReplaceStorageClass(storageClass *storagev1.StorageClass) error {
existing := &storagev1.StorageClass{}
existing.Name = r.storageClaim.Name

if err := r.own(storageClass); err != nil {
return fmt.Errorf("failed to own storageclass: %v", err)
}

if err := r.get(existing); err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("failed to get StorageClass: %v", err)
}

// If present then compare the existing StorageClass with the received StorageClass, and only proceed if they differ.
if reflect.DeepEqual(existing.Parameters, storageClass.Parameters) {
return nil
}

// StorageClass already exists, but parameters have changed. Delete the existing StorageClass and create a new one.
if existing.UID != "" {

// Since we have to update the existing StorageClass, so we will delete the existing StorageClass and create a new one.
r.log.Info("StorageClass needs to be updated, deleting it.", "StorageClass", klog.KRef(storageClass.Namespace, existing.Name))

// Delete the StorageClass.
err := r.delete(existing)
if err != nil {
r.log.Error(err, "Failed to delete StorageClass.", "StorageClass", klog.KRef(storageClass.Namespace, existing.Name))
return fmt.Errorf("failed to delete StorageClass: %v", err)
}
}
r.log.Info("Creating StorageClass.", "StorageClass", klog.KRef(storageClass.Namespace, existing.Name))
err := r.Client.Create(r.ctx, storageClass)
if err != nil {
return fmt.Errorf("failed to create StorageClass: %v", err)
}
return nil
}

func (r *StorageClaimReconciler) get(obj client.Object) error {
key := client.ObjectKeyFromObject(obj)
return r.Client.Get(r.ctx, key, obj)
Expand All @@ -598,53 +563,10 @@ func (r *StorageClaimReconciler) list(obj client.ObjectList, listOptions ...clie
return r.Client.List(r.ctx, obj, listOptions...)
}

func (r *StorageClaimReconciler) delete(obj client.Object) error {
if err := r.Client.Delete(r.ctx, obj); err != nil && !errors.IsNotFound(err) {
return err
}
return nil
}

func (r *StorageClaimReconciler) own(resource metav1.Object) error {
return controllerutil.SetControllerReference(r.storageClaim, resource, r.Scheme)
}

func (r *StorageClaimReconciler) createOrReplaceVolumeSnapshotClass(volumeSnapshotClass *snapapi.VolumeSnapshotClass) error {
existing := &snapapi.VolumeSnapshotClass{}
existing.Name = r.storageClaim.Name

if err := r.own(volumeSnapshotClass); err != nil {
return fmt.Errorf("failed to own volumesnapshotclass: %v", err)
}

if err := r.get(existing); err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("failed to get VolumeSnapshotClass: %v", err)
}

// If present then compare the existing VolumeSnapshotClass parameters with
// the received VolumeSnapshotClass parameters, and only proceed if they differ.
if reflect.DeepEqual(existing.Parameters, volumeSnapshotClass.Parameters) {
return nil
}

// VolumeSnapshotClass already exists, but parameters have changed. Delete the existing VolumeSnapshotClass and create a new one.
if existing.UID != "" {
// Since we have to update the existing VolumeSnapshotClass, so we will delete the existing VolumeSnapshotClass and create a new one.
r.log.Info("VolumeSnapshotClass needs to be updated, deleting it.", "Name", existing.Name)

// Delete the VolumeSnapshotClass.
if err := r.delete(existing); err != nil {
r.log.Error(err, "Failed to delete VolumeSnapshotClass.", "Name", existing.Name)
return fmt.Errorf("failed to delete VolumeSnapshotClass: %v", err)
}
}
r.log.Info("Creating VolumeSnapshotClass.", "Name", existing.Name)
if err := r.Client.Create(r.ctx, volumeSnapshotClass); err != nil {
return fmt.Errorf("failed to create VolumeSnapshotClass: %v", err)
}
return nil
}

func (r *StorageClaimReconciler) hasPersistentVolumes() (bool, error) {
pvList := &corev1.PersistentVolumeList{}
if err := r.list(pvList, client.MatchingFields{pvClusterIDIndexName: r.storageClaimHash}, client.Limit(1)); err != nil {
Expand Down
53 changes: 53 additions & 0 deletions pkg/utils/k8sutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,18 @@ limitations under the License.
package utils

import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"maps"
"os"
"reflect"

"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

// OperatorNamespaceEnvVar is the constant for env variable OPERATOR_NAMESPACE
Expand Down Expand Up @@ -113,3 +118,51 @@ func GetMD5Hash(text string) string {
hash := md5.Sum([]byte(text))
return hex.EncodeToString(hash[:])
}

func CreateOrReplace(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) error {
key := client.ObjectKeyFromObject(obj)
if err := c.Get(ctx, key, obj); err != nil {
if !errors.IsNotFound(err) {
return err
}
if err := mutate(f, key, obj); err != nil {
return err
}
if err := c.Create(ctx, obj); err != nil {
return err
}
return nil
}

existing := obj.DeepCopyObject()
if err := mutate(f, key, obj); err != nil {
return err
}

if reflect.DeepEqual(existing, obj) {
return nil
}

if err := c.Delete(ctx, obj); err != nil {
return err
}

// k8s doesn't allow us to create objects when resourceVersion is set, as we are DeepCopying the
// object, the resource version also gets copied, hence we need to set it to empty before creating it
obj.SetResourceVersion("")
if err := c.Create(ctx, obj); err != nil {
return err
}
return nil
}

// mutate wraps a MutateFn and applies validation to its result.
func mutate(f controllerutil.MutateFn, key client.ObjectKey, obj client.Object) error {
if err := f(); err != nil {
return err
}
if newKey := client.ObjectKeyFromObject(obj); key != newKey {
return fmt.Errorf("MutateFn cannot mutate object name and/or object namespace")
}
return nil
}
Loading

0 comments on commit 13fdfb2

Please sign in to comment.