From 559f1d53ef3501a22e830c8283f8a723bb06081e Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Fri, 20 Sep 2024 13:07:00 -0700 Subject: [PATCH] Bucket Notifications - Add bucketNotifications field to noobaa CRD, adapt reconcile. Signed-off-by: Amit Prinz Setter --- deploy/crds/noobaa.io_noobaas.yaml | 36 ++++++ pkg/apis/noobaa/v1alpha1/noobaa_types.go | 20 +++ .../noobaa/v1alpha1/zz_generated.deepcopy.go | 27 ++++ pkg/bundle/deploy.go | 38 +++++- pkg/system/phase1_verifying.go | 35 +++-- pkg/system/phase2_creating.go | 122 ++++++++++++++---- pkg/system/phase4_configuring.go | 27 ++++ pkg/system/reconciler.go | 4 + 8 files changed, 268 insertions(+), 41 deletions(-) diff --git a/deploy/crds/noobaa.io_noobaas.yaml b/deploy/crds/noobaa.io_noobaas.yaml index 6eb1b66abe..639a844541 100644 --- a/deploy/crds/noobaa.io_noobaas.yaml +++ b/deploy/crds/noobaa.io_noobaas.yaml @@ -1028,6 +1028,42 @@ spec: - guaranteed - much more reliable but need to provide a storage class that supports RWX PVs type: string type: object + bucketNotifications: + description: BucketNotifications (optional) controls bucket notification + options + properties: + connections: + description: |- + Connections - A list of secrets' names that are used by the notifications configrations + (in the TopicArn field). + items: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + type: array + enabled: + description: Enabled - whether bucket notifications is enabled + type: boolean + pvc: + description: |- + PVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used + for holding pending notifications files. + For ODF - If not provided, the default CepthFS storage class will be used to create the PVC. + type: string + required: + - enabled + type: object cleanupPolicy: description: CleanupPolicy (optional) Indicates user's policy for deletion diff --git a/pkg/apis/noobaa/v1alpha1/noobaa_types.go b/pkg/apis/noobaa/v1alpha1/noobaa_types.go index 0d7f4ac79d..829621dda7 100644 --- a/pkg/apis/noobaa/v1alpha1/noobaa_types.go +++ b/pkg/apis/noobaa/v1alpha1/noobaa_types.go @@ -227,6 +227,10 @@ type NooBaaSpec struct { // BucketLogging sets the configuration for bucket logging // +optional BucketLogging BucketLoggingSpec `json:"bucketLogging,omitempty"` + + // BucketNotifications (optional) controls bucket notification options + // +optional + BucketNotifications BucketNotificationsSpec `json:"bucketNotifications,omitempty"` } // AutoscalerSpec defines different actoscaling spec such as autoscaler type and prometheus namespace @@ -258,6 +262,22 @@ type BucketLoggingSpec struct { BucketLoggingPVC *string `json:"bucketLoggingPVC,omitempty"` } +//BucketNotificationsSpec controls bucket notification configuration +type BucketNotificationsSpec struct { + // Enabled - whether bucket notifications is enabled + Enabled bool `json:"enabled"` + + //PVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used + //for holding pending notifications files. + //For ODF - If not provided, the default CepthFS storage class will be used to create the PVC. + // +optional + PVC *string `json:"pvc,omitempty"` + + //Connections - A list of secrets' names that are used by the notifications configrations + //(in the TopicArn field). + Connections []corev1.SecretReference `json:"connections,omitempty"` +} + // LoadBalancerSourceSubnetSpec defines the subnets that will be allowed to access the NooBaa services type LoadBalancerSourceSubnetSpec struct { // S3 is a list of subnets that will be allowed to access the Noobaa S3 service diff --git a/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go index ab86a614ce..894981134a 100644 --- a/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/noobaa/v1alpha1/zz_generated.deepcopy.go @@ -460,6 +460,32 @@ func (in *BucketLoggingSpec) DeepCopy() *BucketLoggingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BucketNotificationsSpec) DeepCopyInto(out *BucketNotificationsSpec) { + *out = *in + if in.PVC != nil { + in, out := &in.PVC, &out.PVC + *out = new(string) + **out = **in + } + if in.Connections != nil { + in, out := &in.Connections, &out.Connections + *out = make([]corev1.SecretReference, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketNotificationsSpec. +func (in *BucketNotificationsSpec) DeepCopy() *BucketNotificationsSpec { + if in == nil { + return nil + } + out := new(BucketNotificationsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheNamespacePolicy) DeepCopyInto(out *CacheNamespacePolicy) { *out = *in @@ -1227,6 +1253,7 @@ func (in *NooBaaSpec) DeepCopyInto(out *NooBaaSpec) { in.LoadBalancerSourceSubnets.DeepCopyInto(&out.LoadBalancerSourceSubnets) out.Autoscaler = in.Autoscaler in.BucketLogging.DeepCopyInto(&out.BucketLogging) + in.BucketNotifications.DeepCopyInto(&out.BucketNotifications) return } diff --git a/pkg/bundle/deploy.go b/pkg/bundle/deploy.go index 03b2b68030..5467257d96 100644 --- a/pkg/bundle/deploy.go +++ b/pkg/bundle/deploy.go @@ -1415,7 +1415,7 @@ spec: status: {} ` -const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "3f88c800238f25e5dd26f3f1bf19028571cc646e3aea0f868bfd2ff600ee3ed1" +const Sha256_deploy_crds_noobaa_io_noobaas_yaml = "e862d263d097ed43f774784eaaf9a616967746b67608fadbe4ca71d93b220ab6" const File_deploy_crds_noobaa_io_noobaas_yaml = `--- apiVersion: apiextensions.k8s.io/v1 @@ -2447,6 +2447,42 @@ spec: - guaranteed - much more reliable but need to provide a storage class that supports RWX PVs type: string type: object + bucketNotifications: + description: BucketNotifications (optional) controls bucket notification + options + properties: + connections: + description: |- + Connections - A list of secrets' names that are used by the notifications configrations + (in the TopicArn field). + items: + description: |- + SecretReference represents a Secret Reference. It has enough information to retrieve secret + in any namespace + properties: + name: + description: name is unique within a namespace to reference + a secret resource. + type: string + namespace: + description: namespace defines the space within which the + secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + type: array + enabled: + description: Enabled - whether bucket notifications is enabled + type: boolean + pvc: + description: |- + PVC (optional) specifies the name of the Persistent Volume Claim (PVC) to be used + for holding pending notifications files. + For ODF - If not provided, the default CepthFS storage class will be used to create the PVC. + type: string + required: + - enabled + type: object cleanupPolicy: description: CleanupPolicy (optional) Indicates user's policy for deletion diff --git a/pkg/system/phase1_verifying.go b/pkg/system/phase1_verifying.go index b4011cf97a..453fd3d259 100644 --- a/pkg/system/phase1_verifying.go +++ b/pkg/system/phase1_verifying.go @@ -67,7 +67,13 @@ func (r *Reconciler) ReconcilePhaseVerifying() error { } if r.NooBaa.Spec.BucketLogging.LoggingType == nbv1.BucketLoggingTypeGuaranteed { - if err := r.checkBucketLoggingPVC(); err != nil { + if err := r.checkPersistentLoggingPVC(r.NooBaa.Spec.BucketLogging.BucketLoggingPVC, r.BucketLoggingPVC, "InvalidBucketLoggingConfiguration"); err != nil { + return err + } + } + + if r.NooBaa.Spec.BucketNotifications.Enabled { + if err := r.checkPersistentLoggingPVC(r.NooBaa.Spec.BucketNotifications.PVC, r.BucketNotificationsPVC, "InvalidBucketNotificationConfiguration"); err != nil { return err } } @@ -266,10 +272,11 @@ func (r *Reconciler) checkExternalPg(postgresDbURL string) error { } // checkBucketLoggingPVC validates the configuration of bucket logging pvc -func (r *Reconciler) checkBucketLoggingPVC() error { - // Rejecting if 'BucketLoggingPVC' is not provided for 'guaranteed' logging and - // also the operator is not running in the ODF environment. - if r.NooBaa.Spec.BucketLogging.BucketLoggingPVC == nil { +func (r *Reconciler) checkPersistentLoggingPVC( + pvcName *string, + pvc *corev1.PersistentVolumeClaim, + errorName string) error { + if pvc == nil { sc := &storagev1.StorageClass{ TypeMeta: metav1.TypeMeta{Kind: "StorageClass"}, ObjectMeta: metav1.ObjectMeta{Name: "ocs-storagecluster-cephfs"}, @@ -278,29 +285,29 @@ func (r *Reconciler) checkBucketLoggingPVC() error { if util.KubeCheck(sc) { return nil } - return util.NewPersistentError("InvalidBucketLoggingConfiguration", + return util.NewPersistentError(errorName, "'Guaranteed' BucketLogging requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'BucketLoggingPVC' to ensure guaranteed logging") } // Check if pvc exists in the cluster - BucketLoggingPVC := &corev1.PersistentVolumeClaim{ + PersistentLoggingPVC := &corev1.PersistentVolumeClaim{ TypeMeta: metav1.TypeMeta{Kind: "PersistenVolumeClaim"}, ObjectMeta: metav1.ObjectMeta{ - Name: *r.NooBaa.Spec.BucketLogging.BucketLoggingPVC, + Name: *pvcName, Namespace: r.Request.Namespace, }, } - if !util.KubeCheck(BucketLoggingPVC) { - return util.NewPersistentError("InvalidBucketLoggingConfiguration", - fmt.Sprintf("The specified BucketLoggingPVC '%s' was not found", BucketLoggingPVC.Name)) + if !util.KubeCheck(PersistentLoggingPVC) { + return util.NewPersistentError(errorName, + fmt.Sprintf("The specified persistent logging pvc '%s' was not found", *pvcName)) } // Check if pvc supports RWX access mode - for _, accessMode := range BucketLoggingPVC.Spec.AccessModes { + for _, accessMode := range PersistentLoggingPVC.Spec.AccessModes { if accessMode == corev1.ReadWriteMany { return nil } } - return util.NewPersistentError("InvalidBucketLoggingConfiguration", - fmt.Sprintf("The specified BucketLoggingPVC '%s' does not support RWX access mode", BucketLoggingPVC.Name)) + return util.NewPersistentError(errorName, + fmt.Sprintf("The specified persistent logging pvc '%s' does not support RWX access mode", *pvcName)) } diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go index a2680069cd..5d33a96a80 100644 --- a/pkg/system/phase2_creating.go +++ b/pkg/system/phase2_creating.go @@ -132,10 +132,28 @@ func (r *Reconciler) ReconcilePhaseCreatingForMainClusters() error { } // create bucket logging pvc if not provided by user for 'Guaranteed' logging in ODF env if r.NooBaa.Spec.BucketLogging.LoggingType == nbv1.BucketLoggingTypeGuaranteed { - if err := r.ReconcileODFBucketLoggingPVC(); err != nil { + if err := r.ReconcileODFPersistentLoggingPVC( + "BucketLoggingPVC", + "InvalidBucketLoggingConfiguration", + "'Guaranteed' BucketLogging requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'BucketLoggingPVC' to ensure guaranteed logging", + r.NooBaa.Spec.BucketLogging.BucketLoggingPVC, + r.BucketLoggingPVC); err != nil { return err } } + + // create notification log pvc if bucket notifications is enabled and pvc was not set explicitly + if r.NooBaa.Spec.BucketNotifications.Enabled { + if err := r.ReconcileODFPersistentLoggingPVC( + "bucketNotifications.pvc", + "InvalidBucketNotificationConfiguration", + "Bucket notifications requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'bucketNotifications.pvc'.", + r.NooBaa.Spec.BucketNotifications.PVC, + r.BucketNotificationsPVC); err != nil { + return err + } + } + util.KubeCreateOptional(util.KubeObject(bundle.File_deploy_scc_core_yaml).(*secv1.SecurityContextConstraints)) if err := r.ReconcileObject(r.ServiceMgmt, r.SetDesiredServiceMgmt); err != nil { return err @@ -450,6 +468,15 @@ func (r *Reconciler) setDesiredCoreEnv(c *corev1.Container) { } } } + + if r.NooBaa.Spec.BucketNotifications.Enabled {//TODO - has pvc + envVar := corev1.EnvVar{ + Name: "NOTIFICATION_LOG_DIR", + Value: "/var/logs/notifications", + } + util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar}); + } + } // SetDesiredCoreApp updates the CoreApp as desired for reconciling @@ -519,6 +546,44 @@ func (r *Reconciler) SetDesiredCoreApp() error { }} util.MergeVolumeMountList(&c.VolumeMounts, &bucketLogVolumeMounts) } + + if r.NooBaa.Spec.BucketNotifications.Enabled { + notificationVolumeMounts := []corev1.VolumeMount{{ + Name: "notif-vol", + MountPath: "/var/logs/notifications", + }} + util.MergeVolumeMountList(&c.VolumeMounts, ¬ificationVolumeMounts) + + notificationVolumes := []corev1.Volume {{ + Name: "notif-vol", + VolumeSource: corev1.VolumeSource { + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource { + ClaimName: r.BucketNotificationsPVC.Name, + }, + }, + }} + util.MergeVolumeList(&podSpec.Volumes, ¬ificationVolumes) + + for _, notifSecret := range r.NooBaa.Spec.BucketNotifications.Connections { + secretVolumeMounts := []corev1.VolumeMount{{ + Name: notifSecret.Name, + MountPath: "/etc/notif_connect/" + notifSecret.Name, + ReadOnly: true, + }} + util.MergeVolumeMountList(&c.VolumeMounts, &secretVolumeMounts) + + secretVolumes := []corev1.Volume{{ + Name: notifSecret.Name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: notifSecret.Name, + }, + }, + }} + util.MergeVolumeList(&podSpec.Volumes, &secretVolumes) + } + } + case "noobaa-log-processor": if c.Image != r.NooBaa.Status.ActualImage { coreImageChanged = true @@ -624,6 +689,7 @@ func (r *Reconciler) SetDesiredCoreApp() error { }} util.MergeVolumeList(&podSpec.Volumes, &bucketLogVolumes) } + return nil } @@ -1163,56 +1229,60 @@ func (r *Reconciler) ReconcileDBConfigMap(cm *corev1.ConfigMap, desiredFunc func return r.isObjectUpdated(result), nil } -// ReconcileODFBucketLoggingPVC ensures the bucket logging PVC is properly configured for 'guaranteed' logging -func (r *Reconciler) ReconcileODFBucketLoggingPVC() error { - log := r.Logger.WithField("func", "ReconcileBucketLoggingPVC") +func (r *Reconciler) ReconcileODFPersistentLoggingPVC( + fieldName string, + errorName string, + errorText string, + pvcName *string, + pvc *corev1.PersistentVolumeClaim) error { + + log := r.Logger.WithField("func", "ReconcileODFPersistentLoggingPVC") - // Return if bucket logging PVC already exist - if r.NooBaa.Spec.BucketLogging.BucketLoggingPVC != nil { - r.BucketLoggingPVC.Name = *r.NooBaa.Spec.BucketLogging.BucketLoggingPVC - log.Infof("BucketLoggingPVC %s already exists and supports RWX access mode. Skipping ReconcileODFBucketLoggingPVC.", r.BucketLoggingPVC.Name) + // Return if persistent logging PVC already exists + if pvc != nil { + pvc.Name = *pvcName; + log.Infof("PersistentLoggingPVC %s already exists and supports RWX access mode. Skipping ReconcileODFPersistentLoggingPVC.", *pvcName) return nil } - util.KubeCheck(r.BucketLoggingPVC) - if r.BucketLoggingPVC.UID != "" { - log.Infof("BucketLoggingPVC %s already exists. Skipping creation.", r.BucketLoggingPVC.Name) + util.KubeCheck(pvc) + if pvc.UID != "" { + log.Infof("Persistent logging PVC %s already exists. Skipping creation.", *pvcName ) return nil } - if err := r.prepareODFBucketLoggingPVC(); err != nil { - return err + if !r.preparePersistentLoggingPVC(pvc, fieldName) { + return util.NewPersistentError(errorName, errorText) } - r.Own(r.BucketLoggingPVC) + r.Own(pvc); - log.Infof("BucketLoggingPVC %s does not exist. Creating...", r.BucketLoggingPVC.Name) - err := r.Client.Create(r.Ctx, r.BucketLoggingPVC) + log.Infof("Persistent loggin PVC %s does not exist. Creating...", *pvcName) + err := r.Client.Create(r.Ctx, pvc) if err != nil { return err } return nil + } -// prepareODFBucketLoggingPVC configures the bucket logging pvc -func (r *Reconciler) prepareODFBucketLoggingPVC() error { - r.BucketLoggingPVC.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} +//prepare persistent logging pvc +func (r *Reconciler) preparePersistentLoggingPVC(pvc *corev1.PersistentVolumeClaim, fieldName string) bool { + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} sc := &storagev1.StorageClass{ TypeMeta: metav1.TypeMeta{Kind: "StorageClass"}, ObjectMeta: metav1.ObjectMeta{Name: "ocs-storagecluster-cephfs"}, } - // fallback to cephfs storageclass to create bucket logging pvc if running on ODF + // fallback to cephfs storageclass to create persistent logging pvc if running on ODF if util.KubeCheck(sc) { - r.Logger.Infof("BucketLoggingPVC not provided, defaulting to 'cephfs' storage class %s to create bucket logging pvc", sc.Name) - r.BucketLoggingPVC.Spec.StorageClassName = &sc.Name + r.Logger.Infof("%s not provided, defaulting to 'cephfs' storage class %s to create persistent logging pvc", fieldName, sc.Name) + pvc.Spec.StorageClassName = &sc.Name + return true; } else { - return util.NewPersistentError("InvalidBucketLoggingConfiguration", - "'Guaranteed' BucketLogging requires a Persistent Volume Claim (PVC) with ReadWriteMany (RWX) access mode. Please specify the 'BucketLoggingPVC' to ensure guaranteed logging") + return false; } - - return nil } // SetDesiredPostgresDBConf fill desired postgres db config map diff --git a/pkg/system/phase4_configuring.go b/pkg/system/phase4_configuring.go index e6f8398693..87062c6c9c 100644 --- a/pkg/system/phase4_configuring.go +++ b/pkg/system/phase4_configuring.go @@ -415,6 +415,15 @@ func (r *Reconciler) SetDesiredDeploymentEndpoint() error { } } + if r.NooBaa.Spec.BucketNotifications.Enabled { + envVar := corev1.EnvVar{ + Name: "NOTIFICATION_LOG_DIR", + Value: "/var/logs/notifications", + } + + util.MergeEnvArrays(&c.Env, &[]corev1.EnvVar{envVar}); + } + c.SecurityContext = &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ Add: []corev1.Capability{"SETUID", "SETGID"}, @@ -533,6 +542,24 @@ func (r *Reconciler) setDesiredEndpointMounts(podSpec *corev1.PodSpec, container util.MergeVolumeMountList(&container.VolumeMounts, &bucketLogVolumeMounts) } + if r.NooBaa.Spec.BucketNotifications.Enabled { + notificationVolumes := []corev1.Volume{{ + Name: "notif-vol", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: r.BucketNotificationsPVC.Name, + }, + }, + }} + util.MergeVolumeList(&podSpec.Volumes, ¬ificationVolumes) + + notificationVolumeMounts := []corev1.VolumeMount{{ + Name: "notif-vol", + MountPath: "/var/logs/notifications", + }} + util.MergeVolumeMountList(&container.VolumeMounts, ¬ificationVolumeMounts) + } + r.setDesiredRootMasterKeyMounts(podSpec, container) for _, nsStore := range namespaceStoreList.Items { diff --git a/pkg/system/reconciler.go b/pkg/system/reconciler.go index 159421d804..9992e8520f 100644 --- a/pkg/system/reconciler.go +++ b/pkg/system/reconciler.go @@ -122,6 +122,7 @@ type Reconciler struct { AdapterHPA *autoscalingv2.HorizontalPodAutoscaler ExternalPgSecret *corev1.Secret ExternalPgSSLSecret *corev1.Secret + BucketNotificationsPVC *corev1.PersistentVolumeClaim } // NewReconciler initializes a reconciler to be used for loading or reconciling a noobaa system @@ -171,6 +172,7 @@ func NewReconciler( DefaultNsfsPvc: util.KubeObject(bundle.File_deploy_internal_nsfs_pvc_cr_yaml).(*corev1.PersistentVolumeClaim), OBCStorageClass: util.KubeObject(bundle.File_deploy_obc_storage_class_yaml).(*storagev1.StorageClass), BucketLoggingPVC: util.KubeObject(bundle.File_deploy_internal_pvc_agent_yaml).(*corev1.PersistentVolumeClaim), + BucketNotificationsPVC: util.KubeObject(bundle.File_deploy_internal_pvc_agent_yaml).(*corev1.PersistentVolumeClaim), PrometheusRule: util.KubeObject(bundle.File_deploy_internal_prometheus_rules_yaml).(*monitoringv1.PrometheusRule), ServiceMonitorMgmt: util.KubeObject(bundle.File_deploy_internal_servicemonitor_mgmt_yaml).(*monitoringv1.ServiceMonitor), ServiceMonitorS3: util.KubeObject(bundle.File_deploy_internal_servicemonitor_s3_yaml).(*monitoringv1.ServiceMonitor), @@ -230,6 +232,7 @@ func NewReconciler( r.KedaScaled.Namespace = r.Request.Namespace r.AdapterHPA.Namespace = r.Request.Namespace r.BucketLoggingPVC.Namespace = r.Request.Namespace + r.BucketNotificationsPVC.Namespace = r.Request.Namespace // Set Names r.NooBaa.Name = r.Request.Name @@ -274,6 +277,7 @@ func NewReconciler( r.KedaScaled.Name = r.Request.Name r.AdapterHPA.Name = r.Request.Name + "-hpav2" r.BucketLoggingPVC.Name = r.Request.Name + "-bucket-logging-pvc" + r.BucketNotificationsPVC.Name = r.Request.Name + "-bucket-notifications-pvc" // Set the target service for routes. r.RouteMgmt.Spec.To.Name = r.ServiceMgmt.Name