From 99af40d9feb82a8acf5a6d548ee55aa6676ada63 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 28 Oct 2021 17:42:15 -0400 Subject: [PATCH 01/11] First pass at scheduled backups. --- api/v1beta1/solrbackup_types.go | 48 +++++ api/v1beta1/solrcloud_types.go | 1 - api/v1beta1/zz_generated.deepcopy.go | 82 ++++++-- .../bases/solr.apache.org_solrbackups.yaml | 104 ++++++++++ controllers/common.go | 3 + controllers/solrbackup_controller.go | 177 +++++++++++------- .../solrprometheusexporter_controller.go | 10 +- controllers/util/backup_util.go | 35 +++- controllers/util/solr_update_util.go | 2 +- helm/solr-operator/crds/crds.yaml | 104 ++++++++++ main.go | 3 +- 11 files changed, 465 insertions(+), 104 deletions(-) diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index 895f9ee1..1a7b5059 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -51,6 +51,13 @@ type SolrBackupSpec struct { // +optional Location string `json:"location,omitempty"` + // Set this backup to be taken recurrently, with options for scheduling and storage. + // + // NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API. + // + // +optional + Recurrence *BackupRecurrence `json:"recurrence,omitempty"` + // Persistence is the specification on how to persist the backup data. // +optional Persistence *PersistenceSource `json:"persistence,omitempty"` @@ -64,6 +71,29 @@ func (spec *SolrBackupSpec) withDefaults(backupName string) (changed bool) { return changed } +// BackupRecurrence defines the recurrence of the incremental backup +type BackupRecurrence struct { + // Perform a backup on the given schedule, in CRON format. + // + // Multiple CRON syntaxes are supported + // - Standard CRON (e.g. "CRON_TZ=Asia/Seoul 0 6 * * ?") + // - Predefined Schedules (e.g. "@yearly", "@weekly", "@daily", etc.) + // - Intervals (e.g. "@every 10h30m") + // + // For more information please check this reference: + // https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format + Schedule string `json:"schedule"` + + // Define the number of backup points to save for this backup at any given time. + // The oldest backups will be deleted if too many exist when a backup is taken. + // If not provided, this defaults to 10. + // + // +kubebuilder:default:=10 + // +kubebuilder:validation:Minimum:=1 + // +optional + MaxSaved int `json:"maxSaved,omitempty"` +} + // PersistenceSource defines the location and method of persisting the backup data. // Exactly one member must be specified. type PersistenceSource struct { @@ -198,9 +228,27 @@ func (spec *VolumePersistenceSource) withDefaults(backupName string) (changed bo // SolrBackupStatus defines the observed state of SolrBackup type SolrBackupStatus struct { + // The current Backup Status, which all fields are added to this struct + Current IndividualSolrBackupStatus `json:",inline"` + + // The scheduled time for the next backup to occur + // +optional + NextScheduledTime *metav1.Time `json:"nextScheduledTime,omitempty"` + + // The status history of recurring backups + // +optional + History []IndividualSolrBackupStatus `json:"history,omitempty"` +} + +// IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup +type IndividualSolrBackupStatus struct { // Version of the Solr being backed up SolrVersion string `json:"solrVersion"` + // The time that this backup was initiated + // +optional + StartTime metav1.Time `json:"startTimestamp,omitempty"` + // The status of each collection's backup progress // +optional CollectionBackupStatuses []CollectionBackupStatus `json:"collectionBackupStatuses,omitempty"` diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go index fb839517..5bfed400 100644 --- a/api/v1beta1/solrcloud_types.go +++ b/api/v1beta1/solrcloud_types.go @@ -35,7 +35,6 @@ const ( DefaultSolrReplicas = int32(3) DefaultSolrRepo = "library/solr" DefaultSolrVersion = "8.9" - DefaultSolrStorage = "5Gi" DefaultSolrJavaMem = "-Xms1g -Xmx2g" DefaultSolrOpts = "" DefaultSolrLogLevel = "INFO" diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 01ef4385..7b75e611 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -76,6 +76,21 @@ func (in *BackupPersistenceStatus) DeepCopy() *BackupPersistenceStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupRecurrence) DeepCopyInto(out *BackupRecurrence) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRecurrence. +func (in *BackupRecurrence) DeepCopy() *BackupRecurrence { + if in == nil { + return nil + } + out := new(BackupRecurrence) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CollectionBackupStatus) DeepCopyInto(out *CollectionBackupStatus) { *out = *in @@ -298,6 +313,43 @@ func (in *GcsRepository) DeepCopy() *GcsRepository { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IndividualSolrBackupStatus) DeepCopyInto(out *IndividualSolrBackupStatus) { + *out = *in + in.StartTime.DeepCopyInto(&out.StartTime) + if in.CollectionBackupStatuses != nil { + in, out := &in.CollectionBackupStatuses, &out.CollectionBackupStatuses + *out = make([]CollectionBackupStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PersistenceStatus != nil { + in, out := &in.PersistenceStatus, &out.PersistenceStatus + *out = new(BackupPersistenceStatus) + (*in).DeepCopyInto(*out) + } + if in.FinishTime != nil { + in, out := &in.FinishTime, &out.FinishTime + *out = (*in).DeepCopy() + } + if in.Successful != nil { + in, out := &in.Successful, &out.Successful + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndividualSolrBackupStatus. +func (in *IndividualSolrBackupStatus) DeepCopy() *IndividualSolrBackupStatus { + if in == nil { + return nil + } + out := new(IndividualSolrBackupStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressOptions) DeepCopyInto(out *IngressOptions) { *out = *in @@ -798,6 +850,11 @@ func (in *SolrBackupSpec) DeepCopyInto(out *SolrBackupSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Recurrence != nil { + in, out := &in.Recurrence, &out.Recurrence + *out = new(BackupRecurrence) + **out = **in + } if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(PersistenceSource) @@ -818,27 +875,18 @@ func (in *SolrBackupSpec) DeepCopy() *SolrBackupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SolrBackupStatus) DeepCopyInto(out *SolrBackupStatus) { *out = *in - if in.CollectionBackupStatuses != nil { - in, out := &in.CollectionBackupStatuses, &out.CollectionBackupStatuses - *out = make([]CollectionBackupStatus, len(*in)) + in.Current.DeepCopyInto(&out.Current) + if in.NextScheduledTime != nil { + in, out := &in.NextScheduledTime, &out.NextScheduledTime + *out = (*in).DeepCopy() + } + if in.History != nil { + in, out := &in.History, &out.History + *out = make([]IndividualSolrBackupStatus, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.PersistenceStatus != nil { - in, out := &in.PersistenceStatus, &out.PersistenceStatus - *out = new(BackupPersistenceStatus) - (*in).DeepCopyInto(*out) - } - if in.FinishTime != nil { - in, out := &in.FinishTime, &out.FinishTime - *out = (*in).DeepCopy() - } - if in.Successful != nil { - in, out := &in.Successful, &out.Successful - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SolrBackupStatus. diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index 53ef5705..ffce6cc7 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -1051,6 +1051,20 @@ spec: - source type: object type: object + recurrence: + description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API." + properties: + maxSaved: + default: 10 + description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 10. + minimum: 1 + type: integer + schedule: + description: "Perform a backup on the given schedule, in CRON format. \n Multiple CRON syntaxes are supported - Standard CRON (e.g. \"CRON_TZ=Asia/Seoul 0 6 * * ?\") - Predefined Schedules (e.g. \"@yearly\", \"@weekly\", \"@daily\", etc.) - Intervals (e.g. \"@every 10h30m\") \n For more information please check this reference: https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format" + type: string + required: + - schedule + type: object repositoryName: description: The name of the repository to use for the backup. Defaults to "legacy_local_repository" if not specified (the auto-configured repository for legacy singleton volumes). type: string @@ -1105,6 +1119,92 @@ spec: finished: description: Whether the backup has finished type: boolean + history: + description: The status history of recurring backups + items: + description: IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup + properties: + collectionBackupStatuses: + description: The status of each collection's backup progress + items: + description: CollectionBackupStatus defines the progress of a Solr Collection's backup + properties: + asyncBackupStatus: + description: The status of the asynchronous backup call to solr + type: string + backupName: + description: BackupName of this collection's backup in Solr + type: string + collection: + description: Solr Collection name + type: string + finishTimestamp: + description: Time that the collection backup finished at + format: date-time + type: string + finished: + description: Whether the backup has finished + type: boolean + inProgress: + description: Whether the collection is being backed up + type: boolean + startTimestamp: + description: Time that the collection backup started at + format: date-time + type: string + successful: + description: Whether the backup was successful + type: boolean + required: + - collection + type: object + type: array + finishTimestamp: + description: Version of the Solr being backed up + format: date-time + type: string + finished: + description: Whether the backup has finished + type: boolean + persistenceStatus: + description: Whether the backups are in progress of being persisted + properties: + finishTimestamp: + description: Time that the collection backup finished at + format: date-time + type: string + finished: + description: Whether the persistence has finished + type: boolean + inProgress: + description: Whether the collection is being backed up + type: boolean + startTimestamp: + description: Time that the collection backup started at + format: date-time + type: string + successful: + description: Whether the backup was successful + type: boolean + type: object + solrVersion: + description: Version of the Solr being backed up + type: string + startTimestamp: + description: The time that this backup was initiated + format: date-time + type: string + successful: + description: Whether the backup was successful + type: boolean + required: + - solrVersion + type: object + type: array + nextScheduledTime: + description: The scheduled time for the next backup to occur + format: date-time + type: string persistenceStatus: description: Whether the backups are in progress of being persisted properties: @@ -1129,6 +1229,10 @@ spec: solrVersion: description: Version of the Solr being backed up type: string + startTimestamp: + description: The time that this backup was initiated + format: date-time + type: string successful: description: Whether the backup was successful type: boolean diff --git a/controllers/common.go b/controllers/common.go index 38a056e3..b86fb003 100644 --- a/controllers/common.go +++ b/controllers/common.go @@ -24,6 +24,9 @@ import ( // Set the requeueAfter if it has not been set, or is greater than the new time to requeue at func updateRequeueAfter(requeueOrNot *reconcile.Result, newWait time.Duration) { + if newWait <= 0 { + requeueOrNot.RequeueAfter = 0 + } if requeueOrNot.RequeueAfter <= 0 || requeueOrNot.RequeueAfter > newWait { requeueOrNot.RequeueAfter = newWait } diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index 228036b0..97792821 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -45,7 +45,7 @@ import ( type SolrBackupReconciler struct { client.Client Scheme *runtime.Scheme - config *rest.Config + Config *rest.Config } //+kubebuilder:rbac:groups="",resources=pods/exec,verbs=create @@ -84,65 +84,103 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) changed := backup.WithDefaults() if changed { logger.Info("Setting default settings for solr-backup") - if err := r.Update(ctx, backup); err != nil { + if err = r.Update(ctx, backup); err != nil { return reconcile.Result{}, err } return reconcile.Result{Requeue: true}, nil } - // When working with the collection backups, auto-requeue after 5 seconds - // to check on the status of the async solr backup calls - requeueOrNot := reconcile.Result{Requeue: true, RequeueAfter: time.Second * 5} + requeueOrNot := reconcile.Result{} - solrCloud, allCollectionsComplete, collectionActionTaken, err := r.reconcileSolrCloudBackup(ctx, backup, logger) - if err != nil { - // TODO Should we be failing the backup for some sub-set of errors here? - logger.Error(err, "Error while taking SolrCloud backup") + // Check if we should start the next backup + if backup.Status.NextScheduledTime != nil { + if backup.Status.NextScheduledTime.UTC().After(time.Now().UTC()) { + // We have hit the next scheduled restart time. + // Set the next scheduled time to nil, and continue through to start the next backup. + requeueOrNot = reconcile.Result{Requeue: true} + } else { + // If we have not hit the next scheduled restart, wait to requeue until that is true. + updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.UTC().Sub(time.Now().UTC())) + } } - if allCollectionsComplete && collectionActionTaken { - // Requeue immediately to start the persisting job - // From here on in the backup lifecycle, requeueing will not happen for the backup. - requeueOrNot = reconcile.Result{RequeueAfter: time.Second * 10} - } else if solrCloud == nil { - requeueOrNot = reconcile.Result{} - } else { - // Only persist if the backup CRD is not finished (something bad happened) - // and the collection backups are all complete (not necessarily successful) - // Do not do this right after the collectionsBackup have been complete, wait till the next cycle - if allCollectionsComplete && !backup.Status.Finished { - if backup.Spec.Persistence != nil { - // We will count on the Job updates to be notified - requeueOrNot = reconcile.Result{} - err = r.persistSolrCloudBackups(ctx, backup, solrCloud, logger) - if err != nil { - logger.Error(err, "Error while persisting SolrCloud backup") + + // Do backup work if a nextScheduledTime is not set and there is no current finish time. + // The nextScheduledTime is only set when the last backup is finished and we are waiting to do the next + if backup.Status.NextScheduledTime == nil && backup.Status.Current.FinishTime == nil { + // When working with the collection backups, auto-requeue after 5 seconds + // to check on the status of the async solr backup calls + updateRequeueAfter(&requeueOrNot, time.Second*5) + + solrCloud, allCollectionsComplete, collectionActionTaken, err1 := r.reconcileSolrCloudBackup(ctx, backup, logger) + if err1 != nil { + // TODO Should we be failing the backup for some sub-set of errors here? + logger.Error(err1, "Error while taking SolrCloud backup") + } + if allCollectionsComplete && collectionActionTaken { + // Requeue immediately to start the persisting job + // From here on in the backup lifecycle, requeueing will not happen for the backup. + updateRequeueAfter(&requeueOrNot, time.Second*10) + } else if solrCloud == nil { + requeueOrNot = reconcile.Result{} + } else { + // Only persist if the backup CRD is not finished (something bad happened) + // and the collection backups are all complete (not necessarily successful) + // Do not do this right after the collectionsBackup have been complete, wait till the next cycle + if allCollectionsComplete && !backup.Status.Current.Finished { + if backup.Spec.Persistence != nil { + // We will count on the Job updates to be notified + requeueOrNot = reconcile.Result{} + err = r.persistSolrCloudBackups(ctx, backup, solrCloud, logger) + if err != nil { + logger.Error(err, "Error while persisting SolrCloud backup") + } + } else { + tru := true + backup.Status.Current.Successful = &tru + now := metav1.Now() + backup.Status.Current.FinishTime = &now + // No need to requeue for backup purposes (unless for the next backup, handled at the end of the reconcile) + requeueOrNot = reconcile.Result{} + if backup.Spec.Recurrence != nil { + // Add the current backup to the front of the history. + // If there is no max + backup.Status.History = append([]solrv1beta1.IndividualSolrBackupStatus{backup.Status.Current}, backup.Status.History...) + + // Remove history if we have too much saved + if len(backup.Status.History) > backup.Spec.Recurrence.MaxSaved { + backup.Status.History = backup.Status.History[:backup.Spec.Recurrence.MaxSaved] + } + + if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.Current.FinishTime.Time); err1 != nil { + logger.Error(err1, "Could not schedule new backup due to back schedule") + } else { + convTime := metav1.NewTime(nextRestartTime) + backup.Status.NextScheduledTime = &convTime + } + + // Reset Current, which is fine since it is now in the history. + backup.Status.Current = solrv1beta1.IndividualSolrBackupStatus{} + } } - } else { - // Persistence not configured for this backup, mark as finished. - tru := true - backup.Status.Finished = true - backup.Status.Successful = &tru - now := metav1.Now() - backup.Status.FinishTime = &now } } - } - if backup.Status.Finished && backup.Status.FinishTime == nil { - now := metav1.Now() - backup.Status.FinishTime = &now - if backup.Spec.Persistence != nil { - backup.Status.Successful = backup.Status.PersistenceStatus.Successful + if backup.Status.Current.Finished && backup.Status.Current.FinishTime == nil { + now := metav1.Now() + backup.Status.Current.FinishTime = &now + if backup.Spec.Persistence != nil { + backup.Status.Current.Successful = backup.Status.Current.PersistenceStatus.Successful + } } - } - if !reflect.DeepEqual(oldStatus, &backup.Status) { - logger.Info("Updating status for solr-backup") - err = r.Status().Update(ctx, backup) - } + if !reflect.DeepEqual(oldStatus, &backup.Status) { + logger.Info("Updating status for solr-backup") + err = r.Status().Update(ctx, backup) + } - if err != nil && backup.Status.Finished { - requeueOrNot = reconcile.Result{} + if backup.Status.NextScheduledTime != nil { + updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now())) + } } return requeueOrNot, err @@ -187,9 +225,9 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac } // This should only occur before the backup processes have been started - if backup.Status.SolrVersion == "" { + if backup.Status.Current.StartTime.IsZero() { // Prep the backup directory in the persistentVolume - err = util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup, r.config) + err = util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup, r.Config) if err != nil { return solrCloud, collectionBackupsFinished, actionTaken, err } @@ -203,7 +241,8 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac } // Only set the solr version at the start of the backup. This shouldn't change throughout the backup. - backup.Status.SolrVersion = solrCloud.Status.Version + backup.Status.Current.SolrVersion = solrCloud.Status.Version + backup.Status.Current.StartTime = metav1.Now() } // Go through each collection specified and reconcile the backup. @@ -223,7 +262,7 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so collectionBackupStatus.Collection = collection backupIndex := -1 // Get the backup status for this collection, if one exists - for i, status := range backup.Status.CollectionBackupStatuses { + for i, status := range backup.Status.Current.CollectionBackupStatuses { if status.Collection == collection { collectionBackupStatus = status backupIndex = i @@ -266,19 +305,19 @@ func reconcileSolrCollectionBackup(backup *solrv1beta1.SolrBackup, solrCloud *so } if backupIndex < 0 { - backup.Status.CollectionBackupStatuses = append(backup.Status.CollectionBackupStatuses, collectionBackupStatus) + backup.Status.Current.CollectionBackupStatuses = append(backup.Status.Current.CollectionBackupStatuses, collectionBackupStatus) } else { - backup.Status.CollectionBackupStatuses[backupIndex] = collectionBackupStatus + backup.Status.Current.CollectionBackupStatuses[backupIndex] = collectionBackupStatus } return collectionBackupStatus.Finished, err } func (r *SolrBackupReconciler) persistSolrCloudBackups(ctx context.Context, backup *solrv1beta1.SolrBackup, solrCloud *solrv1beta1.SolrCloud, logger logr.Logger) (err error) { - if backup.Status.PersistenceStatus == nil { - backup.Status.PersistenceStatus = &solrv1beta1.BackupPersistenceStatus{} + if backup.Status.Current.PersistenceStatus == nil { + backup.Status.Current.PersistenceStatus = &solrv1beta1.BackupPersistenceStatus{} } - if backup.Status.PersistenceStatus.Finished { + if backup.Status.Current.PersistenceStatus.Finished { return nil } now := metav1.Now() @@ -299,18 +338,18 @@ func (r *SolrBackupReconciler) persistSolrCloudBackups(ctx context.Context, back foundPersistenceJob := &batchv1.Job{} err = r.Get(ctx, types.NamespacedName{Name: persistenceJob.Name, Namespace: persistenceJob.Namespace}, foundPersistenceJob) - if err == nil && !backup.Status.PersistenceStatus.InProgress { + if err == nil && !backup.Status.Current.PersistenceStatus.InProgress { } else if err != nil && errors.IsNotFound(err) { logger.Info("Creating Persistence Job", "job", persistenceJob.Name) err = r.Create(ctx, persistenceJob) - backup.Status.PersistenceStatus.InProgress = true - if backup.Status.PersistenceStatus.StartTime == nil { - backup.Status.PersistenceStatus.StartTime = &now + backup.Status.Current.PersistenceStatus.InProgress = true + if backup.Status.Current.PersistenceStatus.StartTime == nil { + backup.Status.Current.PersistenceStatus.StartTime = &now } } else if err != nil { return err } else { - backup.Status.PersistenceStatus.FinishTime = foundPersistenceJob.Status.CompletionTime + backup.Status.Current.PersistenceStatus.FinishTime = foundPersistenceJob.Status.CompletionTime tru := true fals := false numFailLimit := int32(0) @@ -318,17 +357,17 @@ func (r *SolrBackupReconciler) persistSolrCloudBackups(ctx context.Context, back numFailLimit = *foundPersistenceJob.Spec.BackoffLimit } if foundPersistenceJob.Status.Succeeded > 0 { - backup.Status.PersistenceStatus.Successful = &tru + backup.Status.Current.PersistenceStatus.Successful = &tru } else if foundPersistenceJob.Status.Failed > numFailLimit { - backup.Status.PersistenceStatus.Successful = &fals + backup.Status.Current.PersistenceStatus.Successful = &fals } - if backup.Status.PersistenceStatus.Successful != nil { - backup.Status.PersistenceStatus.InProgress = false - backup.Status.PersistenceStatus.Finished = true - backup.Status.PersistenceStatus.FinishTime = &now - backup.Status.Finished = true - backup.Status.Successful = backup.Status.PersistenceStatus.Successful + if backup.Status.Current.PersistenceStatus.Successful != nil { + backup.Status.Current.PersistenceStatus.InProgress = false + backup.Status.Current.PersistenceStatus.Finished = true + backup.Status.Current.PersistenceStatus.FinishTime = &now + backup.Status.Current.Finished = true + backup.Status.Current.Successful = backup.Status.Current.PersistenceStatus.Successful } } return err @@ -339,8 +378,6 @@ func (r *SolrBackupReconciler) persistSolrCloudBackups(ctx context.Context, back // SetupWithManager sets up the controller with the Manager. func (r *SolrBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.config = mgr.GetConfig() - return ctrl.NewControllerManagedBy(mgr). For(&solrv1beta1.SolrBackup{}). Owns(&batchv1.Job{}). diff --git a/controllers/solrprometheusexporter_controller.go b/controllers/solrprometheusexporter_controller.go index b6dd1292..6902df10 100644 --- a/controllers/solrprometheusexporter_controller.go +++ b/controllers/solrprometheusexporter_controller.go @@ -383,7 +383,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForSolrClouds(mgr ctrl.M solrCloudField := ".spec.solrReference.cloud.name" if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, solrCloudField, func(rawObj client.Object) []string { - // grab the SolrCloud object, extract the used configMap... + // grab the SolrPrometheusExporter object, extract the used SolrCloud... exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter) if exporter.Spec.SolrReference.Cloud == nil { return nil @@ -429,7 +429,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForProvidedConfigMaps(mg providedConfigMapField := ".spec.customKubeOptions.configMapOptions.providedConfigMap" if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, providedConfigMapField, func(rawObj client.Object) []string { - // grab the SolrCloud object, extract the used configMap... + // grab the SolrPrometheusExporter object, extract the used configMap... exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter) if exporter.Spec.CustomKubeOptions.ConfigMapOptions == nil { return nil @@ -475,7 +475,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForKeystoreSecret(mgr ct tlsSecretField := ".spec.solrReference.solrTLS.pkcs12Secret" if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, tlsSecretField, func(rawObj client.Object) []string { - // grab the SolrCloud object, extract the referenced TLS secret... + // grab the SolrPrometheusExporter object, extract the referenced TLS secret... exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter) if exporter.Spec.SolrReference.SolrTLS == nil || exporter.Spec.SolrReference.SolrTLS.PKCS12Secret == nil { return nil @@ -493,7 +493,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForTruststoreSecret(mgr tlsSecretField := ".spec.solrReference.solrTLS.trustStoreSecret" if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, tlsSecretField, func(rawObj client.Object) []string { - // grab the SolrCloud object, extract the referenced truststore secret... + // grab the SolrPrometheusExporter object, extract the referenced truststore secret... exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter) if exporter.Spec.SolrReference.SolrTLS == nil || exporter.Spec.SolrReference.SolrTLS.TrustStoreSecret == nil { return nil @@ -511,7 +511,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForBasicAuthSecret(mgr c secretField := ".spec.solrReference.basicAuthSecret" if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, secretField, func(rawObj client.Object) []string { - // grab the SolrCloud object, extract the referenced TLS secret... + // grab the SolrPrometheusExporter object, extract the referenced BasicAuth secret... exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter) if exporter.Spec.SolrReference.BasicAuthSecret == "" { return nil diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go index d0dd2d87..f6a1e285 100644 --- a/controllers/util/backup_util.go +++ b/controllers/util/backup_util.go @@ -23,6 +23,7 @@ import ( solr "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/controllers/util/solr_api" "github.com/go-logr/logr" + "github.com/robfig/cron/v3" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,6 +32,8 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" "net/url" + "strconv" + "time" ) const ( @@ -69,19 +72,19 @@ func CheckStatusOfCollectionBackups(backup *solr.SolrBackup) (allFinished bool) fals := false // Check if all collection backups have been completed, this is updated in the loop - allFinished = len(backup.Status.CollectionBackupStatuses) > 0 + allFinished = len(backup.Status.Current.CollectionBackupStatuses) > 0 // Check if persistence should be skipped if no backup completed successfully anySuccessful := false - for _, collectionStatus := range backup.Status.CollectionBackupStatuses { + for _, collectionStatus := range backup.Status.Current.CollectionBackupStatuses { allFinished = allFinished && collectionStatus.Finished anySuccessful = anySuccessful || (collectionStatus.Successful != nil && *collectionStatus.Successful) } if allFinished && !anySuccessful { - backup.Status.Finished = true - if backup.Status.Successful == nil { - backup.Status.Successful = &fals + backup.Status.Current.Finished = true + if backup.Status.Current.Successful == nil { + backup.Status.Current.Successful = &fals } } return @@ -317,6 +320,11 @@ func GenerateQueryParamsForBackup(backupRepository *solr.SolrBackupRepository, b queryParams.Add("async", AsyncIdForCollectionBackup(collection, backup.Name)) queryParams.Add("location", BackupLocationPath(backupRepository, backup.Spec.Location)) queryParams.Add("repository", backup.Spec.RepositoryName) + + if backup.Spec.Recurrence != nil { + queryParams.Add("maxNumBackupPoints", strconv.Itoa(backup.Spec.Recurrence.MaxSaved)) + } + return queryParams } @@ -391,15 +399,15 @@ func EnsureDirectoryForBackup(solrCloud *solr.SolrCloud, backupRepository *solr. solrCloud.GetAllSolrPodNames()[0], solrCloud.Namespace, []string{"/bin/bash", "-c", "rm -rf " + backupPath + " && mkdir -p " + backupPath}, - *config, + config, ) } return nil } -func RunExecForPod(podName string, namespace string, command []string, config rest.Config) (err error) { +func RunExecForPod(podName string, namespace string, command []string, config *rest.Config) (err error) { client := &kubernetes.Clientset{} - if client, err = kubernetes.NewForConfig(&config); err != nil { + if client, err = kubernetes.NewForConfig(config); err != nil { return err } req := client.CoreV1().RESTClient().Post(). @@ -422,7 +430,7 @@ func RunExecForPod(podName string, namespace string, command []string, config re TTY: false, }, parameterCodec) - exec, err := remotecommand.NewSPDYExecutor(&config, "POST", req.URL()) + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) if err != nil { return fmt.Errorf("error while creating Executor: %v", err) } @@ -440,3 +448,12 @@ func RunExecForPod(podName string, namespace string, command []string, config re return nil } + +func ScheduleNextBackup(restartSchedule string, lastBackupTime time.Time) (nextBackup time.Time, err error) { + if parsedSchedule, parseErr := cron.ParseStandard(restartSchedule); parseErr != nil { + err = parseErr + } else { + nextBackup = parsedSchedule.Next(lastBackupTime) + } + return +} diff --git a/controllers/util/solr_update_util.go b/controllers/util/solr_update_util.go index 72f373d9..e7f47c83 100644 --- a/controllers/util/solr_update_util.go +++ b/controllers/util/solr_update_util.go @@ -76,7 +76,7 @@ func scheduleNextRestartWithTime(restartSchedule string, podTemplateAnnotations err = parseErr } else { nextRestartTime := parsedSchedule.Next(lastScheduledTime) - nextRestart = parsedSchedule.Next(lastScheduledTime).Format(time.RFC3339) + nextRestart = nextRestartTime.Format(time.RFC3339) reconcileWaitDurationTmp := nextRestartTime.Sub(currentTime) reconcileWaitDuration = &reconcileWaitDurationTmp } diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index 43a95519..7cc1278c 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -1051,6 +1051,20 @@ spec: - source type: object type: object + recurrence: + description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API." + properties: + maxSaved: + default: 10 + description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 10. + minimum: 1 + type: integer + schedule: + description: "Perform a backup on the given schedule, in CRON format. \n Multiple CRON syntaxes are supported - Standard CRON (e.g. \"CRON_TZ=Asia/Seoul 0 6 * * ?\") - Predefined Schedules (e.g. \"@yearly\", \"@weekly\", \"@daily\", etc.) - Intervals (e.g. \"@every 10h30m\") \n For more information please check this reference: https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format" + type: string + required: + - schedule + type: object repositoryName: description: The name of the repository to use for the backup. Defaults to "legacy_local_repository" if not specified (the auto-configured repository for legacy singleton volumes). type: string @@ -1105,6 +1119,92 @@ spec: finished: description: Whether the backup has finished type: boolean + history: + description: The status history of recurring backups + items: + description: IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup + properties: + collectionBackupStatuses: + description: The status of each collection's backup progress + items: + description: CollectionBackupStatus defines the progress of a Solr Collection's backup + properties: + asyncBackupStatus: + description: The status of the asynchronous backup call to solr + type: string + backupName: + description: BackupName of this collection's backup in Solr + type: string + collection: + description: Solr Collection name + type: string + finishTimestamp: + description: Time that the collection backup finished at + format: date-time + type: string + finished: + description: Whether the backup has finished + type: boolean + inProgress: + description: Whether the collection is being backed up + type: boolean + startTimestamp: + description: Time that the collection backup started at + format: date-time + type: string + successful: + description: Whether the backup was successful + type: boolean + required: + - collection + type: object + type: array + finishTimestamp: + description: Version of the Solr being backed up + format: date-time + type: string + finished: + description: Whether the backup has finished + type: boolean + persistenceStatus: + description: Whether the backups are in progress of being persisted + properties: + finishTimestamp: + description: Time that the collection backup finished at + format: date-time + type: string + finished: + description: Whether the persistence has finished + type: boolean + inProgress: + description: Whether the collection is being backed up + type: boolean + startTimestamp: + description: Time that the collection backup started at + format: date-time + type: string + successful: + description: Whether the backup was successful + type: boolean + type: object + solrVersion: + description: Version of the Solr being backed up + type: string + startTimestamp: + description: The time that this backup was initiated + format: date-time + type: string + successful: + description: Whether the backup was successful + type: boolean + required: + - solrVersion + type: object + type: array + nextScheduledTime: + description: The scheduled time for the next backup to occur + format: date-time + type: string persistenceStatus: description: Whether the backups are in progress of being persisted properties: @@ -1129,6 +1229,10 @@ spec: solrVersion: description: Version of the Solr being backed up type: string + startTimestamp: + description: The time that this backup was initiated + format: date-time + type: string successful: description: Whether the backup was successful type: boolean diff --git a/main.go b/main.go index ae118d04..11d8606f 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ import ( "flag" "fmt" "github.com/apache/solr-operator/controllers/util/solr_api" - zk_api "github.com/apache/solr-operator/controllers/zk_api" + "github.com/apache/solr-operator/controllers/zk_api" "github.com/apache/solr-operator/version" "github.com/fsnotify/fsnotify" "io/ioutil" @@ -203,6 +203,7 @@ func main() { if err = (&controllers.SolrBackupReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + Config: mgr.GetConfig(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SolrBackup") os.Exit(1) From 5eef36986d9b4fe9b3160138d23d2591ca489b18 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 12:42:41 -0500 Subject: [PATCH 02/11] Fix issues with merge. --- .../bases/solr.apache.org_solrbackups.yaml | 2 +- controllers/solrbackup_controller.go | 85 +++++++++---------- controllers/util/backup_util.go | 4 +- helm/solr-operator/crds/crds.yaml | 2 +- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index 1f52234f..105c793c 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -1173,7 +1173,7 @@ spec: description: Whether the backup has finished type: boolean persistenceStatus: - description: Whether the backups are in progress of being persisted + description: Whether the backups are in progress of being persisted. This feature has been removed as of v0.5.0. properties: finishTimestamp: description: Time that the collection backup finished at diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index aa9c2b7d..1c91f1db 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -30,7 +30,6 @@ import ( "github.com/apache/solr-operator/controllers/util" "github.com/go-logr/logr" - batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -108,55 +107,54 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Do backup work if a nextScheduledTime is not set and there is no current finish time. // The nextScheduledTime is only set when the last backup is finished and we are waiting to do the next if backup.Status.NextScheduledTime == nil && backup.Status.Current.FinishTime == nil { - requeueOrNot := reconcile.Result{} - + requeueOrNot = reconcile.Result{} + solrCloud, _, err1 := r.reconcileSolrCloudBackup(ctx, backup, logger) if err1 != nil { // TODO Should we be failing the backup for some sub-set of errors here? logger.Error(err1, "Error while taking SolrCloud backup") - - // Requeue after 10 seconds for errors. - requeueOrNot = reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10} + + // Requeue after 10 seconds for errors. + requeueOrNot = reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10} } else if backup.Status.Current.Finished { - // Set finish time - now := metav1.Now() - backup.Status.Current.FinishTime = &now - - if backup.Spec.Recurrence != nil { - // Add the current backup to the front of the history. - // If there is no max - backup.Status.History = append([]solrv1beta1.IndividualSolrBackupStatus{backup.Status.Current}, backup.Status.History...) - - // Remove history if we have too much saved - if len(backup.Status.History) > backup.Spec.Recurrence.MaxSaved { - backup.Status.History = backup.Status.History[:backup.Spec.Recurrence.MaxSaved] - } - - if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.Current.FinishTime.Time); err1 != nil { - logger.Error(err1, "Could not schedule new backup due to back schedule") - } else { - convTime := metav1.NewTime(nextRestartTime) - backup.Status.NextScheduledTime = &convTime - } - - // Reset Current, which is fine since it is now in the history. - backup.Status.Current = solrv1beta1.IndividualSolrBackupStatus{} + // Set finish time + now := metav1.Now() + backup.Status.Current.FinishTime = &now + + if backup.Spec.Recurrence != nil { + // Add the current backup to the front of the history. + // If there is no max + backup.Status.History = append([]solrv1beta1.IndividualSolrBackupStatus{backup.Status.Current}, backup.Status.History...) + + // Remove history if we have too much saved + if len(backup.Status.History) > backup.Spec.Recurrence.MaxSaved { + backup.Status.History = backup.Status.History[:backup.Spec.Recurrence.MaxSaved] + } + + if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.Current.FinishTime.Time); err1 != nil { + logger.Error(err1, "Could not schedule new backup due to back schedule") + } else { + convTime := metav1.NewTime(nextRestartTime) + backup.Status.NextScheduledTime = &convTime + } + + // Reset Current, which is fine since it is now in the history. + backup.Status.Current = solrv1beta1.IndividualSolrBackupStatus{} } - } else if solrCloud != nil { - // When working with the collection backups, auto-requeue after 5 seconds - // to check on the status of the async solr backup calls - updateRequeueAfter(&requeueOrNot, time.Second*5) - } + } else if solrCloud != nil { + // When working with the collection backups, auto-requeue after 5 seconds + // to check on the status of the async solr backup calls + updateRequeueAfter(&requeueOrNot, time.Second*5) + } } - if !reflect.DeepEqual(oldStatus, &backup.Status) { - logger.Info("Updating status for solr-backup") - err = r.Status().Update(ctx, backup) - } + if !reflect.DeepEqual(oldStatus, &backup.Status) { + logger.Info("Updating status for solr-backup") + err = r.Status().Update(ctx, backup) + } - if backup.Status.NextScheduledTime != nil { - updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now())) - } + if backup.Status.NextScheduledTime != nil { + updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now())) } return requeueOrNot, err @@ -294,7 +292,7 @@ func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.Solr // SetupWithManager sets up the controller with the Manager. func (r *SolrBackupReconciler) SetupWithManager(mgr ctrl.Manager) (err error) { - r.config = mgr.GetConfig() + r.Config = mgr.GetConfig() ctrlBuilder := ctrl.NewControllerManagedBy(mgr). For(&solrv1beta1.SolrBackup{}) @@ -307,7 +305,6 @@ func (r *SolrBackupReconciler) SetupWithManager(mgr ctrl.Manager) (err error) { return ctrlBuilder.Complete(r) } - func (r *SolrBackupReconciler) indexAndWatchForSolrClouds(mgr ctrl.Manager, ctrlBuilder *builder.Builder) (*builder.Builder, error) { solrCloudField := ".spec.solrCloud" @@ -317,7 +314,7 @@ func (r *SolrBackupReconciler) indexAndWatchForSolrClouds(mgr ctrl.Manager, ctrl }); err != nil { return ctrlBuilder, err } - + return ctrlBuilder.Watches( &source.Kind{Type: &solrv1beta1.SolrCloud{}}, handler.EnqueueRequestsFromMapFunc(func(obj client.Object) []reconcile.Request { diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go index f352b439..cb839bac 100644 --- a/controllers/util/backup_util.go +++ b/controllers/util/backup_util.go @@ -61,13 +61,13 @@ func UpdateStatusOfCollectionBackups(backup *solr.SolrBackup) (allFinished bool) // Check if all collection backups have been completed, this is updated in the loop allFinished = len(backup.Status.Current.CollectionBackupStatuses) > 0 - allSuccessful := len(backup.Status.CollectionBackupStatuses) > 0 + allSuccessful := len(backup.Status.Current.CollectionBackupStatuses) > 0 for _, collectionStatus := range backup.Status.Current.CollectionBackupStatuses { allFinished = allFinished && collectionStatus.Finished allSuccessful = allSuccessful && (collectionStatus.Successful != nil && *collectionStatus.Successful) } - + backup.Status.Current.Finished = allFinished if allFinished && backup.Status.Current.Successful == nil { backup.Status.Current.Successful = &allSuccessful diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index cb044aef..db11541c 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -1173,7 +1173,7 @@ spec: description: Whether the backup has finished type: boolean persistenceStatus: - description: Whether the backups are in progress of being persisted + description: Whether the backups are in progress of being persisted. This feature has been removed as of v0.5.0. properties: finishTimestamp: description: Time that the collection backup finished at From 5f644e20a2a99ac83b4b71f4a05e276a70b9d67e Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 12:43:33 -0500 Subject: [PATCH 03/11] Change Backup maxSaved from 10 to 5. --- api/v1beta1/solrbackup_types.go | 4 ++-- config/crd/bases/solr.apache.org_solrbackups.yaml | 4 ++-- helm/solr-operator/crds/crds.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index 1447a136..dfa16eec 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -89,9 +89,9 @@ type BackupRecurrence struct { // Define the number of backup points to save for this backup at any given time. // The oldest backups will be deleted if too many exist when a backup is taken. - // If not provided, this defaults to 10. + // If not provided, this defaults to 5. // - // +kubebuilder:default:=10 + // +kubebuilder:default:=5 // +kubebuilder:validation:Minimum:=1 // +optional MaxSaved int `json:"maxSaved,omitempty"` diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index 105c793c..fe74b3a0 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -1055,8 +1055,8 @@ spec: description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API." properties: maxSaved: - default: 10 - description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 10. + default: 5 + description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 5. minimum: 1 type: integer schedule: diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index db11541c..9716c085 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -1055,8 +1055,8 @@ spec: description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API." properties: maxSaved: - default: 10 - description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 10. + default: 5 + description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 5. minimum: 1 type: integer schedule: From edc66327cfdbd3df5cfcc32df5a1924b2e9c1733 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 14:26:35 -0500 Subject: [PATCH 04/11] Doesn't work but getting closer. Status is not being written --- api/v1beta1/solrbackup_types.go | 22 +--- .../bases/solr.apache.org_solrbackups.yaml | 8 +- controllers/solrbackup_controller.go | 112 ++++++++++-------- controllers/util/backup_util.go | 14 +-- helm/solr-operator/crds/crds.yaml | 8 +- 5 files changed, 80 insertions(+), 84 deletions(-) diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index dfa16eec..c9d8a2c4 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -21,7 +21,6 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" ) // SolrBackupSpec defines the desired state of SolrBackup @@ -190,23 +189,6 @@ type VolumePersistenceSource struct { BusyBoxImage ContainerImage `json:"busyBoxImage,omitempty"` } -// Deprecated: Will be unused as of v0.5.0 -func (spec *VolumePersistenceSource) withDefaults(backupName string) (changed bool) { - changed = spec.BusyBoxImage.withDefaults(DefaultBusyBoxImageRepo, DefaultBusyBoxImageVersion, DefaultPullPolicy) || changed - - if spec.Path != "" && strings.HasPrefix(spec.Path, "/") { - spec.Path = strings.TrimPrefix(spec.Path, "/") - changed = true - } - - if spec.Filename == "" { - spec.Filename = backupName + ".tgz" - changed = true - } - - return changed -} - // SolrBackupStatus defines the observed state of SolrBackup type SolrBackupStatus struct { // The current Backup Status, which all fields are added to this struct @@ -224,7 +206,8 @@ type SolrBackupStatus struct { // IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup type IndividualSolrBackupStatus struct { // Version of the Solr being backed up - SolrVersion string `json:"solrVersion"` + // +optional + SolrVersion string `json:"solrVersion,omitempty"` // The time that this backup was initiated // +optional @@ -339,6 +322,7 @@ func (sb *SolrBackup) PersistenceJobName() string { //+kubebuilder:printcolumn:name="Cloud",type="string",JSONPath=".spec.solrCloud",description="Solr Cloud" //+kubebuilder:printcolumn:name="Finished",type="boolean",JSONPath=".status.finished",description="Whether the backup has finished" //+kubebuilder:printcolumn:name="Successful",type="boolean",JSONPath=".status.successful",description="Whether the backup was successful" +//+kubebuilder:printcolumn:name="NextBackup",type="boolean",JSONPath=".status.nextScheduledTime",description="Next scheduled time for a recurrent backup" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // SolrBackup is the Schema for the solrbackups API diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index fe74b3a0..fc313b47 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -43,6 +43,10 @@ spec: jsonPath: .status.successful name: Successful type: boolean + - description: Next scheduled time for a recurrent backup + jsonPath: .status.nextScheduledTime + name: NextBackup + type: boolean - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -1203,8 +1207,6 @@ spec: successful: description: Whether the backup was successful type: boolean - required: - - solrVersion type: object type: array nextScheduledTime: @@ -1242,8 +1244,6 @@ spec: successful: description: Whether the backup was successful type: boolean - required: - - solrVersion type: object type: object served: true diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index 1c91f1db..8b24e0f0 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -79,8 +79,6 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) return reconcile.Result{}, err } - oldStatus := backup.Status.DeepCopy() - changed := backup.WithDefaults() if changed { logger.Info("Setting default settings for solr-backup") @@ -90,57 +88,54 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) return reconcile.Result{Requeue: true}, nil } + newStatus := backup.Status.DeepCopy() + requeueOrNot := reconcile.Result{} + var backupNeedsToWait bool + // Check if we should start the next backup - if backup.Status.NextScheduledTime != nil { - if backup.Status.NextScheduledTime.UTC().After(time.Now().UTC()) { + if newStatus.NextScheduledTime != nil { + if newStatus.NextScheduledTime.UTC().After(time.Now().UTC()) { // We have hit the next scheduled restart time. - // Set the next scheduled time to nil, and continue through to start the next backup. - requeueOrNot = reconcile.Result{Requeue: true} + backupNeedsToWait = false + newStatus.NextScheduledTime = nil + + // Add the current backup to the front of the history. + // If there is no max + newStatus.History = append([]solrv1beta1.IndividualSolrBackupStatus{newStatus.Current}, newStatus.History...) + + // Remove history if we have too much saved + if len(newStatus.History) > backup.Spec.Recurrence.MaxSaved { + newStatus.History = newStatus.History[:backup.Spec.Recurrence.MaxSaved] + } + + // Reset Current, which is fine since it is now in the history. + newStatus.Current = solrv1beta1.IndividualSolrBackupStatus{} } else { // If we have not hit the next scheduled restart, wait to requeue until that is true. - updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.UTC().Sub(time.Now().UTC())) + updateRequeueAfter(&requeueOrNot, newStatus.NextScheduledTime.UTC().Sub(time.Now().UTC())) + backupNeedsToWait = true } + } else { + backupNeedsToWait = false } - // Do backup work if a nextScheduledTime is not set and there is no current finish time. - // The nextScheduledTime is only set when the last backup is finished and we are waiting to do the next - if backup.Status.NextScheduledTime == nil && backup.Status.Current.FinishTime == nil { + // Do backup work if we are not waiting and the current backup is not finished + if !backupNeedsToWait && !newStatus.Current.Finished { requeueOrNot = reconcile.Result{} - solrCloud, _, err1 := r.reconcileSolrCloudBackup(ctx, backup, logger) + solrCloud, _, err1 := r.reconcileSolrCloudBackup(ctx, backup, &newStatus.Current, logger) if err1 != nil { // TODO Should we be failing the backup for some sub-set of errors here? logger.Error(err1, "Error while taking SolrCloud backup") // Requeue after 10 seconds for errors. requeueOrNot = reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10} - } else if backup.Status.Current.Finished { + } else if newStatus.Current.Finished { // Set finish time now := metav1.Now() - backup.Status.Current.FinishTime = &now - - if backup.Spec.Recurrence != nil { - // Add the current backup to the front of the history. - // If there is no max - backup.Status.History = append([]solrv1beta1.IndividualSolrBackupStatus{backup.Status.Current}, backup.Status.History...) - - // Remove history if we have too much saved - if len(backup.Status.History) > backup.Spec.Recurrence.MaxSaved { - backup.Status.History = backup.Status.History[:backup.Spec.Recurrence.MaxSaved] - } - - if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.Current.FinishTime.Time); err1 != nil { - logger.Error(err1, "Could not schedule new backup due to back schedule") - } else { - convTime := metav1.NewTime(nextRestartTime) - backup.Status.NextScheduledTime = &convTime - } - - // Reset Current, which is fine since it is now in the history. - backup.Status.Current = solrv1beta1.IndividualSolrBackupStatus{} - } + newStatus.Current.FinishTime = &now } else if solrCloud != nil { // When working with the collection backups, auto-requeue after 5 seconds // to check on the status of the async solr backup calls @@ -148,19 +143,34 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - if !reflect.DeepEqual(oldStatus, &backup.Status) { - logger.Info("Updating status for solr-backup") + // Schedule the next backupTime, if it doesn't have a next scheduled time, it has recurrence and the current backup is finished + if newStatus.NextScheduledTime == nil && backup.Spec.Recurrence != nil && newStatus.Current.Finished { + if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, newStatus.Current.FinishTime.Time); err1 != nil { + logger.Error(err1, "Could not schedule new backup due to back schedule") + } else { + logger.Info("Scheduling Next Backup for time %v", nextRestartTime) + convTime := metav1.NewTime(nextRestartTime) + newStatus.NextScheduledTime = &convTime + } + } + + if !reflect.DeepEqual(*newStatus, backup.Status) { + backup.Status = *newStatus + logger.Info("Updating status for solr-backup", "status", backup.Status) err = r.Status().Update(ctx, backup) + if err != nil { + logger.Error(err, "Could not update Status for solr-backup") + } } - if backup.Status.NextScheduledTime != nil { - updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now())) + if newStatus.NextScheduledTime != nil { + updateRequeueAfter(&requeueOrNot, newStatus.NextScheduledTime.Sub(time.Now())) } return requeueOrNot, err } -func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, logger logr.Logger) (solrCloud *solrv1beta1.SolrCloud, actionTaken bool, err error) { +func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, currentBackupStatus *solrv1beta1.IndividualSolrBackupStatus, logger logr.Logger) (solrCloud *solrv1beta1.SolrCloud, actionTaken bool, err error) { // Get the solrCloud that this backup is for. solrCloud = &solrv1beta1.SolrCloud{} @@ -181,7 +191,7 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac } // First check if the collection backups have been completed - collectionBackupsFinished := util.UpdateStatusOfCollectionBackups(backup) + collectionBackupsFinished := util.UpdateStatusOfCollectionBackups(currentBackupStatus) // If the collectionBackups are complete, then nothing else has to be done here if collectionBackupsFinished { @@ -198,7 +208,7 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac } // This should only occur before the backup processes have been started - if backup.Status.Current.StartTime.IsZero() { + if currentBackupStatus.StartTime.IsZero() { // Prep the backup directory in the persistentVolume err = util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup, r.Config) if err != nil { @@ -212,31 +222,31 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac } // Only set the solr version at the start of the backup. This shouldn't change throughout the backup. - backup.Status.Current.SolrVersion = solrCloud.Status.Version - backup.Status.Current.StartTime = metav1.Now() + currentBackupStatus.SolrVersion = solrCloud.Status.Version + currentBackupStatus.StartTime = metav1.Now() } // Go through each collection specified and reconcile the backup. for _, collection := range backup.Spec.Collections { // This will in-place update the CollectionBackupStatus in the backup object - if _, err = reconcileSolrCollectionBackup(ctx, backup, solrCloud, backupRepository, collection, logger); err != nil { + if _, err = reconcileSolrCollectionBackup(ctx, backup, currentBackupStatus, solrCloud, backupRepository, collection, logger); err != nil { break } } // First check if the collection backups have been completed - util.UpdateStatusOfCollectionBackups(backup) + util.UpdateStatusOfCollectionBackups(currentBackupStatus) return solrCloud, actionTaken, err } -func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, solrCloud *solrv1beta1.SolrCloud, backupRepository *solrv1beta1.SolrBackupRepository, collection string, logger logr.Logger) (finished bool, err error) { +func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, currentBackupStatus *solrv1beta1.IndividualSolrBackupStatus, solrCloud *solrv1beta1.SolrCloud, backupRepository *solrv1beta1.SolrBackupRepository, collection string, logger logr.Logger) (finished bool, err error) { now := metav1.Now() collectionBackupStatus := solrv1beta1.CollectionBackupStatus{} collectionBackupStatus.Collection = collection backupIndex := -1 // Get the backup status for this collection, if one exists - for i, status := range backup.Status.Current.CollectionBackupStatuses { + for i, status := range currentBackupStatus.CollectionBackupStatuses { if status.Collection == collection { collectionBackupStatus = status backupIndex = i @@ -244,7 +254,9 @@ func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.Solr } // If the collection backup hasn't started, start it - if !collectionBackupStatus.InProgress && !collectionBackupStatus.Finished { + if collectionBackupStatus.Finished { + return true, nil + } else if !collectionBackupStatus.InProgress { // Start the backup by calling solr var started bool started, err = util.StartBackupForCollection(ctx, solrCloud, backupRepository, backup, collection, logger) @@ -282,9 +294,9 @@ func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.Solr } if backupIndex < 0 { - backup.Status.Current.CollectionBackupStatuses = append(backup.Status.Current.CollectionBackupStatuses, collectionBackupStatus) + currentBackupStatus.CollectionBackupStatuses = append(currentBackupStatus.CollectionBackupStatuses, collectionBackupStatus) } else { - backup.Status.Current.CollectionBackupStatuses[backupIndex] = collectionBackupStatus + currentBackupStatus.CollectionBackupStatuses[backupIndex] = collectionBackupStatus } return collectionBackupStatus.Finished, err diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go index cb839bac..bff2d15d 100644 --- a/controllers/util/backup_util.go +++ b/controllers/util/backup_util.go @@ -57,20 +57,20 @@ func AsyncIdForCollectionBackup(collection string, backupName string) string { return fmt.Sprintf("%s-%s", backupName, collection) } -func UpdateStatusOfCollectionBackups(backup *solr.SolrBackup) (allFinished bool) { +func UpdateStatusOfCollectionBackups(backupStatus *solr.IndividualSolrBackupStatus) (allFinished bool) { // Check if all collection backups have been completed, this is updated in the loop - allFinished = len(backup.Status.Current.CollectionBackupStatuses) > 0 + allFinished = len(backupStatus.CollectionBackupStatuses) > 0 - allSuccessful := len(backup.Status.Current.CollectionBackupStatuses) > 0 + allSuccessful := len(backupStatus.CollectionBackupStatuses) > 0 - for _, collectionStatus := range backup.Status.Current.CollectionBackupStatuses { + for _, collectionStatus := range backupStatus.CollectionBackupStatuses { allFinished = allFinished && collectionStatus.Finished allSuccessful = allSuccessful && (collectionStatus.Successful != nil && *collectionStatus.Successful) } - backup.Status.Current.Finished = allFinished - if allFinished && backup.Status.Current.Successful == nil { - backup.Status.Current.Successful = &allSuccessful + backupStatus.Finished = allFinished + if allFinished && backupStatus.Successful == nil { + backupStatus.Successful = &allSuccessful } return } diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index 9716c085..6e2155e3 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -43,6 +43,10 @@ spec: jsonPath: .status.successful name: Successful type: boolean + - description: Next scheduled time for a recurrent backup + jsonPath: .status.nextScheduledTime + name: NextBackup + type: boolean - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -1203,8 +1207,6 @@ spec: successful: description: Whether the backup was successful type: boolean - required: - - solrVersion type: object type: array nextScheduledTime: @@ -1242,8 +1244,6 @@ spec: successful: description: Whether the backup was successful type: boolean - required: - - solrVersion type: object type: object served: true From ad7111acdd04c94207016d69cf7d697213653a30 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 15:09:59 -0500 Subject: [PATCH 05/11] Currently works. Still need to test with recurrence --- api/v1beta1/solrbackup_types.go | 2 +- controllers/solrbackup_controller.go | 49 +++++++++----------- controllers/solrcloud_controller.go | 2 +- controllers/util/backup_util.go | 46 ++++++++----------- controllers/util/solr_api/api.go | 55 +++++++++++++++++++++-- controllers/util/solr_api/node_command.go | 28 ++++++++++++ 6 files changed, 119 insertions(+), 63 deletions(-) create mode 100644 controllers/util/solr_api/node_command.go diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index c9d8a2c4..238e4fdb 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -192,7 +192,7 @@ type VolumePersistenceSource struct { // SolrBackupStatus defines the observed state of SolrBackup type SolrBackupStatus struct { // The current Backup Status, which all fields are added to this struct - Current IndividualSolrBackupStatus `json:",inline"` + IndividualSolrBackupStatus `json:",inline"` // The scheduled time for the next backup to occur // +optional diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index 8b24e0f0..795e7d3c 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -88,33 +88,33 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) return reconcile.Result{Requeue: true}, nil } - newStatus := backup.Status.DeepCopy() + oldStatus := backup.Status.DeepCopy() requeueOrNot := reconcile.Result{} var backupNeedsToWait bool // Check if we should start the next backup - if newStatus.NextScheduledTime != nil { - if newStatus.NextScheduledTime.UTC().After(time.Now().UTC()) { + if backup.Status.NextScheduledTime != nil { + if backup.Status.NextScheduledTime.UTC().After(time.Now().UTC()) { // We have hit the next scheduled restart time. backupNeedsToWait = false - newStatus.NextScheduledTime = nil + backup.Status.NextScheduledTime = nil // Add the current backup to the front of the history. // If there is no max - newStatus.History = append([]solrv1beta1.IndividualSolrBackupStatus{newStatus.Current}, newStatus.History...) + backup.Status.History = append([]solrv1beta1.IndividualSolrBackupStatus{backup.Status.IndividualSolrBackupStatus}, backup.Status.History...) // Remove history if we have too much saved - if len(newStatus.History) > backup.Spec.Recurrence.MaxSaved { - newStatus.History = newStatus.History[:backup.Spec.Recurrence.MaxSaved] + if len(backup.Status.History) > backup.Spec.Recurrence.MaxSaved { + backup.Status.History = backup.Status.History[:backup.Spec.Recurrence.MaxSaved] } // Reset Current, which is fine since it is now in the history. - newStatus.Current = solrv1beta1.IndividualSolrBackupStatus{} + backup.Status.IndividualSolrBackupStatus = solrv1beta1.IndividualSolrBackupStatus{} } else { // If we have not hit the next scheduled restart, wait to requeue until that is true. - updateRequeueAfter(&requeueOrNot, newStatus.NextScheduledTime.UTC().Sub(time.Now().UTC())) + updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.UTC().Sub(time.Now().UTC())) backupNeedsToWait = true } } else { @@ -122,20 +122,18 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Do backup work if we are not waiting and the current backup is not finished - if !backupNeedsToWait && !newStatus.Current.Finished { - requeueOrNot = reconcile.Result{} - - solrCloud, _, err1 := r.reconcileSolrCloudBackup(ctx, backup, &newStatus.Current, logger) + if !backupNeedsToWait && !backup.Status.IndividualSolrBackupStatus.Finished { + solrCloud, _, err1 := r.reconcileSolrCloudBackup(ctx, backup, &backup.Status.IndividualSolrBackupStatus, logger) if err1 != nil { // TODO Should we be failing the backup for some sub-set of errors here? logger.Error(err1, "Error while taking SolrCloud backup") // Requeue after 10 seconds for errors. - requeueOrNot = reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10} - } else if newStatus.Current.Finished { + updateRequeueAfter(&requeueOrNot, time.Second*10) + } else if backup.Status.IndividualSolrBackupStatus.Finished { // Set finish time now := metav1.Now() - newStatus.Current.FinishTime = &now + backup.Status.IndividualSolrBackupStatus.FinishTime = &now } else if solrCloud != nil { // When working with the collection backups, auto-requeue after 5 seconds // to check on the status of the async solr backup calls @@ -144,27 +142,20 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Schedule the next backupTime, if it doesn't have a next scheduled time, it has recurrence and the current backup is finished - if newStatus.NextScheduledTime == nil && backup.Spec.Recurrence != nil && newStatus.Current.Finished { - if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, newStatus.Current.FinishTime.Time); err1 != nil { + if backup.Status.NextScheduledTime == nil && backup.Spec.Recurrence != nil && backup.Status.IndividualSolrBackupStatus.Finished { + if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.IndividualSolrBackupStatus.FinishTime.Time); err1 != nil { logger.Error(err1, "Could not schedule new backup due to back schedule") } else { logger.Info("Scheduling Next Backup for time %v", nextRestartTime) convTime := metav1.NewTime(nextRestartTime) - newStatus.NextScheduledTime = &convTime + backup.Status.NextScheduledTime = &convTime + updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now())) } } - if !reflect.DeepEqual(*newStatus, backup.Status) { - backup.Status = *newStatus - logger.Info("Updating status for solr-backup", "status", backup.Status) + if !reflect.DeepEqual(*oldStatus, backup.Status) { + logger.Info("Updating status for solr-backup", "newStatus", backup.Status, "oldStatus", oldStatus) err = r.Status().Update(ctx, backup) - if err != nil { - logger.Error(err, "Could not update Status for solr-backup") - } - } - - if newStatus.NextScheduledTime != nil { - updateRequeueAfter(&requeueOrNot, newStatus.NextScheduledTime.Sub(time.Now())) } return requeueOrNot, err diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 8d0153c4..169d5212 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -103,7 +103,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( changed := instance.WithDefaults() if changed { logger.Info("Setting default settings for SolrCloud") - if err := r.Update(ctx, instance); err != nil { + if err = r.Update(ctx, instance); err != nil { return reconcile.Result{}, err } return reconcile.Result{Requeue: true}, nil diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go index bff2d15d..1d3aa779 100644 --- a/controllers/util/backup_util.go +++ b/controllers/util/backup_util.go @@ -110,45 +110,34 @@ func StartBackupForCollection(ctx context.Context, cloud *solr.SolrCloud, backup } func CheckBackupForCollection(ctx context.Context, cloud *solr.SolrCloud, collection string, backupName string, logger logr.Logger) (finished bool, success bool, asyncStatus string, err error) { - queryParams := url.Values{} - queryParams.Add("action", "REQUESTSTATUS") - queryParams.Add("requestid", AsyncIdForCollectionBackup(collection, backupName)) - - resp := &solr_api.SolrAsyncResponse{} - logger.Info("Calling to check on collection backup", "solrCloud", cloud.Name, "collection", collection) - err = solr_api.CallCollectionsApi(ctx, cloud, queryParams, resp) + + var message string + asyncStatus, message, err = solr_api.CheckAsyncRequest(ctx, cloud, AsyncIdForCollectionBackup(collection, backupName)) if err == nil { - if resp.ResponseHeader.Status == 0 { - asyncStatus = resp.Status.AsyncState - if resp.Status.AsyncState == "completed" { - finished = true - success = true - } - if resp.Status.AsyncState == "failed" { - finished = true - success = false - } + if asyncStatus == "completed" { + finished = true + success = true + } + if asyncStatus == "failed" { + finished = true + success = false } } else { - logger.Error(err, "Error checking on collection backup", "solrCloud", cloud.Name, "collection", collection) + logger.Error(err, "Error checking on collection backup", "solrCloud", cloud.Name, "collection", collection, "message", message) } return finished, success, asyncStatus, err } func DeleteAsyncInfoForBackup(ctx context.Context, cloud *solr.SolrCloud, collection string, backupName string, logger logr.Logger) (err error) { - queryParams := url.Values{} - queryParams.Add("action", "DELETESTATUS") - queryParams.Add("requestid", AsyncIdForCollectionBackup(collection, backupName)) - - resp := &solr_api.SolrAsyncResponse{} - logger.Info("Calling to delete async info for backup command.", "solrCloud", cloud.Name, "collection", collection) - err = solr_api.CallCollectionsApi(ctx, cloud, queryParams, resp) + var message string + message, err = solr_api.DeleteAsyncRequest(ctx, cloud, AsyncIdForCollectionBackup(collection, backupName)) + if err != nil { - logger.Error(err, "Error deleting async data for collection backup", "solrCloud", cloud.Name, "collection", collection) + logger.Error(err, "Error deleting async data for collection backup", "solrCloud", cloud.Name, "collection", collection, "message", message) } return err @@ -179,7 +168,7 @@ func RunExecForPod(podName string, namespace string, command []string, config *r Namespace(namespace). SubResource("exec") scheme := runtime.NewScheme() - if err := corev1.AddToScheme(scheme); err != nil { + if err = corev1.AddToScheme(scheme); err != nil { return fmt.Errorf("error adding to scheme: %v", err) } @@ -193,7 +182,8 @@ func RunExecForPod(podName string, namespace string, command []string, config *r TTY: false, }, parameterCodec) - exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + var exec remotecommand.Executor + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) if err != nil { return fmt.Errorf("error while creating Executor: %v", err) } diff --git a/controllers/util/solr_api/api.go b/controllers/util/solr_api/api.go index 5e4baa17..4ab0a070 100644 --- a/controllers/util/solr_api/api.go +++ b/controllers/util/solr_api/api.go @@ -51,10 +51,10 @@ type SolrAsyncResponse struct { ResponseHeader SolrResponseHeader `json:"responseHeader"` // +optional - RequestId string `json:"requestId"` + RequestId string `json:"requestId,omitempty"` // +optional - Status SolrAsyncStatus `json:"status"` + Status SolrAsyncStatus `json:"status,omitempty"` } type SolrResponseHeader struct { @@ -65,9 +65,56 @@ type SolrResponseHeader struct { type SolrAsyncStatus struct { // Possible states can be found here: https://github.com/apache/solr/blob/releases/lucene-solr%2F8.8.1/solr/solrj/src/java/org/apache/solr/client/solrj/response/RequestStatusState.java - AsyncState string `json:"state"` + // +optional + AsyncState string `json:"state,omitempty"` + + // +optional + Message string `json:"msg,omitempty"` +} + +type SolrAsyncStatusResponse struct { + ResponseHeader SolrResponseHeader `json:"responseHeader"` + + // +optional + Status SolrAsyncStatus `json:"status,omitempty"` +} + +type SolrDeleteRequestStatus struct { + ResponseHeader SolrResponseHeader `json:"responseHeader"` + + // Status of the delete request + // +optional + Status string `json:"status,omitempty"` +} + +func CheckAsyncRequest(ctx context.Context, cloud *solr.SolrCloud, asyncId string) (asyncState string, message string, err error) { + asyncStatus := &SolrAsyncStatusResponse{} + + queryParams := url.Values{} + queryParams.Set("action", "REQUESTSTATUS") + queryParams.Set("requestid", asyncId) + if err = CallCollectionsApi(ctx, cloud, queryParams, asyncStatus); err == nil { + if _, err = CheckForCollectionsApiError("REQUESTSTATUS", asyncStatus.ResponseHeader); err == nil { + asyncState = asyncStatus.Status.AsyncState + message = asyncStatus.Status.Message + } + } + + return +} + +func DeleteAsyncRequest(ctx context.Context, cloud *solr.SolrCloud, asyncId string) (message string, err error) { + deleteStatus := &SolrDeleteRequestStatus{} + + queryParams := url.Values{} + queryParams.Set("action", "DELETESTATUS") + queryParams.Set("requestid", asyncId) + if err = CallCollectionsApi(ctx, cloud, queryParams, deleteStatus); err == nil { + _, err = CheckForCollectionsApiError("DELETESTATUS", deleteStatus.ResponseHeader) + message = deleteStatus.Status + } - Message string `json:"msg"` + return } func CallCollectionsApi(ctx context.Context, cloud *solr.SolrCloud, urlParams url.Values, response interface{}) (err error) { diff --git a/controllers/util/solr_api/node_command.go b/controllers/util/solr_api/node_command.go new file mode 100644 index 00000000..22525202 --- /dev/null +++ b/controllers/util/solr_api/node_command.go @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package solr_api + +type SolrReplaceNodeResponse struct { + ResponseHeader SolrResponseHeader `json:"responseHeader"` + + // +optional + Success string `json:"success,omitempty"` + + // +optional + Failure string `json:"failure,omitempty"` +} \ No newline at end of file From b508128cbdd75c37d724e754895ecefa3ec8269e Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 15:33:33 -0500 Subject: [PATCH 06/11] Working with recurrence. Fixed up print columns as well. --- api/v1beta1/solrbackup_types.go | 7 ++++--- config/crd/bases/solr.apache.org_solrbackups.yaml | 11 ++++++++--- controllers/solrbackup_controller.go | 8 ++++---- helm/solr-operator/crds/crds.yaml | 11 ++++++++--- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index 238e4fdb..d700afd4 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -320,9 +320,10 @@ func (sb *SolrBackup) PersistenceJobName() string { //+kubebuilder:categories=all //+kubebuilder:subresource:status //+kubebuilder:printcolumn:name="Cloud",type="string",JSONPath=".spec.solrCloud",description="Solr Cloud" -//+kubebuilder:printcolumn:name="Finished",type="boolean",JSONPath=".status.finished",description="Whether the backup has finished" -//+kubebuilder:printcolumn:name="Successful",type="boolean",JSONPath=".status.successful",description="Whether the backup was successful" -//+kubebuilder:printcolumn:name="NextBackup",type="boolean",JSONPath=".status.nextScheduledTime",description="Next scheduled time for a recurrent backup" +//+kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Most recent time the backup started" +//+kubebuilder:printcolumn:name="Finished",type="boolean",JSONPath=".status.finished",description="Whether the most recent backup has finished" +//+kubebuilder:printcolumn:name="Successful",type="boolean",JSONPath=".status.successful",description="Whether the most recent backup was successful" +//+kubebuilder:printcolumn:name="NextBackup",type="string",JSONPath=".status.nextScheduledTime",description="Next scheduled time for a recurrent backup",format="date-time" //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // SolrBackup is the Schema for the solrbackups API diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index fc313b47..d7005e7d 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -35,18 +35,23 @@ spec: jsonPath: .spec.solrCloud name: Cloud type: string - - description: Whether the backup has finished + - description: Most recent time the backup started + jsonPath: .status.startTimestamp + name: Started + type: date + - description: Whether the most recent backup has finished jsonPath: .status.finished name: Finished type: boolean - - description: Whether the backup was successful + - description: Whether the most recent backup was successful jsonPath: .status.successful name: Successful type: boolean - description: Next scheduled time for a recurrent backup + format: date-time jsonPath: .status.nextScheduledTime name: NextBackup - type: boolean + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index 795e7d3c..022a5e9f 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -96,7 +96,7 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Check if we should start the next backup if backup.Status.NextScheduledTime != nil { - if backup.Status.NextScheduledTime.UTC().After(time.Now().UTC()) { + if backup.Status.NextScheduledTime.UTC().Before(time.Now().UTC()) { // We have hit the next scheduled restart time. backupNeedsToWait = false backup.Status.NextScheduledTime = nil @@ -143,11 +143,11 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Schedule the next backupTime, if it doesn't have a next scheduled time, it has recurrence and the current backup is finished if backup.Status.NextScheduledTime == nil && backup.Spec.Recurrence != nil && backup.Status.IndividualSolrBackupStatus.Finished { - if nextRestartTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.IndividualSolrBackupStatus.FinishTime.Time); err1 != nil { + if nextBackupTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.IndividualSolrBackupStatus.StartTime.Time); err1 != nil { logger.Error(err1, "Could not schedule new backup due to back schedule") } else { - logger.Info("Scheduling Next Backup for time %v", nextRestartTime) - convTime := metav1.NewTime(nextRestartTime) + logger.Info("Scheduling Next Backup", "time", nextBackupTime) + convTime := metav1.NewTime(nextBackupTime) backup.Status.NextScheduledTime = &convTime updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now())) } diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index 6e2155e3..6735d865 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -35,18 +35,23 @@ spec: jsonPath: .spec.solrCloud name: Cloud type: string - - description: Whether the backup has finished + - description: Most recent time the backup started + jsonPath: .status.startTimestamp + name: Started + type: date + - description: Whether the most recent backup has finished jsonPath: .status.finished name: Finished type: boolean - - description: Whether the backup was successful + - description: Whether the most recent backup was successful jsonPath: .status.successful name: Successful type: boolean - description: Next scheduled time for a recurrent backup + format: date-time jsonPath: .status.nextScheduledTime name: NextBackup - type: boolean + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date From ae318686327a743ff758b69b924c61147eb833c0 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 15:36:35 -0500 Subject: [PATCH 07/11] Formatting and generation. --- api/v1beta1/zz_generated.deepcopy.go | 2 +- controllers/util/solr_api/node_command.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index c2c3a3f7..4e3450ba 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -880,7 +880,7 @@ func (in *SolrBackupSpec) DeepCopy() *SolrBackupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SolrBackupStatus) DeepCopyInto(out *SolrBackupStatus) { *out = *in - in.Current.DeepCopyInto(&out.Current) + in.IndividualSolrBackupStatus.DeepCopyInto(&out.IndividualSolrBackupStatus) if in.NextScheduledTime != nil { in, out := &in.NextScheduledTime, &out.NextScheduledTime *out = (*in).DeepCopy() diff --git a/controllers/util/solr_api/node_command.go b/controllers/util/solr_api/node_command.go index 22525202..d98336b5 100644 --- a/controllers/util/solr_api/node_command.go +++ b/controllers/util/solr_api/node_command.go @@ -25,4 +25,4 @@ type SolrReplaceNodeResponse struct { // +optional Failure string `json:"failure,omitempty"` -} \ No newline at end of file +} From c5b92d4438f885216e13ebd1999b12fa8c2e0788 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 15:41:54 -0500 Subject: [PATCH 08/11] Extraneous logging bugs. --- controllers/solrbackup_controller.go | 2 +- controllers/solrcloud_controller.go | 2 +- controllers/solrprometheusexporter_controller.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index 022a5e9f..a914a78d 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -144,7 +144,7 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Schedule the next backupTime, if it doesn't have a next scheduled time, it has recurrence and the current backup is finished if backup.Status.NextScheduledTime == nil && backup.Spec.Recurrence != nil && backup.Status.IndividualSolrBackupStatus.Finished { if nextBackupTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.IndividualSolrBackupStatus.StartTime.Time); err1 != nil { - logger.Error(err1, "Could not schedule new backup due to back schedule") + logger.Error(err1, "Could not schedule new backup due to bad cron schedule", "cron", backup.Spec.Recurrence.Schedule) } else { logger.Info("Scheduling Next Backup", "time", nextBackupTime) convTime := metav1.NewTime(nextBackupTime) diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 169d5212..06748258 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -323,7 +323,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Set the annotation for a scheduled restart, if necessary. if nextRestartAnnotation, reconcileWaitDuration, err := util.ScheduleNextRestart(instance.Spec.UpdateStrategy.RestartSchedule, foundStatefulSet.Spec.Template.Annotations); err != nil { - logger.Error(err, "Cannot parse restartSchedule cron: %s", instance.Spec.UpdateStrategy.RestartSchedule) + logger.Error(err, "Cannot parse restartSchedule cron", "cron", instance.Spec.UpdateStrategy.RestartSchedule) } else { if nextRestartAnnotation != "" { // Set the new restart time annotation diff --git a/controllers/solrprometheusexporter_controller.go b/controllers/solrprometheusexporter_controller.go index 6902df10..ae8f53aa 100644 --- a/controllers/solrprometheusexporter_controller.go +++ b/controllers/solrprometheusexporter_controller.go @@ -217,7 +217,7 @@ func (r *SolrPrometheusExporterReconciler) Reconcile(ctx context.Context, req ct // Set the annotation for a scheduled restart, if necessary. if nextRestartAnnotation, reconcileWaitDuration, err := util.ScheduleNextRestart(prometheusExporter.Spec.RestartSchedule, foundDeploy.Spec.Template.Annotations); err != nil { - logger.Error(err, "Cannot parse restartSchedule cron: %s", prometheusExporter.Spec.RestartSchedule) + logger.Error(err, "Cannot parse restartSchedule cron", "cron", prometheusExporter.Spec.RestartSchedule) } else { if nextRestartAnnotation != "" { if deploy.Spec.Template.Annotations == nil { From d919a8d1c3f9c48ebc8cbb79725894166475083d Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 15:53:31 -0500 Subject: [PATCH 09/11] Add ability to remove schedule before next run --- controllers/solrbackup_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index a914a78d..6b5d9c35 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -96,7 +96,11 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Check if we should start the next backup if backup.Status.NextScheduledTime != nil { - if backup.Status.NextScheduledTime.UTC().Before(time.Now().UTC()) { + // If the backup no longer has a recurrence specified, remove the next scheduled time + if backup.Spec.Recurrence == nil { + backup.Status.NextScheduledTime = nil + backupNeedsToWait = false + } else if backup.Status.NextScheduledTime.UTC().Before(time.Now().UTC()) { // We have hit the next scheduled restart time. backupNeedsToWait = false backup.Status.NextScheduledTime = nil From 8b92e476ba944a9ab1d42fc4a43da249916e9dee Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 8 Nov 2021 18:06:53 -0500 Subject: [PATCH 10/11] Add changelog entry and documentation. --- docs/solr-backup/README.md | 49 ++++++++++++++++++++++++++++++++--- helm/solr-operator/Chart.yaml | 11 +++++++- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/solr-backup/README.md b/docs/solr-backup/README.md index d5fa6041..0fed2e0a 100644 --- a/docs/solr-backup/README.md +++ b/docs/solr-backup/README.md @@ -28,6 +28,7 @@ For detailed information on how to best configure backups for your use case, ple This page outlines how to create and delete a Kubernetes SolrBackup - [Creation](#creating-an-example-solrbackup) +- [Recurring/Scheduled Backups](#recurring-backups) - [Deletion](#deleting-an-example-solrbackup) - [Repository Types](#supported-repository-types) - [GCS](#gcs-backup-repositories) @@ -99,17 +100,59 @@ The status of our triggered backup can be checked with the command below. ```bash $ kubectl get solrbackups -NAME CLOUD FINISHED SUCCESSFUL AGE -local-backup example true true 72s +NAME CLOUD STARTED FINISHED SUCCESSFUL NEXTBACKUP AGE +test example 123m true false 161m ``` +## Recurring Backups +_Since v0.5.0_ + +The Solr Operator enables taking recurring updates, at a set interval. +Note that this feature requires a SolrCloud running Solr `8.9.0` or older, because it relies on `Incremental` backups. + +By default the Solr Operator will save a maximum of **5** backups at a time, however users can override this using `SolrBackup.spec.recurrence.maxSaved`. +When using `recurrence`, users must provide a Cron-style `schedule` for the interval at which backups should be taken. +Please refer to the [GoLang cron-spec](https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format) for more information on allowed syntax. + +```yaml +apiVersion: solr.apache.org/v1beta1 +kind: SolrBackup +metadata: + name: local-backup + namespace: default +spec: + repositoryName: "local-collection-backups-1" + solrCloud: example + collections: + - techproducts + - books + recurrence: # Store one backup daily, and keep a week at a time. + schedule: "@daily" + maxSaved: 7 +``` + +If using `kubectl`, the standard `get` command will return the time the backup was last started and when the next backup will occur. + +```bash +$ kubectl get solrbackups +NAME CLOUD STARTED FINISHED SUCCESSFUL NEXTBACKUP AGE +test example 123m true true 2021-11-09T00:00:00Z 161m +``` + +Much like when not taking a recurring backup, `SolrBackup.status` will contain the information from the latest, or currently running, backup. +The results of previous backup attempts are stored under `SolrBackup.status.history` (sorted from most recent to oldest). + +You are able to **add or remove** `recurrence` to/from an existing `SolrBackup` object, no matter what stage that `SolrBackup` object is in. +If you add recurrence, then a new backup will be scheduled based on the `startTimestamp` of the last backup. +If you remove recurrence, then the `nextBackupTime` will be removed. +However, if the recurrent backup is already underway, it will not be stopped. + ## Deleting an example SolrBackup Once the operator completes a backup, the SolrBackup instance can be safely deleted. ```bash $ kubectl delete solrbackup local-backup -TODO command output ``` Note that deleting SolrBackup instances doesn't delete the backed up data, which the operator views as already persisted and outside its control. diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml index 257cb628..71f96891 100644 --- a/helm/solr-operator/Chart.yaml +++ b/helm/solr-operator/Chart.yaml @@ -41,7 +41,7 @@ dependencies: condition: zookeeper-operator.install annotations: artifacthub.io/operator: "true" - artifacthub.io/operatorCapabilities: Seamless Upgrades + artifacthub.io/operatorCapabilities: Full Lifecycle artifacthub.io/prerelease: "true" artifacthub.io/recommendations: | - url: https://artifacthub.io/packages/helm/apache-solr/solr @@ -183,6 +183,15 @@ annotations: url: https://github.com/apache/solr-operator/issues/326 - name: Github PR url: https://github.com/apache/solr-operator/pull/358 + - kind: added + description: Scheduled/Recurring SolrBackup support + links: + - name: Github Issue + url: https://github.com/apache/solr-operator/issues/303 + - name: Github PR + url: https://github.com/apache/solr-operator/pull/359 + - name: SolrBackup Documentation + url: https://apache.github.io/solr-operator/docs/solr-backup#recurring-backups artifacthub.io/images: | - name: solr-operator image: apache/solr-operator:v0.5.0-prerelease From 3544a15120c3e54a2f41be93b65a8c71e013efe6 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 9 Nov 2021 11:16:24 -0500 Subject: [PATCH 11/11] Add disabled flag for recurrence. --- api/v1beta1/solrbackup_types.go | 10 +++++ .../bases/solr.apache.org_solrbackups.yaml | 4 ++ controllers/solrbackup_controller.go | 6 +-- controllers/util/backup_util.go | 2 +- docs/solr-backup/README.md | 38 +++++++++++++++++++ helm/solr-operator/crds/crds.yaml | 4 ++ 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go index d700afd4..26652117 100644 --- a/api/v1beta1/solrbackup_types.go +++ b/api/v1beta1/solrbackup_types.go @@ -94,6 +94,16 @@ type BackupRecurrence struct { // +kubebuilder:validation:Minimum:=1 // +optional MaxSaved int `json:"maxSaved,omitempty"` + + // Disable the recurring backups. Note this will not affect any currently-running backup. + // + // +kubebuilder:default:=false + // +optional + Disabled bool `json:"disabled,omitempty"` +} + +func (recurrence *BackupRecurrence) IsEnabled() bool { + return recurrence != nil && !recurrence.Disabled } // PersistenceSource defines the location and method of persisting the backup data. diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml index d7005e7d..aa640fc8 100644 --- a/config/crd/bases/solr.apache.org_solrbackups.yaml +++ b/config/crd/bases/solr.apache.org_solrbackups.yaml @@ -1063,6 +1063,10 @@ spec: recurrence: description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API." properties: + disabled: + default: false + description: Disable the recurring backups. Note this will not affect any currently-running backup. + type: boolean maxSaved: default: 5 description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 5. diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go index 6b5d9c35..c5905c10 100644 --- a/controllers/solrbackup_controller.go +++ b/controllers/solrbackup_controller.go @@ -96,8 +96,8 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Check if we should start the next backup if backup.Status.NextScheduledTime != nil { - // If the backup no longer has a recurrence specified, remove the next scheduled time - if backup.Spec.Recurrence == nil { + // If the backup no longer enabled, remove the next scheduled time + if !backup.Spec.Recurrence.IsEnabled() { backup.Status.NextScheduledTime = nil backupNeedsToWait = false } else if backup.Status.NextScheduledTime.UTC().Before(time.Now().UTC()) { @@ -146,7 +146,7 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Schedule the next backupTime, if it doesn't have a next scheduled time, it has recurrence and the current backup is finished - if backup.Status.NextScheduledTime == nil && backup.Spec.Recurrence != nil && backup.Status.IndividualSolrBackupStatus.Finished { + if backup.Status.NextScheduledTime == nil && backup.Spec.Recurrence.IsEnabled() && backup.Status.IndividualSolrBackupStatus.Finished { if nextBackupTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.IndividualSolrBackupStatus.StartTime.Time); err1 != nil { logger.Error(err1, "Could not schedule new backup due to bad cron schedule", "cron", backup.Spec.Recurrence.Schedule) } else { diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go index 1d3aa779..ac891294 100644 --- a/controllers/util/backup_util.go +++ b/controllers/util/backup_util.go @@ -84,7 +84,7 @@ func GenerateQueryParamsForBackup(backupRepository *solr.SolrBackupRepository, b queryParams.Add("location", BackupLocationPath(backupRepository, backup.Spec.Location)) queryParams.Add("repository", backup.Spec.RepositoryName) - if backup.Spec.Recurrence != nil { + if backup.Spec.Recurrence.IsEnabled() { queryParams.Add("maxNumBackupPoints", strconv.Itoa(backup.Spec.Recurrence.MaxSaved)) } diff --git a/docs/solr-backup/README.md b/docs/solr-backup/README.md index 0fed2e0a..23c9d3da 100644 --- a/docs/solr-backup/README.md +++ b/docs/solr-backup/README.md @@ -147,6 +147,44 @@ If you add recurrence, then a new backup will be scheduled based on the `startTi If you remove recurrence, then the `nextBackupTime` will be removed. However, if the recurrent backup is already underway, it will not be stopped. +### Backup Scheduling + +Backups are scheduled based on the `startTimestamp` of the last backup. +Therefore if a interval schedule such as `@every 1h` is used, and a backup starts on `2021-11-09T03:10:00Z` and ends on `2021-11-09T05:30:00Z`, then the next backup will be started at `2021-11-09T04:10:00Z`. +If the interval is shorter than the time it takes to complete a backup, then the next backup will started directly after the previous backup completes (even though it is delayed from its given schedule). +And the next backup will be scheduled based on the `startTimestamp` of the delayed backup. +So there is a possibility of skew overtime if backups take longer than the allotted schedule. + +If a guaranteed schedule is important, it is recommended to use intervals that are guaranteed to be longer than the time it takes to complete a backup. + +### Temporarily Disabling Recurring Backups + +It is also easy to temporarily disable backups for a time. +Merely add `disabled: true` under the `recurrence` section of the `SolrBackup` resource. +And set `disabled: false`, or just remove the property to re-enable backups. + +Since backups are scheduled based on the `startTimestamp` of the last backup, a new backup may start immediately after you re-enable the recurrence. + +```yaml +apiVersion: solr.apache.org/v1beta1 +kind: SolrBackup +metadata: + name: local-backup + namespace: default +spec: + repositoryName: "local-collection-backups-1" + solrCloud: example + collections: + - techproducts + - books + recurrence: # Store one backup daily, and keep a week at a time. + schedule: "@daily" + maxSaved: 7 + disabled: true +``` + +**Note: this will not stop any backups running at the time that `disabled: true` is set, it will only affect scheduling future backups.** + ## Deleting an example SolrBackup Once the operator completes a backup, the SolrBackup instance can be safely deleted. diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml index 6735d865..7cd6c99f 100644 --- a/helm/solr-operator/crds/crds.yaml +++ b/helm/solr-operator/crds/crds.yaml @@ -1063,6 +1063,10 @@ spec: recurrence: description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API." properties: + disabled: + default: false + description: Disable the recurring backups. Note this will not affect any currently-running backup. + type: boolean maxSaved: default: 5 description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 5.