Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow backup storage locations to specify backup sync period or toggle off sync #1936

Merged
merged 2 commits into from
Oct 24, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelogs/unreleased/1936-betta1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Allow backup storage locations to specify backup sync period or toggle off sync
5 changes: 5 additions & 0 deletions pkg/apis/velero/v1/backup_storage_location.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ type BackupStorageLocationSpec struct {
// AccessMode defines the permissions for the backup storage location.
// +optional
AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"`

// BackupSyncPeriod defines how frequently to sync backup API objects from object storage. A value of 0 disables sync.
// +optional
// +nullable
betta1 marked this conversation as resolved.
Show resolved Hide resolved
BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"`
}

// BackupStorageLocationPhase is the lifecyle phase of a Velero BackupStorageLocation.
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/velero/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

32 changes: 23 additions & 9 deletions pkg/cmd/cli/backuplocation/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package backuplocation
import (
"fmt"
"strings"
"time"

"github.com/pkg/errors"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -54,13 +55,14 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command {
}

type CreateOptions struct {
Name string
Provider string
Bucket string
Prefix string
Config flag.Map
Labels flag.Map
AccessMode *flag.Enum
Name string
Provider string
Bucket string
Prefix string
BackupSyncPeriod time.Duration
Config flag.Map
Labels flag.Map
AccessMode *flag.Enum
}

func NewCreateOptions() *CreateOptions {
Expand All @@ -78,6 +80,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
flags.StringVar(&o.Provider, "provider", o.Provider, "name of the backup storage provider (e.g. aws, azure, gcp)")
flags.StringVar(&o.Bucket, "bucket", o.Bucket, "name of the object storage bucket where backups should be stored")
flags.StringVar(&o.Prefix, "prefix", o.Prefix, "prefix under which all Velero data should be stored within the bucket. Optional.")
flags.DurationVar(&o.BackupSyncPeriod, "backup-sync-period", o.BackupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to `0s` to disable sync")
flags.Var(&o.Config, "config", "configuration key-value pairs")
flags.Var(&o.Labels, "labels", "labels to apply to the backup storage location")
flags.Var(
Expand All @@ -100,6 +103,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto
return errors.New("--bucket is required")
}

if o.BackupSyncPeriod < 0 {
return errors.New("--backup-sync-period must be non-negative")
}

return nil
}

Expand All @@ -109,6 +116,12 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error {
}

func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
var backupSyncPeriod *metav1.Duration

if c.Flags().Changed("backup-sync-period") {
backupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod}
}

backupStorageLocation := &velerov1api.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Namespace: f.Namespace(),
Expand All @@ -123,8 +136,9 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
Prefix: o.Prefix,
},
},
Config: o.Config.Data(),
AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()),
Config: o.Config.Data(),
AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()),
BackupSyncPeriod: backupSyncPeriod,
},
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/cmd/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ func NewCommand(f client.Factory) *cobra.Command {
command.Flags().Var(config.formatFlag, "log-format", fmt.Sprintf("the format for log output. Valid values are %s.", strings.Join(config.formatFlag.AllowedValues(), ", ")))
command.Flags().StringVar(&config.pluginDir, "plugin-dir", config.pluginDir, "directory containing Velero plugins")
command.Flags().StringVar(&config.metricsAddress, "metrics-address", config.metricsAddress, "the address to expose prometheus metrics")
command.Flags().DurationVar(&config.backupSyncPeriod, "backup-sync-period", config.backupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster")
command.Flags().DurationVar(&config.backupSyncPeriod, "backup-sync-period", config.backupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. This is the default sync period if none is explicitly specified for a backup storage location.")
command.Flags().DurationVar(&config.podVolumeOperationTimeout, "restic-timeout", config.podVolumeOperationTimeout, "how long backups/restores of pod volumes should be allowed to run before timing out")
command.Flags().BoolVar(&config.restoreOnly, "restore-only", config.restoreOnly, "run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled. DEPRECATED: this flag will be removed in v2.0. Use read-only backup storage locations instead.")
command.Flags().StringSliceVar(&config.disabledControllers, "disable-controllers", config.disabledControllers, fmt.Sprintf("list of controllers to disable on startup. Valid values are %s", strings.Join(disableControllerList, ",")))
Expand Down
28 changes: 27 additions & 1 deletion pkg/controller/backup_sync_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ type backupSyncController struct {
podVolumeBackupLister listers.PodVolumeBackupLister
namespace string
defaultBackupLocation string
defaultBackupSyncPeriod time.Duration
newPluginManager func(logrus.FieldLogger) clientmgmt.Manager
newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error)
}
Expand Down Expand Up @@ -77,6 +78,7 @@ func NewBackupSyncController(
podVolumeBackupClient: podVolumeBackupClient,
namespace: namespace,
defaultBackupLocation: defaultBackupLocation,
defaultBackupSyncPeriod: syncPeriod,
backupLister: backupInformer.Lister(),
backupStorageLocationLister: backupStorageLocationInformer.Lister(),
podVolumeBackupLister: podVolumeBackupInformer.Lister(),
Expand All @@ -88,7 +90,7 @@ func NewBackupSyncController(
}

c.resyncFunc = c.run
c.resyncPeriod = syncPeriod
c.resyncPeriod = 30 * time.Second
c.cacheSyncWaiters = []cache.InformerSynced{
backupInformer.Informer().HasSynced,
backupStorageLocationInformer.Informer().HasSynced,
Expand Down Expand Up @@ -134,6 +136,30 @@ func (c *backupSyncController) run() {

for _, location := range locations {
log := c.logger.WithField("backupLocation", location.Name)

syncPeriod := c.defaultBackupSyncPeriod
if location.Spec.BackupSyncPeriod != nil {
syncPeriod = location.Spec.BackupSyncPeriod.Duration
if syncPeriod == 0 {
log.Debug("Backup sync period for this location is set to 0, skipping sync")
continue
}

if syncPeriod < 0 {
betta1 marked this conversation as resolved.
Show resolved Hide resolved
log.Debug("Backup sync period must be non-negative")
syncPeriod = c.defaultBackupSyncPeriod
}
}

lastSync := location.Status.LastSyncedTime
if lastSync != nil {
log.Debug("Checking if backups need to be synced at this time for this location")
nextSync := lastSync.Add(syncPeriod)
if time.Now().UTC().Before(nextSync) {
continue
}
}

log.Debug("Checking backup location for backups to sync into cluster")

backupStore, err := c.newBackupStore(location, pluginManager, log)
Expand Down
Loading