diff --git a/pkg/cmd/roachprod/flags.go b/pkg/cmd/roachprod/flags.go index 09319c6bd66e..8cb9db876ef2 100644 --- a/pkg/cmd/roachprod/flags.go +++ b/pkg/cmd/roachprod/flags.go @@ -189,6 +189,13 @@ func initFlags() { "skip-init", startOpts.SkipInit, "skip initializing the cluster") startCmd.Flags().IntVar(&startOpts.StoreCount, "store-count", startOpts.StoreCount, "number of stores to start each node with") + startCmd.Flags().BoolVar(&startOpts.ScheduleBackups, + "schedule-backups", startOpts.ScheduleBackups, + "create a cluster backup schedule once the cluster has started (by default, "+ + "full backup hourly and incremental every 15 minutes)") + startCmd.Flags().StringVar(&startOpts.ScheduleBackupArgs, "schedule-backup-args", "", + `Recurrence and scheduled backup options specification. +Default is "RECURRING '*/15 * * * *' FULL BACKUP '@hourly' WITH SCHEDULE OPTIONS first_run = 'now'"`) startTenantCmd.Flags().StringVarP(&hostCluster, "host-cluster", "H", "", "host cluster") diff --git a/pkg/roachprod/install/cockroach.go b/pkg/roachprod/install/cockroach.go index 5852d7495ebe..5be1ce3835e9 100644 --- a/pkg/roachprod/install/cockroach.go +++ b/pkg/roachprod/install/cockroach.go @@ -26,6 +26,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/config" "github.com/cockroachdb/cockroach/pkg/roachprod/logger" "github.com/cockroachdb/cockroach/pkg/roachprod/ssh" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" ) @@ -87,6 +88,10 @@ type StartOpts struct { Sequential bool ExtraArgs []string + // ScheduleBackups starts a backup schedule once the cluster starts + ScheduleBackups bool + ScheduleBackupArgs string + // systemd limits on resources. NumFilesLimit int64 @@ -152,7 +157,7 @@ func (c *SyncedCluster) Start(ctx context.Context, l *logger.Logger, startOpts S } l.Printf("%s: starting nodes", c.Name) - return c.Parallel(l, "", len(nodes), parallelism, func(nodeIdx int) ([]byte, error) { + if err := c.Parallel(l, "", len(nodes), parallelism, func(nodeIdx int) ([]byte, error) { node := nodes[nodeIdx] // NB: if cockroach started successfully, we ignore the output as it is @@ -193,7 +198,13 @@ func (c *SyncedCluster) Start(ctx context.Context, l *logger.Logger, startOpts S return nil, errors.Wrap(err, "failed to set cluster settings") } return nil, nil - }) + }); err != nil { + return err + } + if startOpts.ScheduleBackups { + return c.createFixedBackupSchedule(ctx, l, startOpts.ScheduleBackupArgs) + } + return nil } // NodeDir returns the data directory for the given node and store. @@ -744,6 +755,40 @@ func (c *SyncedCluster) shouldAdvertisePublicIP() bool { return false } +// createFixedBackupSchedule creates a cluster backup schedule which, by +// default, runs an incremental every 15 minutes and a full every hour. On +// `roachprod create`, the user can provide a different recurrence using the +// 'schedule-backup-args' flag. If roachprod is local, the backups get stored in +// nodelocal, and otherwise in 'gs://cockroachdb-backup-testing'. +func (c *SyncedCluster) createFixedBackupSchedule( + ctx context.Context, l *logger.Logger, scheduledBackupArgs string, +) error { + externalStoragePath := `gs://cockroachdb-backup-testing` + + if c.IsLocal() { + externalStoragePath = `nodelocal://1` + } + l.Printf("%s: creating backup schedule", c.Name) + + collectionPath := fmt.Sprintf(`%s/roachprod-scheduled-backups/%s/%v`, + externalStoragePath, c.Name, timeutil.Now().UnixNano()) + + // Default scheduled backup runs a full backup every hour and an incremental + // every 15 minutes. + scheduleArgs := `RECURRING '*/15 * * * *' +FULL BACKUP '@hourly' +WITH SCHEDULE OPTIONS first_run = 'now'` + + if scheduledBackupArgs != "" { + scheduleArgs = scheduledBackupArgs + } + + createScheduleCmd := fmt.Sprintf(`-e +CREATE SCHEDULE IF NOT EXISTS test_only_backup FOR BACKUP INTO '%s' %s`, + collectionPath, scheduleArgs) + return c.SQL(ctx, l, []string{createScheduleCmd}) +} + // getEnvVars returns all COCKROACH_* environment variables, in the form // "key=value". func getEnvVars() []string { diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index 9fd4fd0970e5..3fd970ceb0b3 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -635,12 +635,14 @@ func Extend(l *logger.Logger, clusterName string, lifetime time.Duration) error // DefaultStartOpts returns a StartOpts populated with default values. func DefaultStartOpts() install.StartOpts { return install.StartOpts{ - Sequential: true, - EncryptedStores: false, - NumFilesLimit: config.DefaultNumFilesLimit, - SkipInit: false, - StoreCount: 1, - TenantID: 2, + Sequential: true, + EncryptedStores: false, + NumFilesLimit: config.DefaultNumFilesLimit, + SkipInit: false, + StoreCount: 1, + TenantID: 2, + ScheduleBackups: false, + ScheduleBackupArgs: "", } }