From d9a739d994ddfaf3f87bc63f9866545e851ea1c1 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Fri, 14 Dec 2018 16:32:05 -0800 Subject: [PATCH 01/19] backup-operator: Add BackupIntervalInSecond This commit added BackupIntervalInSecond in BackupPolicy, which perform periodic backup as #1841 issue describe. This commit is part of #1841. By specifying BackupIntervalInSecond, user can let etcd-backup-operator do periodic backup. The specification of BackupIntervalInSecond is following - unit in sec - >0 means interval. - =0 means explicit disable interval backup, it will just do one time backup. --- Gopkg.lock | 10 ++ pkg/apis/etcd/v1beta2/backup_types.go | 9 +- pkg/backup/backup_manager.go | 4 +- pkg/controller/backup-operator/operator.go | 8 ++ pkg/controller/backup-operator/sync.go | 139 ++++++++++++++++++++- pkg/controller/backup-operator/util.go | 9 ++ 6 files changed, 171 insertions(+), 8 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 933a1b1f0..1578e35b3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -974,6 +974,14 @@ pruneopts = "NT" revision = "0317810137be915b9cf888946c6e115c1bfac693" +[[projects]] + digest = "1:6209fb6d1debd999cd64e943b14314a3ce665424d05369b297096df3992ce688" + name = "k8s.io/kubernetes" + packages = ["pkg/util/slice"] + pruneopts = "NT" + revision = "eec55b9ba98609a46fee712359c7b5b365bdd920" + version = "v1.13.1" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 @@ -1001,6 +1009,7 @@ "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset", "k8s.io/apimachinery/pkg/api/errors", + "k8s.io/apimachinery/pkg/api/meta", "k8s.io/apimachinery/pkg/api/resource", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/apimachinery/pkg/fields", @@ -1038,6 +1047,7 @@ "k8s.io/code-generator/cmd/lister-gen", "k8s.io/code-generator/cmd/openapi-gen", "k8s.io/gengo/args", + "k8s.io/kubernetes/pkg/util/slice", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/pkg/apis/etcd/v1beta2/backup_types.go b/pkg/apis/etcd/v1beta2/backup_types.go index 914f34cb5..93e4c6b59 100644 --- a/pkg/apis/etcd/v1beta2/backup_types.go +++ b/pkg/apis/etcd/v1beta2/backup_types.go @@ -14,7 +14,11 @@ package v1beta2 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) const ( // AWS S3 related consts @@ -94,6 +98,9 @@ type BackupSource struct { type BackupPolicy struct { // TimeoutInSecond is the maximal allowed time in second of the entire backup process. TimeoutInSecond int64 `json:"timeoutInSecond,omitempty"` + // BackupIntervalInSecond is to specify how often operator take snapshot + // 0 is magic number to indicate one-shot backup + BackupIntervalInSecond int64 `json:"backupIntervalInSecond,omitempty"` } // BackupStatus represents the status of the EtcdBackup Custom Resource. diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index e5e6350a9..89eac4392 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "fmt" + "time" "github.com/coreos/etcd-operator/pkg/backup/writer" "github.com/coreos/etcd-operator/pkg/util/constants" @@ -68,7 +69,8 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string) (int64, st return 0, "", fmt.Errorf("failed to receive snapshot (%v)", err) } defer rc.Close() - + s3Path = fmt.Sprintf(s3Path+"_v%d_%s", + rev, time.Now().UTC().Format("2006-01-02-15:04:05")) _, err = bm.bw.Write(ctx, s3Path, rc) if err != nil { return 0, "", fmt.Errorf("failed to write snapshot (%v)", err) diff --git a/pkg/controller/backup-operator/operator.go b/pkg/controller/backup-operator/operator.go index 1a4476160..6c34c25b7 100644 --- a/pkg/controller/backup-operator/operator.go +++ b/pkg/controller/backup-operator/operator.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "os" + "sync" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/client" @@ -45,9 +46,16 @@ type Backup struct { backupCRCli versioned.Interface kubeExtCli apiextensionsclient.Interface + backupRunnerStore sync.Map + createCRD bool } +type BackupRunner struct { + spec api.BackupSpec + cancelFunc context.CancelFunc +} + // New creates a backup operator. func New(createCRD bool) *Backup { return &Backup{ diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index abc4b4caf..d59c78d4d 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -17,12 +17,17 @@ package controller import ( "context" "errors" + "reflect" "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/util/constants" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/util/slice" ) const ( @@ -66,17 +71,134 @@ func (b *Backup) processItem(key string) error { } eb := obj.(*api.EtcdBackup) + + if eb.DeletionTimestamp != nil { + b.deletePeriodicBackupRunner(eb.ObjectMeta.UID) + err := b.removeFinalizerOfPeriodicBackup(eb) + if err != nil { + return err + } + return nil + } // don't process the CR if it has a status since // having a status means that the backup is either made or failed. - if eb.Status.Succeeded || len(eb.Status.Reason) != 0 { + if !isPeriodicBackup(eb) && + (eb.Status.Succeeded || len(eb.Status.Reason) != 0) { return nil } - bs, err := b.handleBackup(&eb.Spec) - // Report backup status - b.reportBackupStatus(bs, err, eb) + + if isPeriodicBackup(eb) && b.isChanged(eb) { + // Stop previous backup runner if it exists + b.deletePeriodicBackupRunner(eb.ObjectMeta.UID) + + // Add finalizer if need + eb, err = b.addFinalizerOfPeriodicBackupIfNeed(eb) + if err != nil { + return err + } + + // Run new backup runner + ticker := time.NewTicker( + time.Duration(eb.Spec.BackupPolicy.BackupIntervalInSecond) * time.Second) + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + go b.periodicRunnerFunc(ctx, ticker, eb) + + // Store cancel function for periodic + b.backupRunnerStore.Store(eb.ObjectMeta.UID, BackupRunner{eb.Spec, cancel}) + + } else if !isPeriodicBackup(eb) { + // Perform backup + bs, err := b.handleBackup(nil, &eb.Spec) + // Report backup status + b.reportBackupStatus(bs, err, eb) + } return err } +func (b *Backup) isChanged(eb *api.EtcdBackup) bool { + backupRunner, exists := b.backupRunnerStore.Load(eb.ObjectMeta.UID) + if !exists { + return true + } + return !reflect.DeepEqual(eb.Spec, backupRunner.(BackupRunner).spec) +} + +func (b *Backup) deletePeriodicBackupRunner(uid types.UID) bool { + backupRunner, exists := b.backupRunnerStore.Load(uid) + if exists { + backupRunner.(BackupRunner).cancelFunc() + b.backupRunnerStore.Delete(uid) + return true + } + return false +} + +func (b *Backup) addFinalizerOfPeriodicBackupIfNeed(eb *api.EtcdBackup) (*api.EtcdBackup, error) { + ebNew := eb.DeepCopyObject() + metadata, err := meta.Accessor(ebNew) + if err != nil { + return eb, err + } + if !slice.ContainsString(metadata.GetFinalizers(), "backup-operator-periodic", nil) { + metadata.SetFinalizers(append(metadata.GetFinalizers(), "backup-operator-periodic")) + _, err := b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Update(ebNew.(*api.EtcdBackup)) + if err != nil { + return eb, err + } + } else { + return eb, nil + } + return ebNew.(*api.EtcdBackup), nil +} + +func (b *Backup) removeFinalizerOfPeriodicBackup(eb *api.EtcdBackup) error { + ebNew := eb.DeepCopyObject() + metadata, err := meta.Accessor(ebNew) + if err != nil { + return err + } + var finalizers []string + for _, finalizer := range metadata.GetFinalizers() { + if finalizer == "backup-operator-periodic" { + continue + } + finalizers = append(finalizers, finalizer) + } + metadata.SetFinalizers(finalizers) + _, err = b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Update(ebNew.(*api.EtcdBackup)) + if err != nil { + return err + } + return nil +} + +func (b *Backup) periodicRunnerFunc(ctx context.Context, t *time.Ticker, eb *api.EtcdBackup) { + defer t.Stop() + for { + select { + case <-ctx.Done(): + break + case <-t.C: + var latestEb *api.EtcdBackup + var err error + for { + latestEb, err = b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Get(eb.Name, metav1.GetOptions{}) + if err != nil { + b.logger.Warningf("Failed to get latest EtcdBackup %v : (%v)", eb.Name, err) + time.Sleep(1) + continue + } + break + } + // Perform backup + bs, err := b.handleBackup(&ctx, &latestEb.Spec) + // Report backup status + b.reportBackupStatus(bs, err, latestEb) + } + } +} + func (b *Backup) reportBackupStatus(bs *api.BackupStatus, berr error, eb *api.EtcdBackup) { if berr != nil { eb.Status.Succeeded = false @@ -116,7 +238,8 @@ func (b *Backup) handleErr(err error, key interface{}) { b.logger.Infof("Dropping etcd backup (%v) out of the queue: %v", key, err) } -func (b *Backup) handleBackup(spec *api.BackupSpec) (*api.BackupStatus, error) { +func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSpec) (*api.BackupStatus, error) { + // TODO(Yuki Nishiwaki) validate spec for periodic job err := validate(spec) if err != nil { return nil, err @@ -128,7 +251,11 @@ func (b *Backup) handleBackup(spec *api.BackupSpec) (*api.BackupStatus, error) { backupTimeout = time.Duration(spec.BackupPolicy.TimeoutInSecond) * time.Second } - ctx, cancel := context.WithTimeout(context.Background(), backupTimeout) + if parentContext == nil { + tmpParent := context.Background() + parentContext = &tmpParent + } + ctx, cancel := context.WithTimeout(*parentContext, backupTimeout) defer cancel() switch spec.StorageType { case api.BackupStorageTypeS3: diff --git a/pkg/controller/backup-operator/util.go b/pkg/controller/backup-operator/util.go index 5cceba083..9c11eddb8 100644 --- a/pkg/controller/backup-operator/util.go +++ b/pkg/controller/backup-operator/util.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "fmt" + api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/util/etcdutil" "github.com/coreos/etcd-operator/pkg/util/k8sutil" @@ -38,3 +39,11 @@ func generateTLSConfig(kubecli kubernetes.Interface, clientTLSSecret, namespace } return tlsConfig, nil } + +func isPeriodicBackup(eb *api.EtcdBackup) bool { + if eb.Spec.BackupPolicy != nil { + return eb.Spec.BackupPolicy.BackupIntervalInSecond != 0 + } else { + return false + } +} From 23314d9bb94ccf2ea4087c86705312f4bdd32d82 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Sat, 15 Dec 2018 08:56:40 -0800 Subject: [PATCH 02/19] backup: Add validation of BackupIntervalInSecond This commit implement validation of BackupIntervalInSecond. After this commit, backup-operator will make sure BackupIntervalInSecond follow following restrictions - <0 is not allowed, failed validation - 0 is valid and disable periodic backup - >0 is valid and means interval --- pkg/controller/backup-operator/sync.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index d59c78d4d..565ae18d2 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -239,7 +239,6 @@ func (b *Backup) handleErr(err error, key interface{}) { } func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSpec) (*api.BackupStatus, error) { - // TODO(Yuki Nishiwaki) validate spec for periodic job err := validate(spec) if err != nil { return nil, err @@ -287,5 +286,8 @@ func validate(spec *api.BackupSpec) error { if len(spec.EtcdEndpoints) == 0 { return errors.New("spec.etcdEndpoints should not be empty") } + if spec.BackupPolicy != nil && spec.BackupPolicy.BackupIntervalInSecond < 0 { + return errros.New("spec.backupPoloicy.backupIntervalInSecond should not be lower than 0") + } return nil } From 3babc481b02c25058eade808da2e65b0deb4a262 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Sat, 15 Dec 2018 09:20:18 -0800 Subject: [PATCH 03/19] backup: Add LastSuccessDate Current backup status is only designed for one-shot snapshot. Always it show lastest results but it would be nice if we could record the last time to successfully take a snapshot. --- pkg/apis/etcd/v1beta2/backup_types.go | 2 ++ pkg/backup/backup_manager.go | 5 ++--- pkg/controller/backup-operator/abs_backup.go | 7 ++++--- pkg/controller/backup-operator/gcs_backup.go | 7 ++++--- pkg/controller/backup-operator/s3_backup.go | 7 ++++--- pkg/controller/backup-operator/sync.go | 1 + 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/apis/etcd/v1beta2/backup_types.go b/pkg/apis/etcd/v1beta2/backup_types.go index 93e4c6b59..32a36e698 100644 --- a/pkg/apis/etcd/v1beta2/backup_types.go +++ b/pkg/apis/etcd/v1beta2/backup_types.go @@ -113,6 +113,8 @@ type BackupStatus struct { EtcdVersion string `json:"etcdVersion,omitempty"` // EtcdRevision is the revision of etcd's KV store where the backup is performed on. EtcdRevision int64 `json:"etcdRevision,omitempty"` + // LastSuccessDate indicate the time to get snapshot last time + LastSuccessDate time.Time `json:"lastSuccessDate,omitempty"` } // S3BackupSource provides the spec how to store backups on S3. diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index 89eac4392..87cb3b931 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -52,7 +52,7 @@ func NewBackupManagerFromWriter(kubecli kubernetes.Interface, bw writer.Writer, // SaveSnap uses backup writer to save etcd snapshot to a specified S3 path // and returns backup etcd server's kv store revision and its version. -func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string) (int64, string, error) { +func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.Time) (int64, string, error) { etcdcli, rev, err := bm.etcdClientWithMaxRevision(ctx) if err != nil { return 0, "", fmt.Errorf("create etcd client failed: %v", err) @@ -69,8 +69,7 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string) (int64, st return 0, "", fmt.Errorf("failed to receive snapshot (%v)", err) } defer rc.Close() - s3Path = fmt.Sprintf(s3Path+"_v%d_%s", - rev, time.Now().UTC().Format("2006-01-02-15:04:05")) + s3Path = fmt.Sprintf(s3Path+"_v%d_%s", rev, now.Format("2006-01-02-15:04:05")) _, err = bm.bw.Write(ctx, s3Path, rc) if err != nil { return 0, "", fmt.Errorf("failed to write snapshot (%v)", err) diff --git a/pkg/controller/backup-operator/abs_backup.go b/pkg/controller/backup-operator/abs_backup.go index d38af1cc8..27db1a1c5 100644 --- a/pkg/controller/backup-operator/abs_backup.go +++ b/pkg/controller/backup-operator/abs_backup.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "fmt" + "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -39,12 +40,12 @@ func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBack if tlsConfig, err = generateTLSConfig(kubecli, clientTLSSecret, namespace); err != nil { return nil, err } - + now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewABSWriter(cli.ABS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path) + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } - return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev}, nil + return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil } diff --git a/pkg/controller/backup-operator/gcs_backup.go b/pkg/controller/backup-operator/gcs_backup.go index 5a882d116..1305eba2d 100644 --- a/pkg/controller/backup-operator/gcs_backup.go +++ b/pkg/controller/backup-operator/gcs_backup.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "fmt" + "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -40,12 +41,12 @@ func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBack if tlsConfig, err = generateTLSConfig(kubecli, clientTLSSecret, namespace); err != nil { return nil, err } - + now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewGCSWriter(cli.GCS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path) + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } - return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev}, nil + return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil } diff --git a/pkg/controller/backup-operator/s3_backup.go b/pkg/controller/backup-operator/s3_backup.go index caf0c18ce..c60d0c383 100644 --- a/pkg/controller/backup-operator/s3_backup.go +++ b/pkg/controller/backup-operator/s3_backup.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "fmt" + "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -41,12 +42,12 @@ func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3Backup if tlsConfig, err = generateTLSConfig(kubecli, clientTLSSecret, namespace); err != nil { return nil, err } - + now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewS3Writer(cli.S3), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path) + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } - return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev}, nil + return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil } diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index 565ae18d2..7c45597c8 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -207,6 +207,7 @@ func (b *Backup) reportBackupStatus(bs *api.BackupStatus, berr error, eb *api.Et eb.Status.Succeeded = true eb.Status.EtcdRevision = bs.EtcdRevision eb.Status.EtcdVersion = bs.EtcdVersion + eb.Status.LastSuccessDate = bs.LastSuccessDate } _, err := b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Update(eb) if err != nil { From e10633f1085430d1165b7c6b0b102a8b49df57ec Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Sat, 15 Dec 2018 14:42:58 -0800 Subject: [PATCH 04/19] backup: Add MaxBackups in BackupPolicy This commit added MaxBackups attributs which let backup-operator delete older snapshots if the number of snapshots exceeded than MaxBackups. Specification of MaxBackups is following - <0 is not allowed, which cause validation failure - =0 is to indicate MaxBackups is infinite, which let not operator delete any exisiting snapshots - >0 is to indicate the max number of snapshot --- pkg/apis/etcd/v1beta2/backup_types.go | 3 ++ pkg/backup/backup_manager.go | 22 +++++++++ pkg/backup/writer/abs_writer.go | 47 ++++++++++++++++++++ pkg/backup/writer/gcs_writer.go | 35 +++++++++++++++ pkg/backup/writer/s3_writer.go | 36 +++++++++++++++ pkg/backup/writer/writer.go | 6 +++ pkg/controller/backup-operator/abs_backup.go | 8 +++- pkg/controller/backup-operator/gcs_backup.go | 8 +++- pkg/controller/backup-operator/s3_backup.go | 8 +++- pkg/controller/backup-operator/sync.go | 19 +++++--- 10 files changed, 184 insertions(+), 8 deletions(-) diff --git a/pkg/apis/etcd/v1beta2/backup_types.go b/pkg/apis/etcd/v1beta2/backup_types.go index 32a36e698..621dbf964 100644 --- a/pkg/apis/etcd/v1beta2/backup_types.go +++ b/pkg/apis/etcd/v1beta2/backup_types.go @@ -101,6 +101,9 @@ type BackupPolicy struct { // BackupIntervalInSecond is to specify how often operator take snapshot // 0 is magic number to indicate one-shot backup BackupIntervalInSecond int64 `json:"backupIntervalInSecond,omitempty"` + // MaxBackups is to specify how many backups we want to keep + // 0 is magic number to indicate un-limited backups + MaxBackups int `json:"maxBackups,omitempty"` } // BackupStatus represents the status of the EtcdBackup Custom Resource. diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index 87cb3b931..2751611d7 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "fmt" + "sort" "time" "github.com/coreos/etcd-operator/pkg/backup/writer" @@ -77,6 +78,27 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.T return rev, resp.Version, nil } +// EnsureMaxbackup to ensure the number of snapshot is under maxcount +// if the number of snapshot exceeded than maxcount, delete oldest snapshot +func (bm *BackupManager) EnsureMaxbackup(ctx context.Context, s3Path string, maxCount int) error { + + savedSnapShots, err := bm.bw.List(ctx, s3Path) + if err != nil { + return fmt.Errorf("failed to get exisiting snapashots: %v", err) + } + sort.Sort(sort.Reverse(sort.StringSlice(savedSnapShots))) + for i, snapshotPath := range savedSnapShots { + if i < maxCount { + continue + } + err := bm.bw.Delete(ctx, snapshotPath) + if err != nil { + return fmt.Errorf("failed to delete snapshot: %v", err) + } + } + return nil +} + // etcdClientWithMaxRevision gets the etcd endpoint with the maximum kv store revision // and returns the etcd client of that member. func (bm *BackupManager) etcdClientWithMaxRevision(ctx context.Context) (*clientv3.Client, int64, error) { diff --git a/pkg/backup/writer/abs_writer.go b/pkg/backup/writer/abs_writer.go index d7fc1fd08..743e5b758 100644 --- a/pkg/backup/writer/abs_writer.go +++ b/pkg/backup/writer/abs_writer.go @@ -100,3 +100,50 @@ func (absw *absWriter) Write(ctx context.Context, path string, r io.Reader) (int return blob.Properties.ContentLength, nil } + +func (absw *absWriter) List(ctx context.Context, basePath string) ([]string, error) { + // TODO: support context. + container, _, err := util.ParseBucketAndKey(basePath) + if err != nil { + return nil, err + } + + containerRef := absw.abs.GetContainerReference(container) + containerExists, err := containerRef.Exists() + if err != nil { + return nil, err + } + if !containerExists { + return nil, fmt.Errorf("container %v does not exist", container) + } + + blobs, err := containerRef.ListBlobs( + storage.ListBlobsParameters{Prefix: basePath}) + if err != nil { + return nil, err + } + blobKeys := []string{} + for _, blob := range blobs.Blobs { + blobKeys = append(blobKeys, container+"/"+blob.Name) + } + return blobKeys, nil +} + +func (absw *absWriter) Delete(ctx context.Context, path string) error { + // TODO: support context. + container, key, err := util.ParseBucketAndKey(path) + if err != nil { + return err + } + containerRef := absw.abs.GetContainerReference(container) + containerExists, err := containerRef.Exists() + if err != nil { + return err + } + if !containerExists { + return fmt.Errorf("container %v does not exist", container) + } + + blob := containerRef.GetBlobReference(key) + return blob.Delete(&storage.DeleteBlobOptions{}) +} diff --git a/pkg/backup/writer/gcs_writer.go b/pkg/backup/writer/gcs_writer.go index eda6142f9..61b6cd056 100644 --- a/pkg/backup/writer/gcs_writer.go +++ b/pkg/backup/writer/gcs_writer.go @@ -23,6 +23,7 @@ import ( "cloud.google.com/go/storage" "github.com/sirupsen/logrus" + "google.golang.org/api/iterator" ) var _ Writer = &gcsWriter{} @@ -58,3 +59,37 @@ func (gcsw *gcsWriter) Write(ctx context.Context, path string, r io.Reader) (int } return n, err } + +func (gcsw *gcsWriter) List(ctx context.Context, basePath string) ([]string, error) { + bucket, key, err := util.ParseBucketAndKey(basePath) + if err != nil { + return nil, err + } + objects := gcsw.gcs.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: key}) + if objects == nil { + return nil, fmt.Errorf("failed to get objects having %s prefix", key) + } + + objectKeys := []string{} + + for { + objAttrs, err := objects.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + objectKeys = append(objectKeys, bucket+"/"+objAttrs.Name) + } + return objectKeys, nil +} + +func (gcsw *gcsWriter) Delete(ctx context.Context, path string) error { + bucket, key, err := util.ParseBucketAndKey(path) + if err != nil { + return err + } + + return gcsw.gcs.Bucket(bucket).Object(key).Delete(ctx) +} diff --git a/pkg/backup/writer/s3_writer.go b/pkg/backup/writer/s3_writer.go index 9ffe37f42..9bbc0aea7 100644 --- a/pkg/backup/writer/s3_writer.go +++ b/pkg/backup/writer/s3_writer.go @@ -67,3 +67,39 @@ func (s3w *s3Writer) Write(ctx context.Context, path string, r io.Reader) (int64 } return *resp.ContentLength, nil } + +// List return the file paths which match the given s3 path +func (s3w *s3Writer) List(ctx context.Context, basePath string) ([]string, error) { + bk, key, err := util.ParseBucketAndKey(basePath) + if err != nil { + return nil, err + } + + objects, err := s3w.s3.ListObjectsWithContext(ctx, + &s3.ListObjectsInput{ + Bucket: aws.String(bk), + Prefix: aws.String(key), + }) + if err != nil { + return nil, err + } + objectKeys := []string{} + for _, object := range objects.Contents { + objectKeys = append(objectKeys, bk+"/"+ *object.Key) + } + return objectKeys, nil +} + +func (s3w *s3Writer) Delete(ctx context.Context, path string) error { + bk, key, err := util.ParseBucketAndKey(path) + if err != nil { + return err + } + + _, err = s3w.s3.DeleteObjectWithContext(ctx, + &s3.DeleteObjectInput{ + Bucket: aws.String(bk), + Key: aws.String(key), + }) + return err +} diff --git a/pkg/backup/writer/writer.go b/pkg/backup/writer/writer.go index 37322710e..ea5e26898 100644 --- a/pkg/backup/writer/writer.go +++ b/pkg/backup/writer/writer.go @@ -23,4 +23,10 @@ import ( type Writer interface { // Write writes a backup file to the given path and returns size of written file. Write(ctx context.Context, path string, r io.Reader) (int64, error) + + // List a backup files + List(ctx context.Context, basePath string) ([]string, error) + + // Delete a backup file + Delete(ctx context.Context, path string) error } diff --git a/pkg/controller/backup-operator/abs_backup.go b/pkg/controller/backup-operator/abs_backup.go index 27db1a1c5..47642cac9 100644 --- a/pkg/controller/backup-operator/abs_backup.go +++ b/pkg/controller/backup-operator/abs_backup.go @@ -29,7 +29,7 @@ import ( ) // handleABS saves etcd cluster's backup to specificed ABS path. -func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBackupSource, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { +func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBackupSource, endpoints []string, clientTLSSecret, namespace string, maxBackup int) (*api.BackupStatus, error) { // TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx. cli, err := absfactory.NewClientFromSecret(kubecli, namespace, s.ABSSecret) if err != nil { @@ -47,5 +47,11 @@ func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBack if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } + if maxBackup > 0 { + err := bm.EnsureMaxbackup(ctx, s.Path, maxBackup) + if err != nil { + return nil, fmt.Errorf("succeeded in saving snapshot but failed to delete old snapshot (%v)", err) + } + } return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil } diff --git a/pkg/controller/backup-operator/gcs_backup.go b/pkg/controller/backup-operator/gcs_backup.go index 1305eba2d..1b926e5e5 100644 --- a/pkg/controller/backup-operator/gcs_backup.go +++ b/pkg/controller/backup-operator/gcs_backup.go @@ -29,7 +29,7 @@ import ( ) // handleGCS saves etcd cluster's backup to specificed GCS path. -func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBackupSource, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { +func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBackupSource, endpoints []string, clientTLSSecret, namespace string, maxBackup int) (*api.BackupStatus, error) { // TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx. cli, err := gcsfactory.NewClientFromSecret(ctx, kubecli, namespace, s.GCPSecret) if err != nil { @@ -48,5 +48,11 @@ func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBack if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } + if maxBackup > 0 { + err := bm.EnsureMaxbackup(ctx, s.Path, maxBackup) + if err != nil { + return nil, fmt.Errorf("succeeded in saving snapshot but failed to delete old snapshot (%v)", err) + } + } return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil } diff --git a/pkg/controller/backup-operator/s3_backup.go b/pkg/controller/backup-operator/s3_backup.go index c60d0c383..089e9b5b7 100644 --- a/pkg/controller/backup-operator/s3_backup.go +++ b/pkg/controller/backup-operator/s3_backup.go @@ -30,7 +30,7 @@ import ( // TODO: replace this with generic backend interface for other options (PV, Azure) // handleS3 saves etcd cluster's backup to specificed S3 path. -func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []string, clientTLSSecret, namespace string) (*api.BackupStatus, error) { +func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []string, clientTLSSecret, namespace string, maxBackup int) (*api.BackupStatus, error) { // TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx. cli, err := s3factory.NewClientFromSecret(kubecli, namespace, s.Endpoint, s.AWSSecret) if err != nil { @@ -49,5 +49,11 @@ func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3Backup if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } + if maxBackup > 0 { + err := bm.EnsureMaxbackup(ctx, s.Path, maxBackup) + if err != nil { + return nil, fmt.Errorf("succeeded in saving snapshot but failed to delete old snapshot (%v)", err) + } + } return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil } diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index 7c45597c8..b1e79e475 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -250,6 +250,10 @@ func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSp if spec.BackupPolicy != nil && spec.BackupPolicy.TimeoutInSecond > 0 { backupTimeout = time.Duration(spec.BackupPolicy.TimeoutInSecond) * time.Second } + backupMaxCount := 0 + if spec.BackupPolicy != nil && spec.BackupPolicy.MaxBackups > 0 { + backupMaxCount = spec.BackupPolicy.MaxBackups + } if parentContext == nil { tmpParent := context.Background() @@ -259,19 +263,19 @@ func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSp defer cancel() switch spec.StorageType { case api.BackupStorageTypeS3: - bs, err := handleS3(ctx, b.kubecli, spec.S3, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) + bs, err := handleS3(ctx, b.kubecli, spec.S3, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace, backupMaxCount) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeABS: - bs, err := handleABS(ctx, b.kubecli, spec.ABS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) + bs, err := handleABS(ctx, b.kubecli, spec.ABS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace, backupMaxCount) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeGCS: - bs, err := handleGCS(ctx, b.kubecli, spec.GCS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace) + bs, err := handleGCS(ctx, b.kubecli, spec.GCS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace, backupMaxCount) if err != nil { return nil, err } @@ -287,8 +291,13 @@ func validate(spec *api.BackupSpec) error { if len(spec.EtcdEndpoints) == 0 { return errors.New("spec.etcdEndpoints should not be empty") } - if spec.BackupPolicy != nil && spec.BackupPolicy.BackupIntervalInSecond < 0 { - return errros.New("spec.backupPoloicy.backupIntervalInSecond should not be lower than 0") + if spec.BackupPolicy != nil { + if spec.BackupPolicy.BackupIntervalInSecond < 0 { + return errors.New("spec.backupPoloicy.backupIntervalInSecond should not be lower than 0") + } + if spec.BackupPolicy.MaxBackups < 0 { + return errors.New("spec.backupPolicy.MaxBackups should not be lower than 0") + } } return nil } From d36d8887650a127bf4ecb1bbed7fae2acc1950bd Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 17 Dec 2018 10:08:19 +0900 Subject: [PATCH 05/19] backup: Use path without adding info if one-shot After support periodic backup, backup-operator added revision number and date to s3 path as following /_v_. This behaviour has been applied even if backup is one-shot backup, therfore this change broke exisiting behaviour. This commit brough back original behaviour which use s3 path without adding anything /, if backup is not periodic --- pkg/backup/backup_manager.go | 6 ++++-- pkg/controller/backup-operator/abs_backup.go | 5 +++-- pkg/controller/backup-operator/gcs_backup.go | 5 +++-- pkg/controller/backup-operator/s3_backup.go | 5 +++-- pkg/controller/backup-operator/sync.go | 15 +++++++++------ pkg/controller/backup-operator/util.go | 6 +++--- 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index 2751611d7..c8faa8da2 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -53,7 +53,7 @@ func NewBackupManagerFromWriter(kubecli kubernetes.Interface, bw writer.Writer, // SaveSnap uses backup writer to save etcd snapshot to a specified S3 path // and returns backup etcd server's kv store revision and its version. -func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.Time) (int64, string, error) { +func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.Time, isPeriodic bool) (int64, string, error) { etcdcli, rev, err := bm.etcdClientWithMaxRevision(ctx) if err != nil { return 0, "", fmt.Errorf("create etcd client failed: %v", err) @@ -70,7 +70,9 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.T return 0, "", fmt.Errorf("failed to receive snapshot (%v)", err) } defer rc.Close() - s3Path = fmt.Sprintf(s3Path+"_v%d_%s", rev, now.Format("2006-01-02-15:04:05")) + if isPeriodic { + s3Path = fmt.Sprintf(s3Path+"_v%d_%s", rev, now.Format("2006-01-02-15:04:05")) + } _, err = bm.bw.Write(ctx, s3Path, rc) if err != nil { return 0, "", fmt.Errorf("failed to write snapshot (%v)", err) diff --git a/pkg/controller/backup-operator/abs_backup.go b/pkg/controller/backup-operator/abs_backup.go index 47642cac9..edb6bc775 100644 --- a/pkg/controller/backup-operator/abs_backup.go +++ b/pkg/controller/backup-operator/abs_backup.go @@ -29,7 +29,8 @@ import ( ) // handleABS saves etcd cluster's backup to specificed ABS path. -func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBackupSource, endpoints []string, clientTLSSecret, namespace string, maxBackup int) (*api.BackupStatus, error) { +func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBackupSource, endpoints []string, clientTLSSecret, + namespace string, isPeriodic bool, maxBackup int) (*api.BackupStatus, error) { // TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx. cli, err := absfactory.NewClientFromSecret(kubecli, namespace, s.ABSSecret) if err != nil { @@ -43,7 +44,7 @@ func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBack now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewABSWriter(cli.ABS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now) + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now, isPeriodic) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } diff --git a/pkg/controller/backup-operator/gcs_backup.go b/pkg/controller/backup-operator/gcs_backup.go index 1b926e5e5..30cb5d715 100644 --- a/pkg/controller/backup-operator/gcs_backup.go +++ b/pkg/controller/backup-operator/gcs_backup.go @@ -29,7 +29,8 @@ import ( ) // handleGCS saves etcd cluster's backup to specificed GCS path. -func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBackupSource, endpoints []string, clientTLSSecret, namespace string, maxBackup int) (*api.BackupStatus, error) { +func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBackupSource, endpoints []string, clientTLSSecret, + namespace string, isPeriodic bool, maxBackup int) (*api.BackupStatus, error) { // TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx. cli, err := gcsfactory.NewClientFromSecret(ctx, kubecli, namespace, s.GCPSecret) if err != nil { @@ -44,7 +45,7 @@ func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBack now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewGCSWriter(cli.GCS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now) + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now, isPeriodic) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } diff --git a/pkg/controller/backup-operator/s3_backup.go b/pkg/controller/backup-operator/s3_backup.go index 089e9b5b7..0a73ede7f 100644 --- a/pkg/controller/backup-operator/s3_backup.go +++ b/pkg/controller/backup-operator/s3_backup.go @@ -30,7 +30,8 @@ import ( // TODO: replace this with generic backend interface for other options (PV, Azure) // handleS3 saves etcd cluster's backup to specificed S3 path. -func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []string, clientTLSSecret, namespace string, maxBackup int) (*api.BackupStatus, error) { +func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3BackupSource, endpoints []string, clientTLSSecret, + namespace string, isPeriodic bool, maxBackup int) (*api.BackupStatus, error) { // TODO: controls NewClientFromSecret with ctx. This depends on upstream kubernetes to support API calls with ctx. cli, err := s3factory.NewClientFromSecret(kubecli, namespace, s.Endpoint, s.AWSSecret) if err != nil { @@ -45,7 +46,7 @@ func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3Backup now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewS3Writer(cli.S3), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now) + rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now, isPeriodic) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index b1e79e475..f82f428a6 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -82,12 +82,12 @@ func (b *Backup) processItem(key string) error { } // don't process the CR if it has a status since // having a status means that the backup is either made or failed. - if !isPeriodicBackup(eb) && + if !isPeriodicBackup(&eb.Spec) && (eb.Status.Succeeded || len(eb.Status.Reason) != 0) { return nil } - if isPeriodicBackup(eb) && b.isChanged(eb) { + if isPeriodicBackup(&eb.Spec) && b.isChanged(eb) { // Stop previous backup runner if it exists b.deletePeriodicBackupRunner(eb.ObjectMeta.UID) @@ -107,7 +107,7 @@ func (b *Backup) processItem(key string) error { // Store cancel function for periodic b.backupRunnerStore.Store(eb.ObjectMeta.UID, BackupRunner{eb.Spec, cancel}) - } else if !isPeriodicBackup(eb) { + } else if !isPeriodicBackup(&eb.Spec) { // Perform backup bs, err := b.handleBackup(nil, &eb.Spec) // Report backup status @@ -263,19 +263,22 @@ func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSp defer cancel() switch spec.StorageType { case api.BackupStorageTypeS3: - bs, err := handleS3(ctx, b.kubecli, spec.S3, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace, backupMaxCount) + bs, err := handleS3(ctx, b.kubecli, spec.S3, spec.EtcdEndpoints, spec.ClientTLSSecret, + b.namespace, isPeriodicBackup(spec), backupMaxCount) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeABS: - bs, err := handleABS(ctx, b.kubecli, spec.ABS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace, backupMaxCount) + bs, err := handleABS(ctx, b.kubecli, spec.ABS, spec.EtcdEndpoints, spec.ClientTLSSecret, + b.namespace, isPeriodicBackup(spec), backupMaxCount) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeGCS: - bs, err := handleGCS(ctx, b.kubecli, spec.GCS, spec.EtcdEndpoints, spec.ClientTLSSecret, b.namespace, backupMaxCount) + bs, err := handleGCS(ctx, b.kubecli, spec.GCS, spec.EtcdEndpoints, spec.ClientTLSSecret, + b.namespace, isPeriodicBackup(spec), backupMaxCount) if err != nil { return nil, err } diff --git a/pkg/controller/backup-operator/util.go b/pkg/controller/backup-operator/util.go index 9c11eddb8..6aab66daa 100644 --- a/pkg/controller/backup-operator/util.go +++ b/pkg/controller/backup-operator/util.go @@ -40,9 +40,9 @@ func generateTLSConfig(kubecli kubernetes.Interface, clientTLSSecret, namespace return tlsConfig, nil } -func isPeriodicBackup(eb *api.EtcdBackup) bool { - if eb.Spec.BackupPolicy != nil { - return eb.Spec.BackupPolicy.BackupIntervalInSecond != 0 +func isPeriodicBackup(ebSpec *api.BackupSpec) bool { + if ebSpec.BackupPolicy != nil { + return ebSpec.BackupPolicy.BackupIntervalInSecond != 0 } else { return false } From 31c5f4bcf0b25d18b1cbabcebeae3c764bccd393 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 17 Dec 2018 10:19:55 +0900 Subject: [PATCH 06/19] backup: Reset reason if backup succeeded --- pkg/controller/backup-operator/sync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index f82f428a6..8e90ac132 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -204,6 +204,7 @@ func (b *Backup) reportBackupStatus(bs *api.BackupStatus, berr error, eb *api.Et eb.Status.Succeeded = false eb.Status.Reason = berr.Error() } else { + eb.Status.Reason = "" eb.Status.Succeeded = true eb.Status.EtcdRevision = bs.EtcdRevision eb.Status.EtcdVersion = bs.EtcdVersion From 2dc177c3e2d84fa370e991c16caed91b308c0565 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 17 Dec 2018 14:19:56 +0900 Subject: [PATCH 07/19] backup: Update the codegen files --- pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go b/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go index e5f49a19d..4ca9b9942 100644 --- a/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go @@ -134,6 +134,7 @@ func (in *BackupSpec) DeepCopy() *BackupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = *in + in.LastSuccessDate.DeepCopyInto(&out.LastSuccessDate) return } @@ -217,7 +218,7 @@ func (in *EtcdBackup) DeepCopyInto(out *EtcdBackup) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) return } From fce86f11d5e20f0f862a9ba3146acc534d625d42 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 17 Dec 2018 14:29:05 +0900 Subject: [PATCH 08/19] backup: fix typo --- pkg/backup/backup_manager.go | 6 +++--- pkg/backup/writer/writer.go | 2 +- pkg/controller/backup-operator/sync.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index c8faa8da2..07186a7c4 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -80,13 +80,13 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.T return rev, resp.Version, nil } -// EnsureMaxbackup to ensure the number of snapshot is under maxcount +// EnsureMaxBackup to ensure the number of snapshot is under maxcount // if the number of snapshot exceeded than maxcount, delete oldest snapshot -func (bm *BackupManager) EnsureMaxbackup(ctx context.Context, s3Path string, maxCount int) error { +func (bm *BackupManager) EnsureMaxBackup(ctx context.Context, s3Path string, maxCount int) error { savedSnapShots, err := bm.bw.List(ctx, s3Path) if err != nil { - return fmt.Errorf("failed to get exisiting snapashots: %v", err) + return fmt.Errorf("failed to get exisiting snapshots: %v", err) } sort.Sort(sort.Reverse(sort.StringSlice(savedSnapShots))) for i, snapshotPath := range savedSnapShots { diff --git a/pkg/backup/writer/writer.go b/pkg/backup/writer/writer.go index ea5e26898..e1b9762b4 100644 --- a/pkg/backup/writer/writer.go +++ b/pkg/backup/writer/writer.go @@ -24,7 +24,7 @@ type Writer interface { // Write writes a backup file to the given path and returns size of written file. Write(ctx context.Context, path string, r io.Reader) (int64, error) - // List a backup files + // List backup files List(ctx context.Context, basePath string) ([]string, error) // Delete a backup file diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index 8e90ac132..18a3d5786 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -297,7 +297,7 @@ func validate(spec *api.BackupSpec) error { } if spec.BackupPolicy != nil { if spec.BackupPolicy.BackupIntervalInSecond < 0 { - return errors.New("spec.backupPoloicy.backupIntervalInSecond should not be lower than 0") + return errors.New("spec.backupPolicy.backupIntervalInSecond should not be lower than 0") } if spec.BackupPolicy.MaxBackups < 0 { return errors.New("spec.backupPolicy.MaxBackups should not be lower than 0") From d519aa7617a3bad79e514b65d26a748085b08cb1 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Wed, 19 Dec 2018 00:45:01 +0700 Subject: [PATCH 09/19] backup: Refactoring --- pkg/backup/writer/s3_writer.go | 2 +- pkg/controller/backup-operator/sync.go | 29 +++++++++++++------------- pkg/controller/backup-operator/util.go | 3 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/backup/writer/s3_writer.go b/pkg/backup/writer/s3_writer.go index 9bbc0aea7..623180e36 100644 --- a/pkg/backup/writer/s3_writer.go +++ b/pkg/backup/writer/s3_writer.go @@ -85,7 +85,7 @@ func (s3w *s3Writer) List(ctx context.Context, basePath string) ([]string, error } objectKeys := []string{} for _, object := range objects.Contents { - objectKeys = append(objectKeys, bk+"/"+ *object.Key) + objectKeys = append(objectKeys, bk+"/"+*object.Key) } return objectKeys, nil } diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index 18a3d5786..2b7013052 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -80,14 +80,16 @@ func (b *Backup) processItem(key string) error { } return nil } + isPeriodic := isPeriodicBackup(&eb.Spec) + // don't process the CR if it has a status since // having a status means that the backup is either made or failed. - if !isPeriodicBackup(&eb.Spec) && + if !isPeriodic && (eb.Status.Succeeded || len(eb.Status.Reason) != 0) { return nil } - if isPeriodicBackup(&eb.Spec) && b.isChanged(eb) { + if isPeriodic && b.isChanged(eb) { // Stop previous backup runner if it exists b.deletePeriodicBackupRunner(eb.ObjectMeta.UID) @@ -107,9 +109,9 @@ func (b *Backup) processItem(key string) error { // Store cancel function for periodic b.backupRunnerStore.Store(eb.ObjectMeta.UID, BackupRunner{eb.Spec, cancel}) - } else if !isPeriodicBackup(&eb.Spec) { + } else if !isPeriodic { // Perform backup - bs, err := b.handleBackup(nil, &eb.Spec) + bs, err := b.handleBackup(nil, &eb.Spec, false) // Report backup status b.reportBackupStatus(bs, err, eb) } @@ -146,10 +148,9 @@ func (b *Backup) addFinalizerOfPeriodicBackupIfNeed(eb *api.EtcdBackup) (*api.Et if err != nil { return eb, err } - } else { - return eb, nil + return ebNew.(*api.EtcdBackup), nil } - return ebNew.(*api.EtcdBackup), nil + return eb, nil } func (b *Backup) removeFinalizerOfPeriodicBackup(eb *api.EtcdBackup) error { @@ -192,7 +193,7 @@ func (b *Backup) periodicRunnerFunc(ctx context.Context, t *time.Ticker, eb *api break } // Perform backup - bs, err := b.handleBackup(&ctx, &latestEb.Spec) + bs, err := b.handleBackup(&ctx, &latestEb.Spec, true) // Report backup status b.reportBackupStatus(bs, err, latestEb) } @@ -240,7 +241,7 @@ func (b *Backup) handleErr(err error, key interface{}) { b.logger.Infof("Dropping etcd backup (%v) out of the queue: %v", key, err) } -func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSpec) (*api.BackupStatus, error) { +func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSpec, isPeriodic bool) (*api.BackupStatus, error) { err := validate(spec) if err != nil { return nil, err @@ -265,21 +266,21 @@ func (b *Backup) handleBackup(parentContext *context.Context, spec *api.BackupSp switch spec.StorageType { case api.BackupStorageTypeS3: bs, err := handleS3(ctx, b.kubecli, spec.S3, spec.EtcdEndpoints, spec.ClientTLSSecret, - b.namespace, isPeriodicBackup(spec), backupMaxCount) + b.namespace, isPeriodic, backupMaxCount) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeABS: bs, err := handleABS(ctx, b.kubecli, spec.ABS, spec.EtcdEndpoints, spec.ClientTLSSecret, - b.namespace, isPeriodicBackup(spec), backupMaxCount) + b.namespace, isPeriodic, backupMaxCount) if err != nil { return nil, err } return bs, nil case api.BackupStorageTypeGCS: bs, err := handleGCS(ctx, b.kubecli, spec.GCS, spec.EtcdEndpoints, spec.ClientTLSSecret, - b.namespace, isPeriodicBackup(spec), backupMaxCount) + b.namespace, isPeriodic, backupMaxCount) if err != nil { return nil, err } @@ -297,10 +298,10 @@ func validate(spec *api.BackupSpec) error { } if spec.BackupPolicy != nil { if spec.BackupPolicy.BackupIntervalInSecond < 0 { - return errors.New("spec.backupPolicy.backupIntervalInSecond should not be lower than 0") + return errors.New("spec.BackupPolicy.BackupIntervalInSecond should not be lower than 0") } if spec.BackupPolicy.MaxBackups < 0 { - return errors.New("spec.backupPolicy.MaxBackups should not be lower than 0") + return errors.New("spec.BackupPolicy.MaxBackups should not be lower than 0") } } return nil diff --git a/pkg/controller/backup-operator/util.go b/pkg/controller/backup-operator/util.go index 6aab66daa..22efa5fba 100644 --- a/pkg/controller/backup-operator/util.go +++ b/pkg/controller/backup-operator/util.go @@ -43,7 +43,6 @@ func generateTLSConfig(kubecli kubernetes.Interface, clientTLSSecret, namespace func isPeriodicBackup(ebSpec *api.BackupSpec) bool { if ebSpec.BackupPolicy != nil { return ebSpec.BackupPolicy.BackupIntervalInSecond != 0 - } else { - return false } + return false } From d1ae2c42dfaef5e7c773de47f302b23ebc9d8348 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Wed, 19 Dec 2018 00:50:54 +0700 Subject: [PATCH 10/19] backup: Use meta/v1.Time instead of time.Time After I generated the code based on k8s object (zz_generated.deepcopy.go), we happened to be in failing to build. This is because all k8s custom resource's fileds should implement DeepCopyInto but time.Time we added doesn't implement it. For this purpose we should have used meta/v1.Time which is the implementation to implement all necessary function for k8s object and same function of time.Time. And also this commit include some refactoring which is pointed out in code-review --- pkg/apis/etcd/v1beta2/backup_types.go | 4 +--- pkg/backup/backup_manager.go | 14 ++++++++------ pkg/controller/backup-operator/abs_backup.go | 8 +++----- pkg/controller/backup-operator/gcs_backup.go | 8 +++----- pkg/controller/backup-operator/s3_backup.go | 8 +++----- pkg/controller/backup-operator/sync.go | 11 ++--------- 6 files changed, 20 insertions(+), 33 deletions(-) diff --git a/pkg/apis/etcd/v1beta2/backup_types.go b/pkg/apis/etcd/v1beta2/backup_types.go index 621dbf964..90790c201 100644 --- a/pkg/apis/etcd/v1beta2/backup_types.go +++ b/pkg/apis/etcd/v1beta2/backup_types.go @@ -15,8 +15,6 @@ package v1beta2 import ( - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -117,7 +115,7 @@ type BackupStatus struct { // EtcdRevision is the revision of etcd's KV store where the backup is performed on. EtcdRevision int64 `json:"etcdRevision,omitempty"` // LastSuccessDate indicate the time to get snapshot last time - LastSuccessDate time.Time `json:"lastSuccessDate,omitempty"` + LastSuccessDate metav1.Time `json:"lastSuccessDate,omitempty"` } // S3BackupSource provides the spec how to store backups on S3. diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index 07186a7c4..1abebd2e9 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -26,6 +26,7 @@ import ( "github.com/coreos/etcd/clientv3" "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -53,21 +54,22 @@ func NewBackupManagerFromWriter(kubecli kubernetes.Interface, bw writer.Writer, // SaveSnap uses backup writer to save etcd snapshot to a specified S3 path // and returns backup etcd server's kv store revision and its version. -func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.Time, isPeriodic bool) (int64, string, error) { +func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, isPeriodic bool) (int64, string, *metav1.Time, error) { + now := time.Now().UTC() etcdcli, rev, err := bm.etcdClientWithMaxRevision(ctx) if err != nil { - return 0, "", fmt.Errorf("create etcd client failed: %v", err) + return 0, "", nil, fmt.Errorf("create etcd client failed: %v", err) } defer etcdcli.Close() resp, err := etcdcli.Status(ctx, etcdcli.Endpoints()[0]) if err != nil { - return 0, "", fmt.Errorf("failed to retrieve etcd version from the status call: %v", err) + return 0, "", nil, fmt.Errorf("failed to retrieve etcd version from the status call: %v", err) } rc, err := etcdcli.Snapshot(ctx) if err != nil { - return 0, "", fmt.Errorf("failed to receive snapshot (%v)", err) + return 0, "", nil, fmt.Errorf("failed to receive snapshot (%v)", err) } defer rc.Close() if isPeriodic { @@ -75,9 +77,9 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, now time.T } _, err = bm.bw.Write(ctx, s3Path, rc) if err != nil { - return 0, "", fmt.Errorf("failed to write snapshot (%v)", err) + return 0, "", nil, fmt.Errorf("failed to write snapshot (%v)", err) } - return rev, resp.Version, nil + return rev, resp.Version, &metav1.Time{now}, nil } // EnsureMaxBackup to ensure the number of snapshot is under maxcount diff --git a/pkg/controller/backup-operator/abs_backup.go b/pkg/controller/backup-operator/abs_backup.go index edb6bc775..c1c9c786d 100644 --- a/pkg/controller/backup-operator/abs_backup.go +++ b/pkg/controller/backup-operator/abs_backup.go @@ -18,7 +18,6 @@ import ( "context" "crypto/tls" "fmt" - "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -41,18 +40,17 @@ func handleABS(ctx context.Context, kubecli kubernetes.Interface, s *api.ABSBack if tlsConfig, err = generateTLSConfig(kubecli, clientTLSSecret, namespace); err != nil { return nil, err } - now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewABSWriter(cli.ABS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now, isPeriodic) + rev, etcdVersion, now, err := bm.SaveSnap(ctx, s.Path, isPeriodic) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } if maxBackup > 0 { - err := bm.EnsureMaxbackup(ctx, s.Path, maxBackup) + err := bm.EnsureMaxBackup(ctx, s.Path, maxBackup) if err != nil { return nil, fmt.Errorf("succeeded in saving snapshot but failed to delete old snapshot (%v)", err) } } - return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil + return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: *now}, nil } diff --git a/pkg/controller/backup-operator/gcs_backup.go b/pkg/controller/backup-operator/gcs_backup.go index 30cb5d715..1f1f5a921 100644 --- a/pkg/controller/backup-operator/gcs_backup.go +++ b/pkg/controller/backup-operator/gcs_backup.go @@ -18,7 +18,6 @@ import ( "context" "crypto/tls" "fmt" - "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -42,18 +41,17 @@ func handleGCS(ctx context.Context, kubecli kubernetes.Interface, s *api.GCSBack if tlsConfig, err = generateTLSConfig(kubecli, clientTLSSecret, namespace); err != nil { return nil, err } - now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewGCSWriter(cli.GCS), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now, isPeriodic) + rev, etcdVersion, now, err := bm.SaveSnap(ctx, s.Path, isPeriodic) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } if maxBackup > 0 { - err := bm.EnsureMaxbackup(ctx, s.Path, maxBackup) + err := bm.EnsureMaxBackup(ctx, s.Path, maxBackup) if err != nil { return nil, fmt.Errorf("succeeded in saving snapshot but failed to delete old snapshot (%v)", err) } } - return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil + return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: *now}, nil } diff --git a/pkg/controller/backup-operator/s3_backup.go b/pkg/controller/backup-operator/s3_backup.go index 0a73ede7f..190916c17 100644 --- a/pkg/controller/backup-operator/s3_backup.go +++ b/pkg/controller/backup-operator/s3_backup.go @@ -18,7 +18,6 @@ import ( "context" "crypto/tls" "fmt" - "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/pkg/backup" @@ -43,18 +42,17 @@ func handleS3(ctx context.Context, kubecli kubernetes.Interface, s *api.S3Backup if tlsConfig, err = generateTLSConfig(kubecli, clientTLSSecret, namespace); err != nil { return nil, err } - now := time.Now().UTC() bm := backup.NewBackupManagerFromWriter(kubecli, writer.NewS3Writer(cli.S3), tlsConfig, endpoints, namespace) - rev, etcdVersion, err := bm.SaveSnap(ctx, s.Path, now, isPeriodic) + rev, etcdVersion, now, err := bm.SaveSnap(ctx, s.Path, isPeriodic) if err != nil { return nil, fmt.Errorf("failed to save snapshot (%v)", err) } if maxBackup > 0 { - err := bm.EnsureMaxbackup(ctx, s.Path, maxBackup) + err := bm.EnsureMaxBackup(ctx, s.Path, maxBackup) if err != nil { return nil, fmt.Errorf("succeeded in saving snapshot but failed to delete old snapshot (%v)", err) } } - return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: now}, nil + return &api.BackupStatus{EtcdVersion: etcdVersion, EtcdRevision: rev, LastSuccessDate: *now}, nil } diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index 2b7013052..b2a65e41a 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -74,11 +74,7 @@ func (b *Backup) processItem(key string) error { if eb.DeletionTimestamp != nil { b.deletePeriodicBackupRunner(eb.ObjectMeta.UID) - err := b.removeFinalizerOfPeriodicBackup(eb) - if err != nil { - return err - } - return nil + return b.removeFinalizerOfPeriodicBackup(eb) } isPeriodic := isPeriodicBackup(&eb.Spec) @@ -168,10 +164,7 @@ func (b *Backup) removeFinalizerOfPeriodicBackup(eb *api.EtcdBackup) error { } metadata.SetFinalizers(finalizers) _, err = b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Update(ebNew.(*api.EtcdBackup)) - if err != nil { - return err - } - return nil + return err } func (b *Backup) periodicRunnerFunc(ctx context.Context, t *time.Ticker, eb *api.EtcdBackup) { From 74097170d91c89c9fa31e715242d5b0ee87257e1 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 14 Jan 2019 14:32:57 +0900 Subject: [PATCH 11/19] backup: minor fixes (naming, extra space) --- pkg/backup/backup_manager.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index 1abebd2e9..f0254892c 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -84,9 +84,8 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, isPeriodic // EnsureMaxBackup to ensure the number of snapshot is under maxcount // if the number of snapshot exceeded than maxcount, delete oldest snapshot -func (bm *BackupManager) EnsureMaxBackup(ctx context.Context, s3Path string, maxCount int) error { - - savedSnapShots, err := bm.bw.List(ctx, s3Path) +func (bm *BackupManager) EnsureMaxBackup(ctx context.Context, basePath string, maxCount int) error { + savedSnapShots, err := bm.bw.List(ctx, basePath) if err != nil { return fmt.Errorf("failed to get exisiting snapshots: %v", err) } From 8737c73a58d2fa23c886e861657cde6d9a6c3c2f Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 14 Jan 2019 14:40:54 +0900 Subject: [PATCH 12/19] backup: Add periodically backup example --- .../etcd-backup-operator/periodic_backup_cr.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 example/etcd-backup-operator/periodic_backup_cr.yaml diff --git a/example/etcd-backup-operator/periodic_backup_cr.yaml b/example/etcd-backup-operator/periodic_backup_cr.yaml new file mode 100644 index 000000000..d5e9a3dc0 --- /dev/null +++ b/example/etcd-backup-operator/periodic_backup_cr.yaml @@ -0,0 +1,16 @@ +apiVersion: "etcd.database.coreos.com/v1beta2" +kind: "EtcdBackup" +metadata: + name: example-etcd-cluster-periodic-backup +spec: + etcdEndpoints: [] + storageType: S3 + backupPolicy: + # 0 > enable periodic backup + backupIntervalInSecond: 125 + maxBackups: 4 + s3: + # The format of "path" must be: "/" + # e.g: "mybucket/etcd.backup" + path: + awsSecret: From e7dfe4b432823fa3ecc177c12a6a624612af9c49 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Tue, 15 Jan 2019 21:30:16 +0900 Subject: [PATCH 13/19] backup: add e2e slow test for periodic backup --- test/e2e/e2eslow/backup_restore_test.go | 97 +++++++++++++++++++++++++ test/e2e/e2eutil/util.go | 11 +++ test/e2e/e2eutil/wait_util.go | 15 ++++ 3 files changed, 123 insertions(+) diff --git a/test/e2e/e2eslow/backup_restore_test.go b/test/e2e/e2eslow/backup_restore_test.go index ddb149553..67caecd80 100644 --- a/test/e2e/e2eslow/backup_restore_test.go +++ b/test/e2e/e2eslow/backup_restore_test.go @@ -15,15 +15,20 @@ package e2eslow import ( + "context" "errors" "fmt" "math/rand" "os" "path" + "sort" + "strings" "testing" "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" + "github.com/coreos/etcd-operator/pkg/backup/writer" + "github.com/coreos/etcd-operator/pkg/util/awsutil/s3factory" "github.com/coreos/etcd-operator/pkg/util/etcdutil" "github.com/coreos/etcd-operator/pkg/util/k8sutil" "github.com/coreos/etcd-operator/pkg/util/retryutil" @@ -78,6 +83,7 @@ func TestBackupAndRestore(t *testing.T) { s3Path := path.Join(os.Getenv("TEST_S3_BUCKET"), "jenkins", suffix, time.Now().Format(time.RFC3339), "etcd.backup") testEtcdBackupOperatorForS3Backup(t, clusterName, operatorClientTLSSecret, s3Path) + testEtcdBackupOperatorForPeriodicS3Backup(t, clusterName, operatorClientTLSSecret, s3Path) testEtcdRestoreOperatorForS3Source(t, clusterName, s3Path) } @@ -166,6 +172,97 @@ func testEtcdBackupOperatorForS3Backup(t *testing.T, clusterName, operatorClient t.Logf("backup for cluster (%s) has been saved", clusterName) } +// testEtcdBackupOperatorForPeriodicS3Backup test if etcd backup operator can periodically backup and upload to s3. +// This e2e test would check basic function of periodic backup and MaxBackup functionality +func testEtcdBackupOperatorForPeriodicS3Backup(t *testing.T, clusterName, operatorClientTLSSecret, s3Path string) { + f := framework.Global + + endpoints, err := getEndpoints(f.KubeClient, true, f.Namespace, clusterName) + if err != nil { + t.Fatalf("failed to get endpoints: %v", err) + } + backupCR := e2eutil.NewS3Backup(endpoints, clusterName, s3Path, os.Getenv("TEST_AWS_SECRET"), operatorClientTLSSecret) + // enable periodic backup + backupCR.Spec.BackupPolicy = &api.BackupPolicy{BackupIntervalInSecond: 5, MaxBackups: 2} + backupS3Source := backupCR.Spec.BackupSource.S3 + + // initialize s3 client + s3cli, err := s3factory.NewClientFromSecret( + f.KubeClient, f.Namespace, backupS3Source.Endpoint, backupS3Source.AWSSecret) + if err != nil { + t.Fatalf("failed to initialize s3client: %v", err) + } + wr := writer.NewS3Writer(s3cli.S3) + + // check if there is existing backup file + allBackups, err := wr.List(context.Background(), backupS3Source.Path) + if err != nil { + t.Fatalf("failed to list backup files: %v", err) + } + if len(allBackups) > 0 { + t.Logf("existing backup file is detected: %s", strings.Join(allBackups, ",")) + // try to delete all existing backup files + if err := e2eutil.DeleteBackupFiles(wr, allBackups); err != nil { + t.Fatalf("failed to delete existing backup: %v", err) + } + // make sure no exisiting backups + // will wait for 10 sec until deleting operation completed + if err := e2eutil.WaitUntilNoBackupFiles(wr, backupS3Source.Path, 10); err != nil { + t.Fatalf("failed to make sure no old backup: %v", err) + } + } + + // create etcdbackup resource + eb, err := f.CRClient.EtcdV1beta2().EtcdBackups(f.Namespace).Create(backupCR) + if err != nil { + t.Fatalf("failed to create etcd back cr: %v", err) + } + defer func() { + if err := f.CRClient.EtcdV1beta2().EtcdBackups(f.Namespace).Delete(eb.Name, nil); err != nil { + t.Fatalf("failed to delete etcd backup cr: %v", err) + } + // cleanup backup files + allBackups, err = wr.List(context.Background(), backupS3Source.Path) + if err != nil { + t.Fatalf("failed to list backup files: %v", err) + } + if err := e2eutil.DeleteBackupFiles(wr, allBackups); err != nil { + t.Fatalf("failed to cleanup backup files: %v", err) + } + }() + + var firstBackup string + var periodicBackup, maxBackup bool + // Check if periodic backup is correctly performed + // Check if maxBackup is correctly performed + err = retryutil.Retry(time.Second, 20, func() (bool, error) { + allBackups, err = wr.List(context.Background(), backupS3Source.Path) + sort.Strings(allBackups) + if err != nil { + return false, fmt.Errorf("failed to list backup files: %v", err) + } + if len(allBackups) > 0 { + if firstBackup == "" { + firstBackup = allBackups[0] + } + // Check if firt seen backup file is deleted or not + if firstBackup != allBackups[0] { + maxBackup = true + } + if len(allBackups) > 1 { + periodicBackup = true + } + } + if periodicBackup && maxBackup { + return true, nil + } + return false, nil + }) + if err != nil { + t.Fatalf("failed to verify periodic bakcup: %v", err) + } +} + // testEtcdRestoreOperatorForS3Source tests if the restore-operator can restore an etcd cluster from an S3 restore source func testEtcdRestoreOperatorForS3Source(t *testing.T, clusterName, s3Path string) { f := framework.Global diff --git a/test/e2e/e2eutil/util.go b/test/e2e/e2eutil/util.go index 3e2060199..ea1def9a4 100644 --- a/test/e2e/e2eutil/util.go +++ b/test/e2e/e2eutil/util.go @@ -16,10 +16,12 @@ package e2eutil import ( "bytes" + "context" "fmt" "testing" "time" + "github.com/coreos/etcd-operator/pkg/backup/writer" "github.com/coreos/etcd-operator/pkg/util/k8sutil" "k8s.io/api/core/v1" @@ -62,3 +64,12 @@ func printContainerStatus(buf *bytes.Buffer, ss []v1.ContainerStatus) { } } } + +func DeleteBackupFiles(wr writer.Writer, files []string) error { + for _, v := range files { + if err := wr.Delete(context.Background(), v); err != nil { + return err + } + } + return nil +} diff --git a/test/e2e/e2eutil/wait_util.go b/test/e2e/e2eutil/wait_util.go index 42d273a50..9a55f75d6 100644 --- a/test/e2e/e2eutil/wait_util.go +++ b/test/e2e/e2eutil/wait_util.go @@ -16,12 +16,14 @@ package e2eutil import ( "bytes" + "context" "fmt" "strings" "testing" "time" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" + "github.com/coreos/etcd-operator/pkg/backup/writer" "github.com/coreos/etcd-operator/pkg/generated/clientset/versioned" "github.com/coreos/etcd-operator/pkg/util" "github.com/coreos/etcd-operator/pkg/util/k8sutil" @@ -275,3 +277,16 @@ func WaitUntilOperatorReady(kubecli kubernetes.Interface, namespace, name string } return nil } + +func WaitUntilNoBackupFiles(wr writer.Writer, path string, timeout int) error { + return retryutil.Retry(time.Second, timeout, func() (bool, error) { + allBackups, err := wr.List(context.Background(), path) + if err != nil { + return false, fmt.Errorf("failed to list backup files: %v", err) + } + if len(allBackups) > 0 { + return false, fmt.Errorf("%d existing backup files are detected", len(allBackups)) + } + return true, nil + }) +} From 1cdb3aaab0698fc7d068bbe9eaeacf602606b77a Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Tue, 15 Jan 2019 21:47:42 +0900 Subject: [PATCH 14/19] Update generated k8s code license for new year(2019) --- pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go | 2 +- pkg/generated/clientset/versioned/clientset.go | 2 +- pkg/generated/clientset/versioned/doc.go | 2 +- pkg/generated/clientset/versioned/fake/clientset_generated.go | 2 +- pkg/generated/clientset/versioned/fake/doc.go | 2 +- pkg/generated/clientset/versioned/fake/register.go | 2 +- pkg/generated/clientset/versioned/scheme/doc.go | 2 +- pkg/generated/clientset/versioned/scheme/register.go | 2 +- pkg/generated/clientset/versioned/typed/etcd/v1beta2/doc.go | 2 +- .../clientset/versioned/typed/etcd/v1beta2/etcd_client.go | 2 +- .../clientset/versioned/typed/etcd/v1beta2/etcdbackup.go | 2 +- .../clientset/versioned/typed/etcd/v1beta2/etcdcluster.go | 2 +- .../clientset/versioned/typed/etcd/v1beta2/etcdrestore.go | 2 +- .../clientset/versioned/typed/etcd/v1beta2/fake/doc.go | 2 +- .../versioned/typed/etcd/v1beta2/fake/fake_etcd_client.go | 2 +- .../versioned/typed/etcd/v1beta2/fake/fake_etcdbackup.go | 2 +- .../versioned/typed/etcd/v1beta2/fake/fake_etcdcluster.go | 2 +- .../versioned/typed/etcd/v1beta2/fake/fake_etcdrestore.go | 2 +- .../versioned/typed/etcd/v1beta2/generated_expansion.go | 2 +- pkg/generated/informers/externalversions/etcd/interface.go | 2 +- .../informers/externalversions/etcd/v1beta2/etcdbackup.go | 2 +- .../informers/externalversions/etcd/v1beta2/etcdcluster.go | 2 +- .../informers/externalversions/etcd/v1beta2/etcdrestore.go | 2 +- .../informers/externalversions/etcd/v1beta2/interface.go | 2 +- pkg/generated/informers/externalversions/factory.go | 2 +- pkg/generated/informers/externalversions/generic.go | 2 +- .../externalversions/internalinterfaces/factory_interfaces.go | 2 +- pkg/generated/listers/etcd/v1beta2/etcdbackup.go | 2 +- pkg/generated/listers/etcd/v1beta2/etcdcluster.go | 2 +- pkg/generated/listers/etcd/v1beta2/etcdrestore.go | 2 +- pkg/generated/listers/etcd/v1beta2/expansion_generated.go | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go b/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go index 4ca9b9942..c4ff202e8 100644 --- a/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/etcd/v1beta2/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index dc8961a40..e214728f2 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go index 3ef04b81b..e1edc9927 100644 --- a/pkg/generated/clientset/versioned/doc.go +++ b/pkg/generated/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 75791873c..388f98a7e 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go index 16d5871f5..4e0863d5a 100644 --- a/pkg/generated/clientset/versioned/fake/doc.go +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index 7ecefdaad..925a0ea85 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go index d2030dc97..becbf6d26 100644 --- a/pkg/generated/clientset/versioned/scheme/doc.go +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index ab9a544a3..897ed5ade 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/doc.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/doc.go index db3564b15..ae465c016 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/doc.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcd_client.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcd_client.go index 48e9c71b2..8931fb343 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcd_client.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcd_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdbackup.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdbackup.go index 4267fcf39..6be92f348 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdbackup.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdbackup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdcluster.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdcluster.go index 3cd3918d6..a3880f22a 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdcluster.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdcluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdrestore.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdrestore.go index e5dae4b50..647c7abb8 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdrestore.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/etcdrestore.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/doc.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/doc.go index dc4fe85bd..e143a66b5 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcd_client.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcd_client.go index 79f27104d..063204540 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcd_client.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcd_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdbackup.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdbackup.go index 4e9027b6c..05d4eddd3 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdbackup.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdbackup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdcluster.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdcluster.go index 78a91b07c..818d5fa99 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdcluster.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdcluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdrestore.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdrestore.go index b45746fe6..7aa929364 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdrestore.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/fake/fake_etcdrestore.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/generated_expansion.go b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/generated_expansion.go index efe619c63..faf0799cc 100644 --- a/pkg/generated/clientset/versioned/typed/etcd/v1beta2/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/etcd/v1beta2/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/etcd/interface.go b/pkg/generated/informers/externalversions/etcd/interface.go index e3c125c54..5e290b3c6 100644 --- a/pkg/generated/informers/externalversions/etcd/interface.go +++ b/pkg/generated/informers/externalversions/etcd/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/etcd/v1beta2/etcdbackup.go b/pkg/generated/informers/externalversions/etcd/v1beta2/etcdbackup.go index 7e9207932..05a0d71ee 100644 --- a/pkg/generated/informers/externalversions/etcd/v1beta2/etcdbackup.go +++ b/pkg/generated/informers/externalversions/etcd/v1beta2/etcdbackup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/etcd/v1beta2/etcdcluster.go b/pkg/generated/informers/externalversions/etcd/v1beta2/etcdcluster.go index a27eb939f..d4fdcd3b1 100644 --- a/pkg/generated/informers/externalversions/etcd/v1beta2/etcdcluster.go +++ b/pkg/generated/informers/externalversions/etcd/v1beta2/etcdcluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/etcd/v1beta2/etcdrestore.go b/pkg/generated/informers/externalversions/etcd/v1beta2/etcdrestore.go index e6a5f9782..944dd5583 100644 --- a/pkg/generated/informers/externalversions/etcd/v1beta2/etcdrestore.go +++ b/pkg/generated/informers/externalversions/etcd/v1beta2/etcdrestore.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/etcd/v1beta2/interface.go b/pkg/generated/informers/externalversions/etcd/v1beta2/interface.go index b19ca9fd7..91f3919ad 100644 --- a/pkg/generated/informers/externalversions/etcd/v1beta2/interface.go +++ b/pkg/generated/informers/externalversions/etcd/v1beta2/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index ee85cb000..dd8e1feef 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 712a0079c..6951a7b44 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go index 31b7e8e94..97d813f72 100644 --- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/listers/etcd/v1beta2/etcdbackup.go b/pkg/generated/listers/etcd/v1beta2/etcdbackup.go index 455ee676d..e78a9b49c 100644 --- a/pkg/generated/listers/etcd/v1beta2/etcdbackup.go +++ b/pkg/generated/listers/etcd/v1beta2/etcdbackup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/listers/etcd/v1beta2/etcdcluster.go b/pkg/generated/listers/etcd/v1beta2/etcdcluster.go index 4bd608f0e..27f0b3e24 100644 --- a/pkg/generated/listers/etcd/v1beta2/etcdcluster.go +++ b/pkg/generated/listers/etcd/v1beta2/etcdcluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/listers/etcd/v1beta2/etcdrestore.go b/pkg/generated/listers/etcd/v1beta2/etcdrestore.go index 7fff4247a..35f32fc38 100644 --- a/pkg/generated/listers/etcd/v1beta2/etcdrestore.go +++ b/pkg/generated/listers/etcd/v1beta2/etcdrestore.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/generated/listers/etcd/v1beta2/expansion_generated.go b/pkg/generated/listers/etcd/v1beta2/expansion_generated.go index 6357f3122..361c893bd 100644 --- a/pkg/generated/listers/etcd/v1beta2/expansion_generated.go +++ b/pkg/generated/listers/etcd/v1beta2/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The etcd-operator Authors +Copyright 2019 The etcd-operator Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From ab9b78ef650bbb84f3bc5a5bf6815f997dabf3fe Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Wed, 16 Jan 2019 11:12:49 +0900 Subject: [PATCH 15/19] backup: Minor fix "composite literal uses unkeyed" --- pkg/backup/backup_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/backup/backup_manager.go b/pkg/backup/backup_manager.go index f0254892c..a82ae32eb 100644 --- a/pkg/backup/backup_manager.go +++ b/pkg/backup/backup_manager.go @@ -79,7 +79,7 @@ func (bm *BackupManager) SaveSnap(ctx context.Context, s3Path string, isPeriodic if err != nil { return 0, "", nil, fmt.Errorf("failed to write snapshot (%v)", err) } - return rev, resp.Version, &metav1.Time{now}, nil + return rev, resp.Version, &metav1.Time{Time: now}, nil } // EnsureMaxBackup to ensure the number of snapshot is under maxcount From bb3b72c5a69124bee21d0323727a5c28de3b5585 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Wed, 16 Jan 2019 12:59:08 +0900 Subject: [PATCH 16/19] backup: Update CHANGELOG.md for periodic support --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 492ceb3d5..b0d32f4a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ ### Changed +- EtcdBackup: Support periodically backup. This change added 3 new fileds in EtcdBackup schema, 2 variables is in spec, 1 varialbe is in status. + - in spec.backupPolicy + - maxBackup which indicate maximum number of backup to keep + - backupIntervalInSecond which indicate how often do backup operation. + - in status + - LastSuccessDate which indicate the last time to succeed in taking backup + ### Removed ### Fixed From e084e9b9d305006d261495e3ddb1551a839fa102 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Sat, 9 Feb 2019 20:03:10 +0900 Subject: [PATCH 17/19] backup: Move periodbackup test after restore test restore test expected backup file to be present but periodic backup test actually cleanup backup file to be created so we failed to perform restore test because of that. that's why we moved periodic test after restore test --- test/e2e/e2eslow/backup_restore_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/e2eslow/backup_restore_test.go b/test/e2e/e2eslow/backup_restore_test.go index 67caecd80..deff374ea 100644 --- a/test/e2e/e2eslow/backup_restore_test.go +++ b/test/e2e/e2eslow/backup_restore_test.go @@ -82,9 +82,11 @@ func TestBackupAndRestore(t *testing.T) { s3Path := path.Join(os.Getenv("TEST_S3_BUCKET"), "jenkins", suffix, time.Now().Format(time.RFC3339), "etcd.backup") + // Backup then restore tests testEtcdBackupOperatorForS3Backup(t, clusterName, operatorClientTLSSecret, s3Path) - testEtcdBackupOperatorForPeriodicS3Backup(t, clusterName, operatorClientTLSSecret, s3Path) testEtcdRestoreOperatorForS3Source(t, clusterName, s3Path) + // Periodic backup test + testEtcdBackupOperatorForPeriodicS3Backup(t, clusterName, operatorClientTLSSecret, s3Path) } func verifyAWSEnvVars() error { From aee3c8d89f77b0f89dfbcdd753a854d7fdcb2d4f Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Sat, 9 Feb 2019 20:15:34 +0900 Subject: [PATCH 18/19] backup: Fixed the bug to get operator in infinite-loop --- pkg/controller/backup-operator/sync.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index b2a65e41a..b47c44b0b 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -175,18 +175,22 @@ func (b *Backup) periodicRunnerFunc(ctx context.Context, t *time.Ticker, eb *api break case <-t.C: var latestEb *api.EtcdBackup + var bs *api.BackupStatus var err error - for { + for i := 1; i < 6; i++ { latestEb, err = b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Get(eb.Name, metav1.GetOptions{}) if err != nil { - b.logger.Warningf("Failed to get latest EtcdBackup %v : (%v)", eb.Name, err) + b.logger.Warningf("[Attempt: %d/5] Failed to get latest EtcdBackup %v : (%v)", + i, eb.Name, err) time.Sleep(1) continue } break } - // Perform backup - bs, err := b.handleBackup(&ctx, &latestEb.Spec, true) + if err == nil { + // Perform backup + bs, err = b.handleBackup(&ctx, &latestEb.Spec, true) + } // Report backup status b.reportBackupStatus(bs, err, latestEb) } From 70aa25f4369ed2f1b3fa58b90330570e6aa3d777 Mon Sep 17 00:00:00 2001 From: Yuki Nishiwaki Date: Mon, 18 Feb 2019 20:35:15 +0900 Subject: [PATCH 19/19] backup: Only Retry in the case of transient error --- pkg/controller/backup-operator/sync.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/controller/backup-operator/sync.go b/pkg/controller/backup-operator/sync.go index b47c44b0b..5decdbf37 100644 --- a/pkg/controller/backup-operator/sync.go +++ b/pkg/controller/backup-operator/sync.go @@ -24,6 +24,7 @@ import ( "github.com/coreos/etcd-operator/pkg/util/constants" "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -177,11 +178,17 @@ func (b *Backup) periodicRunnerFunc(ctx context.Context, t *time.Ticker, eb *api var latestEb *api.EtcdBackup var bs *api.BackupStatus var err error - for i := 1; i < 6; i++ { + retryLimit := 5 + for i := 1; i < retryLimit+1; i++ { latestEb, err = b.backupCRCli.EtcdV1beta2().EtcdBackups(b.namespace).Get(eb.Name, metav1.GetOptions{}) if err != nil { - b.logger.Warningf("[Attempt: %d/5] Failed to get latest EtcdBackup %v : (%v)", - i, eb.Name, err) + if apierrors.IsNotFound(err) { + b.logger.Infof("Could not find EtcdBackup. Stopping periodic backup for EtcdBackup CR %v", + eb.Name) + break + } + b.logger.Warningf("[Attempt: %d/%d] Failed to get latest EtcdBackup %v : (%v)", + i, retryLimit, eb.Name, err) time.Sleep(1) continue }