From 2162c125e4e29869dbcd773ae13dfb0f6d437e2e Mon Sep 17 00:00:00 2001 From: Emruz Hossain Date: Thu, 10 Feb 2022 20:43:03 +0600 Subject: [PATCH] Refactor restore process status handling + add cross namespace tests (#1419) Signed-off-by: Emruz Hossain --- go.mod | 2 +- go.sum | 7 +- pkg/controller/init_container.go | 4 +- pkg/controller/repository.go | 2 +- pkg/controller/restore_session.go | 529 +++--------------- pkg/controller/usage_policy.go | 2 +- test/e2e/framework/backup_configuration.go | 23 + test/e2e/framework/common.go | 10 +- test/e2e/framework/deployment.go | 24 +- test/e2e/framework/framework.go | 20 +- test/e2e/framework/pvc.go | 6 +- test/e2e/framework/repository.go | 24 +- test/e2e/framework/restore_session.go | 46 +- test/e2e/misc/cross_namespace.go | 231 ++++++++ vendor/modules.txt | 4 +- .../apimachinery/apis/constants.go | 22 +- .../v1alpha1/openapi_generated.go | 16 + .../apis/stash/v1alpha1/openapi_generated.go | 16 + .../apis/stash/v1beta1/backup_batch_types.go | 1 + .../apis/stash/v1beta1/openapi_generated.go | 16 + .../apis/ui/v1alpha1/openapi_generated.go | 16 + .../stash.appscode.com_backupbatches.yaml | 3 + .../apimachinery/pkg/conditions/restore.go | 52 +- .../pkg/invoker/restore_invoker.go | 96 +++- .../apimachinery/pkg/invoker/restorebatch.go | 136 +++-- .../pkg/invoker/restoresession.go | 120 ++-- 26 files changed, 813 insertions(+), 615 deletions(-) create mode 100644 test/e2e/misc/cross_namespace.go diff --git a/go.mod b/go.mod index 164b0ac9a..6436cf39d 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( kmodules.xyz/openshift v0.0.0-20210618001443-f2507caa512f kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027 kmodules.xyz/webhook-runtime v0.0.0-20211116181908-909a755cc9d1 - stash.appscode.dev/apimachinery v0.17.1-0.20220203132324-9c5287433ad7 + stash.appscode.dev/apimachinery v0.17.1-0.20220210134237-79d844fbde2c ) require ( diff --git a/go.sum b/go.sum index c850205b1..b4e22e220 100644 --- a/go.sum +++ b/go.sum @@ -1240,16 +1240,13 @@ kmodules.xyz/client-go v0.0.0-20210617233340-13d22e91512b/go.mod h1:A6GAK6xP5zBu kmodules.xyz/client-go v0.0.0-20211013093146-1fbfd52e78c9/go.mod h1:0gkPeALtYjB27OHt4rd6+ZmMgoVTHVLtEJQeU23/gtA= kmodules.xyz/client-go v0.0.0-20211107190155-5bb4090d2728/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I= kmodules.xyz/client-go v0.0.0-20220104114408-2a3a05dbe89f/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo= -kmodules.xyz/client-go v0.0.0-20220131065336-bb3e98486e83/go.mod h1:aOwnhdxO0uh54ds1wQYRlKVtYlzLyakaesmMQeupVek= kmodules.xyz/client-go v0.0.0-20220203031013-1de48437aaf3 h1:CWux6RrrTkplf9F0ChJwkEEQuOorbc5rzmqC7uJUUZU= kmodules.xyz/client-go v0.0.0-20220203031013-1de48437aaf3/go.mod h1:aOwnhdxO0uh54ds1wQYRlKVtYlzLyakaesmMQeupVek= kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278 h1:sFmqh4EaiZ4K2FkkGvrDFddstq8GSf6ogH24IAsuKew= kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY= kmodules.xyz/crd-schema-fuzz v0.0.0-20210618002152-fae23aef5fb4/go.mod h1:IIkUctlfoptoci0BOrsUf8ya+MOG5uaeh1PE4uzaIbA= -kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd/go.mod h1:/XjDeILFV2wBota5kHo21DMvOt08nSAk1vm6buCuwt4= kmodules.xyz/custom-resources v0.0.0-20220208103158-61b298634e43 h1:mwW2DgP7sAMambZe7Met/e9nrBYnzYgM/lupbm7jgGM= kmodules.xyz/custom-resources v0.0.0-20220208103158-61b298634e43/go.mod h1:/XjDeILFV2wBota5kHo21DMvOt08nSAk1vm6buCuwt4= -kmodules.xyz/objectstore-api v0.0.0-20211116180107-8720be0c9bf7/go.mod h1:IICnDdPFOEeGXdaPVHOGYfdwD1cyh/p1I/TWMkyNTIE= kmodules.xyz/objectstore-api v0.0.0-20211207131029-3271069de43e h1:hbnb7Zy6pe0IwWWdIVbgfzBLM3kmppUMDpf7Sxy11d8= kmodules.xyz/objectstore-api v0.0.0-20211207131029-3271069de43e/go.mod h1:IICnDdPFOEeGXdaPVHOGYfdwD1cyh/p1I/TWMkyNTIE= kmodules.xyz/offshoot-api v0.0.0-20210829122105-6f4d481b0c61/go.mod h1:3LECbAL3FgbyK80NP3V3Pmiuo/a3hFWg/PR6SPFhTns= @@ -1297,5 +1294,5 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -stash.appscode.dev/apimachinery v0.17.1-0.20220203132324-9c5287433ad7 h1:C3TtXd5Efy81zI2DV2o2nBYNqEK+nXW8s57gubLsUrM= -stash.appscode.dev/apimachinery v0.17.1-0.20220203132324-9c5287433ad7/go.mod h1:9EmPzpKoqgD3BZUM59kQGL7oMrjU+oEn4T3HO1whvwA= +stash.appscode.dev/apimachinery v0.17.1-0.20220210134237-79d844fbde2c h1:Lyyp0gQq9VhLIml9myRxC92VovU3oNvz/s0WsYxpxrE= +stash.appscode.dev/apimachinery v0.17.1-0.20220210134237-79d844fbde2c/go.mod h1:MDzqJ66A2QZKAHRksfHT5crOD29a0S5Hfuy/D5hHAjw= diff --git a/pkg/controller/init_container.go b/pkg/controller/init_container.go index 988e25961..3e0651804 100644 --- a/pkg/controller/init_container.go +++ b/pkg/controller/init_container.go @@ -157,7 +157,7 @@ func (c *StashController) handleInitContainerInjectionFailure(w *wapi.Workload, klog.Warningf("Failed to inject stash init-container into %s %s/%s. Reason: %v", w.Kind, w.Namespace, w.Name, err) // Set "StashInitContainerInjected" condition to "False" - cerr := conditions.SetInitContainerInjectedConditionToFalse(inv, ref, err) + cerr := conditions.SetInitContainerInjectedConditionToFalse(inv, &ref, err) // write event to respective resource _, err2 := eventer.CreateEvent( @@ -175,7 +175,7 @@ func (c *StashController) handleInitContainerInjectionSuccess(w *wapi.Workload, klog.Infof("Successfully injected stash init-container into %s %s/%s.", w.Kind, w.Namespace, w.Name) // Set "StashInitContainerInjected" condition to "True" - cerr := conditions.SetInitContainerInjectedConditionToTrue(inv, ref) + cerr := conditions.SetInitContainerInjectedConditionToTrue(inv, &ref) // write event to respective resource _, err2 := eventer.CreateEvent( diff --git a/pkg/controller/repository.go b/pkg/controller/repository.go index 891ca0c90..158d9c752 100644 --- a/pkg/controller/repository.go +++ b/pkg/controller/repository.go @@ -225,7 +225,7 @@ func (c *StashController) requeueRepositoryReferences(repository *api.Repository c.bcQueue.GetQueue().Add(key) default: - return fmt.Errorf("Reference kind %q is unknown", ref.Kind) + return fmt.Errorf("reference kind %q is unknown", ref.Kind) } } diff --git a/pkg/controller/restore_session.go b/pkg/controller/restore_session.go index 6b86515b5..745c4f22f 100644 --- a/pkg/controller/restore_session.go +++ b/pkg/controller/restore_session.go @@ -33,7 +33,6 @@ import ( "stash.appscode.dev/apimachinery/pkg/invoker" "stash.appscode.dev/apimachinery/pkg/metrics" api_util "stash.appscode.dev/apimachinery/pkg/util" - "stash.appscode.dev/stash/pkg/eventer" stash_rbac "stash.appscode.dev/stash/pkg/rbac" "stash.appscode.dev/stash/pkg/resolve" "stash.appscode.dev/stash/pkg/util" @@ -45,7 +44,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" kmapi "kmodules.xyz/client-go/api/v1" batch_util "kmodules.xyz/client-go/batch/v1" @@ -217,24 +215,20 @@ func (c *StashController) applyRestoreInvokerReconciliationLogic(inv invoker.Res // ================= Don't Process Completed Invoker =========================== status := inv.GetStatus() - if status.Phase == api_v1beta1.RestoreFailed || - status.Phase == api_v1beta1.RestoreSucceeded || - status.Phase == api_v1beta1.RestorePhaseUnknown { + if invoker.IsRestoreCompleted(status.Phase) { klog.Infof("Skipping processing %s %s/%s. Reason: phase is %q.", inv.GetTypeMeta().Kind, invMeta.Namespace, invMeta.Name, status.Phase, ) + err = c.ensureMetricsPushed(inv) + if err != nil { + return conditions.SetMetricsPushedConditionToFalse(inv, nil, err) + } return nil } - // ensure that target phases are up-to-date - err = c.ensureRestoreTargetPhases(inv) - if err != nil { - return err - } - // ======================== Set Global Conditions ============================ if inv.GetDriver() == api_v1beta1.ResticSnapshotter { // Check whether Repository exist or not @@ -317,57 +311,31 @@ func (c *StashController) applyRestoreInvokerReconciliationLogic(inv invoker.Res } } } - - // check whether restore process has completed or running and set its phase accordingly - phase, err := c.getRestorePhase(inv.GetStatus()) + phase := inv.GetStatus().Phase // ==================== Execute Global PostRestore Hooks =========================== // if the restore process has completed(Failed or Succeeded or Unknown), then execute global postRestore hook if not yet executed - if restoreCompleted(phase) && !globalPostRestoreHookExecuted(inv) { - hookErr := util.ExecuteHook(c.clientConfig, inv.GetGlobalHooks(), apis.PostRestoreHook, os.Getenv("MY_POD_NAME"), os.Getenv("MY_POD_NAMESPACE")) - if hookErr != nil { - condErr := conditions.SetGlobalPostRestoreHookSucceededConditionToFalse(inv, hookErr) - // set restore phase failed - return c.setRestorePhaseFailed(inv, errors.NewAggregate([]error{err, hookErr, condErr})) - } - condErr := conditions.SetGlobalPostRestoreHookSucceededConditionToTrue(inv) - if condErr != nil { - return condErr + if invoker.IsRestoreCompleted(phase) && !globalPostRestoreHookExecuted(inv) { + err = util.ExecuteHook(c.clientConfig, inv.GetGlobalHooks(), apis.PostRestoreHook, os.Getenv("MY_POD_NAME"), os.Getenv("MY_POD_NAMESPACE")) + if err != nil { + return conditions.SetGlobalPostRestoreHookSucceededConditionToFalse(inv, err) } - } - - // ==================== Set Restore Invoker Phase ====================================== - if phase == api_v1beta1.RestorePhaseInvalid { - klog.Infof("Skipping processing invalid Restoresession.") - - if inv.GetStatus().Phase == api_v1beta1.RestorePhaseInvalid { - return nil + err = conditions.SetGlobalPostRestoreHookSucceededConditionToTrue(inv) + if err != nil { + return err } - return c.setRestorePhaseInvalid(inv) - } else if phase == api_v1beta1.RestoreFailed { - // one or more target has failed to complete their restore process. - // mark entire restore process as failure. - // now, set restore phase "Failed", create an event. and send respective metrics. - return c.setRestorePhaseFailed(inv, err) - } else if phase == api_v1beta1.RestorePhaseUnknown { - return c.setRestorePhaseUnknown(inv, err) - } else if phase == api_v1beta1.RestoreSucceeded { - // all targets has completed their restore process successfully. - // now, set restore phase "Succeeded", create an event, and send respective metrics . - return c.setRestorePhaseSucceeded(inv) } // ==================== Execute Global PreRestore Hook ===================== // if global preRestore hook exist and not executed yet, then execute the preRestoreHook if !globalPreRestoreHookExecuted(inv) { - hookErr := util.ExecuteHook(c.clientConfig, inv.GetGlobalHooks(), apis.PreBackupHook, os.Getenv("MY_POD_NAME"), os.Getenv("MY_POD_NAMESPACE")) - if hookErr != nil { - condErr := conditions.SetGlobalPreRestoreHookSucceededConditionToFalse(inv, hookErr) - return c.setRestorePhaseFailed(inv, errors.NewAggregate([]error{hookErr, condErr})) + err = util.ExecuteHook(c.clientConfig, inv.GetGlobalHooks(), apis.PreBackupHook, os.Getenv("MY_POD_NAME"), os.Getenv("MY_POD_NAMESPACE")) + if err != nil { + return conditions.SetGlobalPreRestoreHookSucceededConditionToFalse(inv, err) } - condErr := conditions.SetGlobalPreRestoreHookSucceededConditionToTrue(inv) - if condErr != nil { - return condErr + err = conditions.SetGlobalPreRestoreHookSucceededConditionToTrue(inv) + if err != nil { + return err } } @@ -381,12 +349,6 @@ func (c *StashController) applyRestoreInvokerReconciliationLogic(inv invoker.Res // ----------------- Ensure Execution Order ------------------- if inv.GetExecutionOrder() == api_v1beta1.Sequential && !inv.NextInOrder(targetInfo.Target.Ref, inv.GetStatus().TargetStatus) { - // restore order is sequential and the current target is not yet to be executed. - // so, set its phase to "Pending". - err = c.setRestoreTargetPhasePending(inv, i) - if err != nil { - return err - } continue } @@ -409,9 +371,7 @@ func (c *StashController) applyRestoreInvokerReconciliationLogic(inv invoker.Res tref.Name, err.Error(), ) - // Set the "RestoreTargetFound" condition to "Unknown" - cerr := conditions.SetRestoreTargetFoundConditionToUnknown(inv, i, err) - return errors.NewAggregate([]error{err, cerr}) + return conditions.SetRestoreTargetFoundConditionToUnknown(inv, i, err) } if !targetExist { @@ -455,47 +415,40 @@ func (c *StashController) applyRestoreInvokerReconciliationLogic(inv invoker.Res tref.Name, ) if err != nil { - return c.handleWorkloadControllerTriggerFailure(invokerRef, err) + msg := fmt.Sprintf("failed to trigger workload controller for %s %s/%s. Reason: %v", tref.Kind, invMeta.Namespace, tref.Name, err) + klog.Warning(msg) + return conditions.SetRestorerEnsuredToFalse(inv, &tref, msg) } case RestorerCSIDriver: // VolumeSnapshotter driver has been used. So, ensure VolumeRestorer job err := c.ensureVolumeRestorerJob(inv, i) if err != nil { - // Failed to ensure VolumeRestorer job. So, set "RestoreJobCreated" condition to "False" - cerr := conditions.SetRestoreJobCreatedConditionToFalse(inv, &tref, err) - return c.handleRestoreJobCreationFailure(inv, errors.NewAggregate([]error{err, cerr})) - } - // Successfully created VolumeRestorer job. So, set "RestoreJobCreated" condition to "True" - cerr := conditions.SetRestoreJobCreatedConditionToTrue(inv, &tref) - if cerr != nil { - return cerr + msg := fmt.Sprintf("failed to ensure volume snapshotter job for %s %s/%s. Reason: %v", tref.Kind, invMeta.Namespace, tref.Name, err) + klog.Warning(msg) + return conditions.SetRestorerEnsuredToFalse(inv, &tref, msg) } case RestorerJob: // Restic driver has been used. Ensure restore job. err = c.ensureRestoreJob(inv, i) if err != nil { - // Failed to ensure restorer job. So, set "RestoreJobCreated" condition to "False" - cerr := conditions.SetRestoreJobCreatedConditionToFalse(inv, &tref, err) - // Set RestoreSession phase "Failed" and send prometheus metrics. - return c.handleRestoreJobCreationFailure(inv, errors.NewAggregate([]error{err, cerr})) - } - err = conditions.SetRestoreJobCreatedConditionToTrue(inv, &tref) - if err != nil { - return err + msg := fmt.Sprintf("failed to ensure restore job for %s %s/%s. Reason: %v", tref.Kind, invMeta.Namespace, tref.Name, err) + klog.Warning(msg) + return conditions.SetRestorerEnsuredToFalse(inv, &tref, msg) } default: - return fmt.Errorf("unable to idenitfy restorer entity") + msg := fmt.Sprintf("unable to identify restorer entity for target %s %s/%s", tref.Kind, invMeta.Namespace, tref.Name) + klog.Warning(msg) + return conditions.SetRestorerEnsuredToFalse(inv, &tref, msg) } - // Set target phase "Running" - err = c.setRestoreTargetPhaseRunning(inv, i) + msg := fmt.Sprintf("Restorer job/init-container has been ensured successfully for %s %s/%s.", tref.Kind, invMeta.Namespace, tref.Name) + err = conditions.SetRestorerEnsuredToTrue(inv, &tref, msg) if err != nil { return err } + return c.initiateTargetRestore(inv, i) } } - - // Restorer entity has been ensured. Set RestoreSession phase to "Running". - return c.setRestorePhaseRunning(inv) + return nil } func (c *StashController) ensureRestoreJob(inv invoker.RestoreInvoker, index int) error { invMeta := inv.GetObjectMeta() @@ -850,289 +803,6 @@ func (c *StashController) ensureVolumeRestorerJob(inv invoker.RestoreInvoker, in return err } -func (c *StashController) setRestorePhaseRunning(inv invoker.RestoreInvoker) error { - // update restore invoker status - err := inv.UpdateStatus(invoker.RestoreInvokerStatus{ - Phase: api_v1beta1.RestoreRunning, - }) - if err != nil { - return err - } - - // crate event against the restore invoker - err = inv.CreateEvent( - core.EventTypeNormal, - "", - eventer.EventReasonRestoreRunning, - fmt.Sprintf("restore has been started for %s %s/%s", - inv.GetTypeMeta().Kind, - inv.GetObjectMeta().Namespace, - inv.GetObjectMeta().Name, - ), - ) - return err -} - -func (c *StashController) setRestorePhaseSucceeded(inv invoker.RestoreInvoker) error { - var err error - invMeta := inv.GetObjectMeta() - // total restore session duration is the difference between the time when restore invoker was created and when it completed - sessionDuration := time.Since(invMeta.CreationTimestamp.Time) - // update restore invoker status - err = inv.UpdateStatus(invoker.RestoreInvokerStatus{ - Phase: api_v1beta1.RestoreSucceeded, - SessionDuration: sessionDuration.Round(time.Second).String(), - }) - if err != nil { - return err - } - - // crate event against the restore invoker - err = inv.CreateEvent( - core.EventTypeNormal, - "", - eventer.EventReasonRestoreSucceeded, - "Restore has been completed successfully", - ) - - // if there is any error during writing the event, log i. we have to send metrics even if we fail to write the event. - if err != nil { - klog.Errorf("failed to write event in %s %s/%s. Reason: %v", - inv.GetTypeMeta().Kind, - invMeta.Namespace, - invMeta.Name, - err, - ) - } - // send restore metrics - metricsOpt := &metrics.MetricsOptions{ - Enabled: true, - PushgatewayURL: metrics.GetPushgatewayURL(), - JobName: fmt.Sprintf("%s-%s-%s", strings.ToLower(inv.GetTypeMeta().Kind), invMeta.Namespace, invMeta.Name), - } - // send target specific metrics - for _, target := range inv.GetStatus().TargetStatus { - metricErr := metricsOpt.SendRestoreTargetMetrics(c.clientConfig, inv, target.Ref) - if err != nil { - return metricErr - } - } - // send restore session metrics - return metricsOpt.SendRestoreSessionMetrics(inv) -} - -func (c *StashController) setRestorePhaseFailed(inv invoker.RestoreInvoker, restoreErr error) error { - var err error - invMeta := inv.GetObjectMeta() - // total restore session duration is the difference between the time when restore invoker was created and when it completed - sessionDuration := time.Since(invMeta.CreationTimestamp.Time) - - // update restore invoker status - err = inv.UpdateStatus(invoker.RestoreInvokerStatus{ - Phase: api_v1beta1.RestoreFailed, - SessionDuration: sessionDuration.Round(time.Second).String(), - }) - if err != nil { - return err - } - - // crate event against the restore invoker - err = inv.CreateEvent( - core.EventTypeWarning, - "", - eventer.EventReasonRestoreFailed, - fmt.Sprintf("Restore has failed to complete. Reason: %v", restoreErr), - ) - - // if there is any error during writing the event, log i. we have to send metrics even if we fail to write the event. - if err != nil { - klog.Errorf("failed to write event in %s %s/%s. Reason: %v", - inv.GetTypeMeta().Kind, - invMeta.Namespace, - invMeta.Name, - err, - ) - } - // send restore metrics - metricsOpt := &metrics.MetricsOptions{ - Enabled: true, - PushgatewayURL: metrics.GetPushgatewayURL(), - JobName: fmt.Sprintf("%s-%s-%s", strings.ToLower(inv.GetTypeMeta().Kind), invMeta.Namespace, invMeta.Name), - } - // send target specific metrics - for _, target := range inv.GetStatus().TargetStatus { - metricErr := metricsOpt.SendRestoreTargetMetrics(c.clientConfig, inv, target.Ref) - if err != nil { - return errors.NewAggregate([]error{restoreErr, metricErr}) - } - } - // send restore session metrics - err = metricsOpt.SendRestoreSessionMetrics(inv) - return errors.NewAggregate([]error{restoreErr, err}) -} - -func (c *StashController) setRestorePhaseInvalid(inv invoker.RestoreInvoker) error { - return inv.UpdateStatus(invoker.RestoreInvokerStatus{ - Phase: api_v1beta1.RestorePhaseInvalid, - }) -} - -func (c *StashController) setRestorePhaseUnknown(inv invoker.RestoreInvoker, restoreErr error) error { - var err error - invMeta := inv.GetObjectMeta() - // total restore session duration is the difference between the time when restore invoker was created and when it completed - sessionDuration := time.Since(invMeta.CreationTimestamp.Time) - - // update restore invoker status - err = inv.UpdateStatus(invoker.RestoreInvokerStatus{ - Phase: api_v1beta1.RestorePhaseUnknown, - SessionDuration: sessionDuration.Round(time.Second).String(), - }) - if err != nil { - return err - } - - // crate event against the restore invoker - err = inv.CreateEvent( - core.EventTypeWarning, - "", - eventer.EventReasonRestorePhaseUnknown, - fmt.Sprintf("Unable to ensure whether restore has completed or not. Reason: %v", restoreErr), - ) - - // if there is any error during writing the event, log i. we have to send metrics even if we fail to write the event. - if err != nil { - klog.Errorf("failed to write event in %s %s/%s. Reason: %v", - inv.GetTypeMeta().Kind, - invMeta.Namespace, - invMeta.Name, - err, - ) - } - // send restore metrics - metricsOpt := &metrics.MetricsOptions{ - Enabled: true, - PushgatewayURL: metrics.GetPushgatewayURL(), - JobName: fmt.Sprintf("%s-%s-%s", strings.ToLower(inv.GetTypeMeta().Kind), invMeta.Namespace, invMeta.Name), - } - // send target specific metrics - for _, target := range inv.GetStatus().TargetStatus { - metricErr := metricsOpt.SendRestoreTargetMetrics(c.clientConfig, inv, target.Ref) - if err != nil { - return errors.NewAggregate([]error{restoreErr, metricErr}) - } - } - // send restore session metrics - err = metricsOpt.SendRestoreSessionMetrics(inv) - return errors.NewAggregate([]error{restoreErr, err}) -} - -func (c *StashController) setRestoreTargetPhasePending(inv invoker.RestoreInvoker, index int) error { - targetInfo := inv.GetTargetInfo()[index] - err := inv.UpdateStatus(invoker.RestoreInvokerStatus{ - TargetStatus: []api_v1beta1.RestoreMemberStatus{ - { - Ref: targetInfo.Target.Ref, - Phase: api_v1beta1.TargetRestorePending, - }, - }, - }) - return err -} - -func (c *StashController) setRestoreTargetPhaseRunning(inv invoker.RestoreInvoker, index int) error { - targetInfo := inv.GetTargetInfo()[index] - totalHosts, err := c.getTotalHosts(targetInfo.Target, inv.GetObjectMeta().Namespace, inv.GetDriver()) - if err != nil { - return err - } - err = inv.UpdateStatus(invoker.RestoreInvokerStatus{ - TargetStatus: []api_v1beta1.RestoreMemberStatus{ - { - Ref: targetInfo.Target.Ref, - TotalHosts: totalHosts, - Phase: api_v1beta1.TargetRestoreRunning, - }, - }, - }) - return err -} - -func (c *StashController) getRestorePhase(status invoker.RestoreInvokerStatus) (api_v1beta1.RestorePhase, error) { - if kmapi.IsConditionFalse(status.Conditions, apis.ValidationPassed) { - return api_v1beta1.RestorePhaseInvalid, nil - } - - // If the Phase is empty or "Pending" then return it. controller will process accordingly - if status.Phase == "" || status.Phase == api_v1beta1.RestorePending { - return api_v1beta1.RestorePending, nil - } - - // If any of the target fail, then mark the entire restore process as "Failed". - // Mark the entire restore process "Succeeded" only and if only the restore of all targets has succeeded. - // Otherwise, mark the restore process as "Running". - completedTargets := 0 - var errList []error - for _, target := range status.TargetStatus { - if target.Phase == api_v1beta1.TargetRestoreSucceeded || - target.Phase == api_v1beta1.TargetRestoreFailed { - completedTargets = completedTargets + 1 - } - if target.Phase == api_v1beta1.TargetRestoreFailed { - errList = append(errList, fmt.Errorf("restore failed for target: %s/%s", target.Ref.Kind, target.Ref.Name)) - } - if target.Phase == api_v1beta1.TargetRestoreRunning { - return api_v1beta1.RestoreRunning, nil - } - } - - if errList != nil { - return api_v1beta1.RestoreFailed, errors.NewAggregate(errList) - } - - // check if any of the target phase is "Unknown". if any of their phase is "Unknown", then consider entire restore process phase is unknown. - for _, target := range status.TargetStatus { - if target.Phase == api_v1beta1.TargetRestorePhaseUnknown { - return api_v1beta1.RestorePhaseUnknown, fmt.Errorf("restore phase is 'Unknown' for target: %s/%s", target.Ref.Kind, target.Ref.Name) - } - } - - if completedTargets != len(status.TargetStatus) { - return api_v1beta1.RestoreRunning, nil - } - - // Restore has been completed successfully for all targets. - return api_v1beta1.RestoreSucceeded, nil -} - -func (c *StashController) handleRestoreJobCreationFailure(inv invoker.RestoreInvoker, restoreErr error) error { - invMeta := inv.GetObjectMeta() - klog.Errorf("failed to ensure restore job for %s %s/%s. Reason: %v", - inv.GetTypeMeta().Kind, - invMeta.Namespace, - invMeta.Name, - restoreErr, - ) - - // write event to RestoreSession - err := inv.CreateEvent( - core.EventTypeWarning, - "", - eventer.EventReasonRestoreJobCreationFailed, - fmt.Sprintf("failed to create restore job. Reason: %v", restoreErr), - ) - if err != nil { - klog.Errorf("failed to write event for %s %s/%s. Reason: ", - inv.GetTypeMeta().Kind, - invMeta.Namespace, - invMeta.Name, - ) - } - - // set RestoreSession phase failed - return c.setRestorePhaseFailed(inv, restoreErr) -} - func getRestoreJobName(invokerMeta metav1.ObjectMeta, suffix string) string { return meta.ValidNameWithPrefixNSuffix(apis.PrefixStashRestore, strings.ReplaceAll(invokerMeta.Name, ".", "-"), suffix) } @@ -1155,17 +825,11 @@ func (c *StashController) restorerEntity(ref api_v1beta1.TargetRef, driver api_v } } -func restoreCompleted(phase api_v1beta1.RestorePhase) bool { - return phase == api_v1beta1.RestoreFailed || - phase == api_v1beta1.RestoreSucceeded || - phase == api_v1beta1.RestorePhaseUnknown -} - func (c *StashController) requeueRestoreInvoker(inv invoker.RestoreInvoker, key string) error { invTypeMeta := inv.GetTypeMeta() switch invTypeMeta.Kind { case api_v1beta1.ResourceKindRestoreSession: - c.rsQueue.GetQueue().AddAfter(key, requeueTimeInterval) + c.restoreSessionQueue.GetQueue().AddAfter(key, requeueTimeInterval) default: return fmt.Errorf("unable to requeue. Reason: Restore invoker %s %s is not supported", invTypeMeta.APIVersion, @@ -1175,67 +839,6 @@ func (c *StashController) requeueRestoreInvoker(inv invoker.RestoreInvoker, key return nil } -func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker) error { - status := inv.GetStatus() - // Skipping phase calculation here if the restoresession is invalid - if kmapi.IsConditionTrue(status.Conditions, apis.RepositoryFound) && - kmapi.IsConditionTrue(status.Conditions, apis.BackendSecretFound) && - kmapi.IsConditionFalse(status.Conditions, apis.ValidationPassed) { - return nil - } - - targetStats := status.TargetStatus - for i, target := range status.TargetStatus { - if target.TotalHosts == nil { - targetStats[i].Phase = api_v1beta1.TargetRestorePending - continue - } - // check if any host failed to restore, running, or it's phase 'Unknown' - anyHostFailed := false - anyHostPhaseUnknown := false - anyHostRunning := false - for _, hostStats := range target.Stats { - if hostStats.Phase == api_v1beta1.HostRestoreFailed { - anyHostFailed = true - break - } - if hostStats.Phase == api_v1beta1.HostRestoreUnknown { - anyHostPhaseUnknown = true - break - } - if hostStats.Phase == api_v1beta1.HostRestoreRunning { - anyHostRunning = true - break - } - } - // if any host fail to restore, the overall target phase should be "Failed" - if anyHostFailed { - targetStats[i].Phase = api_v1beta1.TargetRestoreFailed - continue - } - // if any host's restore phase is 'Unknown', the overall target phase should be "Unknown" - if anyHostPhaseUnknown { - targetStats[i].Phase = api_v1beta1.TargetRestorePhaseUnknown - continue - } - // Restore should be succeeded only if all the conditions of this restore target are true - allConditionTrue := true - for _, c := range target.Conditions { - if c.Status == core.ConditionFalse { - allConditionTrue = false - break - } - } - // if some host hasn't completed their restore yet, phase should be "Running" - if !anyHostRunning && *target.TotalHosts <= int32(len(target.Stats)) && allConditionTrue { - targetStats[i].Phase = api_v1beta1.TargetRestoreSucceeded - continue - } - targetStats[i].Phase = api_v1beta1.TargetRestoreRunning - } - return inv.UpdateStatus(invoker.RestoreInvokerStatus{TargetStatus: targetStats}) -} - func globalPostRestoreHookExecuted(inv invoker.RestoreInvoker) bool { if inv.GetGlobalHooks() == nil || inv.GetGlobalHooks().PostRestore == nil { return true @@ -1253,16 +856,18 @@ func globalPreRestoreHookExecuted(inv invoker.RestoreInvoker) bool { } func targetRestoreInitiated(inv invoker.RestoreInvoker, targetRef api_v1beta1.TargetRef) bool { - for _, target := range inv.GetStatus().TargetStatus { + status := inv.GetStatus() + if invoker.TargetRestoreCompleted(targetRef, status.TargetStatus) { + return true + } + for _, target := range status.TargetStatus { if invoker.TargetMatched(target.Ref, targetRef) { - return target.Phase == api_v1beta1.TargetRestoreRunning || - target.Phase == api_v1beta1.TargetRestoreSucceeded || - target.Phase == api_v1beta1.TargetRestoreFailed || - target.Phase == api_v1beta1.TargetRestorePhaseUnknown + return target.Phase == api_v1beta1.TargetRestoreRunning } } return false } + func (c *StashController) getRestoreRBACOptions(inv invoker.RestoreInvoker) (stash_rbac.RBACOptions, error) { invMeta := inv.GetObjectMeta() repo := inv.GetRepoRef() @@ -1297,3 +902,49 @@ func (c *StashController) getRestoreRBACOptions(inv invoker.RestoreInvoker) (sta } return rbacOptions, nil } + +func (c *StashController) initiateTargetRestore(inv invoker.RestoreInvoker, index int) error { + targetInfo := inv.GetTargetInfo()[index] + totalHosts, err := c.getTotalHosts(targetInfo.Target, inv.GetObjectMeta().Namespace, inv.GetDriver()) + if err != nil { + return err + } + return inv.UpdateStatus(invoker.RestoreInvokerStatus{ + TargetStatus: []api_v1beta1.RestoreMemberStatus{ + { + Ref: targetInfo.Target.Ref, + TotalHosts: totalHosts, + }, + }, + }) +} + +func (c *StashController) ensureMetricsPushed(inv invoker.RestoreInvoker) error { + metricsPushed, err := inv.IsConditionTrue(nil, apis.MetricsPushed) + if err != nil { + return err + } + if metricsPushed { + return nil + } + + // send restore metrics + metricsOpt := &metrics.MetricsOptions{ + Enabled: true, + PushgatewayURL: metrics.GetPushgatewayURL(), + JobName: fmt.Sprintf("%s-%s-%s", strings.ToLower(inv.GetTypeMeta().Kind), inv.GetObjectMeta().Namespace, inv.GetObjectMeta().Name), + } + // send target specific metrics + for _, target := range inv.GetStatus().TargetStatus { + err = metricsOpt.SendRestoreTargetMetrics(c.clientConfig, inv, target.Ref) + if err != nil { + return err + } + } + // send restore session metrics + err = metricsOpt.SendRestoreSessionMetrics(inv) + if err != nil { + return err + } + return conditions.SetMetricsPushedConditionToTrue(inv, nil) +} diff --git a/pkg/controller/usage_policy.go b/pkg/controller/usage_policy.go index fe7e39679..f2d088295 100644 --- a/pkg/controller/usage_policy.go +++ b/pkg/controller/usage_policy.go @@ -55,7 +55,7 @@ func (c *StashController) validateAgainstUsagePolicy(repo kmapi.ObjectReference, } if !repository.UsageAllowed(namespace) { - return fmt.Errorf("Namespace %q is not allowed to refer Repository %q of %q namespace. Please, check the `usagePolicy` of the Repository.", curNamespace, repo.Name, repo.Namespace) + return fmt.Errorf("namespace %q is not allowed to refer Repository %q of %q namespace. Please, check the `usagePolicy` of the Repository", curNamespace, repo.Name, repo.Namespace) } return nil diff --git a/test/e2e/framework/backup_configuration.go b/test/e2e/framework/backup_configuration.go index 66cceefef..d9bd3b313 100644 --- a/test/e2e/framework/backup_configuration.go +++ b/test/e2e/framework/backup_configuration.go @@ -24,6 +24,7 @@ import ( "stash.appscode.dev/apimachinery/apis" "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" + "stash.appscode.dev/apimachinery/pkg/invoker" . "github.com/onsi/gomega" "gomodules.xyz/x/crypto/rand" @@ -149,3 +150,25 @@ func getBackupCronJobName(objMeta metav1.ObjectMeta) string { func getBackupJobName(backupSessionName string, index string) string { return meta_util.ValidNameWithPrefixNSuffix(apis.PrefixStashBackup, strings.ReplaceAll(backupSessionName, ".", "-"), index) } + +func (f *Framework) EventuallyBackupInvokerPhase(invoker invoker.BackupInvoker) GomegaAsyncAssertion { + return Eventually( + func() v1beta1.BackupInvokerPhase { + switch invoker.GetTypeMeta().Kind { + case v1beta1.ResourceKindBackupConfiguration: + bc, err := f.StashClient.StashV1beta1().BackupConfigurations(invoker.GetObjectMeta().Namespace).Get(context.TODO(), invoker.GetObjectMeta().Name, metav1.GetOptions{}) + if err != nil { + return "" + } + return bc.Status.Phase + case v1beta1.ResourceKindBackupBatch: + bb, err := f.StashClient.StashV1beta1().BackupBatches(invoker.GetObjectMeta().Namespace).Get(context.TODO(), invoker.GetObjectMeta().Name, metav1.GetOptions{}) + if err != nil { + return "" + } + return bb.Status.Phase + default: + return "" + } + }, WaitTimeOut, PullInterval) +} diff --git a/test/e2e/framework/common.go b/test/e2e/framework/common.go index 5141d4351..a143d84bf 100644 --- a/test/e2e/framework/common.go +++ b/test/e2e/framework/common.go @@ -24,6 +24,7 @@ import ( api "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" api_v1alpha1 "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" + invoker2 "stash.appscode.dev/apimachinery/pkg/invoker" . "stash.appscode.dev/stash/test/e2e/matcher" . "github.com/onsi/ginkgo" @@ -73,8 +74,9 @@ func (fi *Invocation) SetupWorkloadBackup(objMeta metav1.ObjectMeta, repo *api_v backupConfig, err := fi.CreateBackupConfigForWorkload(objMeta, repo, kind, transformFuncs...) Expect(err).NotTo(HaveOccurred()) - By("Verifying that backup triggering CronJob has been created") - fi.EventuallyCronJobCreated(backupConfig.ObjectMeta).Should(BeTrue()) + By("Verifying that backup invoker is ready") + inv := invoker2.NewBackupConfigurationInvoker(fi.StashClient, backupConfig) + fi.EventuallyBackupInvokerPhase(inv).Should(BeEquivalentTo(v1beta1.BackupInvokerReady)) By("Verifying that sidecar has been injected") switch kind { @@ -247,7 +249,9 @@ func (fi *Invocation) CreateRestoreSessionForWorkload(objMeta metav1.ObjectMeta, By("Creating RestoreSession") err := fi.CreateRestoreSession(restoreSession) - Expect(err).NotTo(HaveOccurred()) + if err != nil { + return nil, err + } fi.AppendToCleanupList(restoreSession) return restoreSession, nil diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index c25561640..6b9b0bba5 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -27,6 +27,7 @@ import ( "gomodules.xyz/pointer" "gomodules.xyz/x/crypto/rand" apps "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" kerr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -113,21 +114,8 @@ func (fi *Invocation) DeployDeployment(name string, replica int32, volName strin // append test case specific suffix so that name does not conflict during parallel test pvcName := fmt.Sprintf("%s-%s", volName, fi.app) - // If the PVC does not exist, create PVC for Deployment - pvc, err := fi.KubeClient.CoreV1().PersistentVolumeClaims(fi.namespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) - if err != nil { - if kerr.IsNotFound(err) { - pvc, err = fi.CreateNewPVC(pvcName) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - // Generate Deployment definition - deployment := fi.Deployment(name, pvc.Name, volName) + deployment := fi.Deployment(name, pvcName, volName) deployment.Spec.Replicas = &replica // transformFuncs provides a array of functions that made test specific change on the Deployment @@ -136,6 +124,14 @@ func (fi *Invocation) DeployDeployment(name string, replica int32, volName strin fn(&deployment) } + // If the PVC does not exist, create PVC for Deployment + _, err := fi.CreateNewPVC(pvcName, func(p *core.PersistentVolumeClaim) { + p.Namespace = deployment.Namespace + }) + if err != nil && !kerr.IsAlreadyExists(err) { + return nil, err + } + By("Deploying Deployment: " + deployment.Name) createdDeployment, err := fi.CreateDeployment(deployment) if err != nil { diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 33d5e0f62..a45ed9709 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ka "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "kmodules.xyz/client-go/meta" appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" ) @@ -81,11 +82,14 @@ func NewInvocation() *Invocation { } func (f *Framework) Invoke() *Invocation { - return &Invocation{ + inv := &Invocation{ Framework: f, app: rand.WithUniqSuffix("stash-e2e"), testResources: make([]interface{}, 0), } + inv.backupNamespace = meta.NameWithSuffix(inv.app, "backup") + inv.restoreNamespace = meta.NameWithSuffix(inv.app, "restore") + return inv } func (fi *Invocation) AppLabel() string { @@ -96,8 +100,18 @@ func (fi *Invocation) App() string { return fi.app } +func (fi *Invocation) BackupNamespace() string { + return fi.backupNamespace +} + +func (fi *Invocation) RestoreNamespace() string { + return fi.restoreNamespace +} + type Invocation struct { *Framework - app string - testResources []interface{} + app string + backupNamespace string + restoreNamespace string + testResources []interface{} } diff --git a/test/e2e/framework/pvc.go b/test/e2e/framework/pvc.go index 5e5577797..f422b1fcb 100644 --- a/test/e2e/framework/pvc.go +++ b/test/e2e/framework/pvc.go @@ -58,10 +58,14 @@ func (f *Framework) CreatePersistentVolumeClaim(pvc *core.PersistentVolumeClaim) return f.KubeClient.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}) } -func (fi *Invocation) CreateNewPVC(name string) (*core.PersistentVolumeClaim, error) { +func (fi *Invocation) CreateNewPVC(name string, transformFuncs ...func(p *core.PersistentVolumeClaim)) (*core.PersistentVolumeClaim, error) { // Generate PVC definition pvc := fi.PersistentVolumeClaim(name) + for _, fn := range transformFuncs { + fn(pvc) + } + By(fmt.Sprintf("Creating PVC: %s/%s", pvc.Namespace, pvc.Name)) createdPVC, err := fi.CreatePersistentVolumeClaim(pvc) if err != nil { diff --git a/test/e2e/framework/repository.go b/test/e2e/framework/repository.go index 002ae715a..b6c770786 100644 --- a/test/e2e/framework/repository.go +++ b/test/e2e/framework/repository.go @@ -23,12 +23,14 @@ import ( "stash.appscode.dev/apimachinery/apis" "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" + v1alpha1_util "stash.appscode.dev/apimachinery/client/clientset/versioned/typed/stash/v1alpha1/util" "stash.appscode.dev/stash/pkg/util" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "gomodules.xyz/stow" "gomodules.xyz/x/crypto/rand" + core "k8s.io/api/core/v1" kerr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "kmodules.xyz/client-go/tools/portforward" @@ -345,17 +347,19 @@ func (fi *Invocation) SetupMinioRepository(transformFuncs ...func(repo *v1alpha1 for _, fn := range transformFuncs { fn(repo) } - // Create Repository + return fi.CreateRepository(repo, &cred) +} + +func (fi *Invocation) CreateRepository(repo *v1alpha1.Repository, cred *core.Secret) (*v1alpha1.Repository, error) { By("Creating Repository") - repo, err = fi.StashClient.StashV1alpha1().Repositories(repo.Namespace).Create(context.TODO(), repo, metav1.CreateOptions{}) + repo, err := fi.StashClient.StashV1alpha1().Repositories(repo.Namespace).Create(context.TODO(), repo, metav1.CreateOptions{}) if err != nil { return repo, err } // If `.spec.wipeOut` is set to "true", then the corresponding Secret is required to delete the remote repository. // Hence we need to delete the Repository object first. - fi.AppendToCleanupList(repo, &cred) - + fi.AppendToCleanupList(repo, cred) return repo, nil } @@ -544,3 +548,15 @@ func (fi *Invocation) SetupB2Repository(maxConnection int64, transformFuncs ...f return repo, nil } + +func (fi *Invocation) AllowNamespace(repo *v1alpha1.Repository, allowedNamespace v1alpha1.FromNamespaces) (*v1alpha1.Repository, error) { + repo, _, err := v1alpha1_util.PatchRepository(context.TODO(), fi.StashClient.StashV1alpha1(), repo, func(repository *v1alpha1.Repository) *v1alpha1.Repository { + repository.Spec.UsagePolicy = &v1alpha1.UsagePolicy{ + AllowedNamespaces: v1alpha1.AllowedNamespaces{ + From: &allowedNamespace, + }, + } + return repository + }, metav1.PatchOptions{}) + return repo, err +} diff --git a/test/e2e/framework/restore_session.go b/test/e2e/framework/restore_session.go index 020907bce..6c4872adb 100644 --- a/test/e2e/framework/restore_session.go +++ b/test/e2e/framework/restore_session.go @@ -23,6 +23,7 @@ import ( "stash.appscode.dev/apimachinery/apis" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" + "stash.appscode.dev/apimachinery/pkg/invoker" . "github.com/onsi/gomega" "gomodules.xyz/x/arrays" @@ -72,14 +73,23 @@ func (fi Invocation) DeleteRestoreSession(meta metav1.ObjectMeta) error { func (f *Framework) EventuallyRestoreProcessCompleted(meta metav1.ObjectMeta, invokerKind string) GomegaAsyncAssertion { return Eventually( func() bool { - rs, err := f.StashClient.StashV1beta1().RestoreSessions(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) - if err != nil { - return false + var restorePhase v1beta1.RestorePhase + if invokerKind == v1beta1.ResourceKindRestoreSession { + rs, err := f.StashClient.StashV1beta1().RestoreSessions(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) + if err != nil { + return false + } + restorePhase = rs.Status.Phase + } else { + rb, err := f.StashClient.StashV1beta1().RestoreBatches(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) + if err != nil { + return false + } + restorePhase = rb.Status.Phase } - - if rs.Status.Phase == v1beta1.RestoreSucceeded || - rs.Status.Phase == v1beta1.RestoreFailed || - rs.Status.Phase == v1beta1.RestorePhaseUnknown { + if restorePhase == v1beta1.RestoreSucceeded || + restorePhase == v1beta1.RestoreFailed || + restorePhase == v1beta1.RestorePhaseUnknown { return true } return false @@ -142,3 +152,25 @@ func ruleExist(rule v1beta1.Rule, rules []v1beta1.Rule) bool { } return false } + +func (f *Framework) EventuallyRestoreInvokerPhase(invoker invoker.RestoreInvoker) GomegaAsyncAssertion { + return Eventually( + func() v1beta1.RestorePhase { + switch invoker.GetTypeMeta().Kind { + case v1beta1.ResourceKindRestoreSession: + rs, err := f.StashClient.StashV1beta1().RestoreSessions(invoker.GetObjectMeta().Namespace).Get(context.TODO(), invoker.GetObjectMeta().Name, metav1.GetOptions{}) + if err != nil { + return "" + } + return rs.Status.Phase + case v1beta1.ResourceKindRestoreBatch: + rb, err := f.StashClient.StashV1beta1().RestoreBatches(invoker.GetObjectMeta().Namespace).Get(context.TODO(), invoker.GetObjectMeta().Name, metav1.GetOptions{}) + if err != nil { + return "" + } + return rb.Status.Phase + default: + return "" + } + }, WaitTimeOut, PullInterval) +} diff --git a/test/e2e/misc/cross_namespace.go b/test/e2e/misc/cross_namespace.go new file mode 100644 index 000000000..0616d5f49 --- /dev/null +++ b/test/e2e/misc/cross_namespace.go @@ -0,0 +1,231 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the AppsCode Community License 1.0.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package misc + +import ( + "context" + + "stash.appscode.dev/apimachinery/apis" + "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" + "stash.appscode.dev/apimachinery/apis/stash/v1beta1" + "stash.appscode.dev/apimachinery/pkg/invoker" + "stash.appscode.dev/stash/test/e2e/framework" + . "stash.appscode.dev/stash/test/e2e/matcher" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + allowSameNamespace = v1alpha1.NamespacesFromSame + allowAllNamespaces = v1alpha1.NamespacesFromAll +) + +var _ = Describe("Cross Namespace", func() { + + var f *framework.Invocation + + BeforeEach(func() { + f = framework.NewInvocation() + + By("Creating backup namespace: " + f.BackupNamespace()) + err := f.CreateNamespace(f.NewNamespace(f.BackupNamespace())) + Expect(err).NotTo(HaveOccurred()) + + By("Create restore namespace: " + f.RestoreNamespace()) + err = f.CreateNamespace(f.NewNamespace(f.RestoreNamespace())) + Expect(err).NotTo(HaveOccurred()) + + }) + + JustAfterEach(func() { + f.PrintDebugInfoOnFailure() + }) + + AfterEach(func() { + err := f.CleanupTestResources() + Expect(err).NotTo(HaveOccurred()) + // StatefulSet's PVCs are not get cleanup by the CleanupTestResources() function. + // Hence, we need to cleanup them manually. + f.CleanupUndeletedPVCs() + + By("Deleting namespace: " + f.BackupNamespace()) + err = f.DeleteNamespace(f.BackupNamespace()) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting namespace: " + f.RestoreNamespace()) + err = f.DeleteNamespace(f.RestoreNamespace()) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("No UsagePolicy", func() { + It("should allow BackupConfigurations only from the same namespace", func() { + deployment := f.Deployment(framework.SourceDeployment, framework.SourcePVC, framework.SourceVolume) + + repo, err := f.SetupMinioRepository() + Expect(err).NotTo(HaveOccurred()) + + _, err = f.CreateBackupConfigForWorkload(deployment.ObjectMeta, repo, apis.KindDeployment, func(bc *v1beta1.BackupConfiguration) { + bc.Namespace = repo.Namespace + bc.Spec.Repository.Namespace = f.Namespace() + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should reject BackupConfiguration from different namespaces", func() { + deployment := f.Deployment(framework.SourceDeployment, framework.SourcePVC, framework.SourceVolume) + + repo, err := f.SetupMinioRepository() + Expect(err).NotTo(HaveOccurred()) + + _, err = f.CreateBackupConfigForWorkload(deployment.ObjectMeta, repo, apis.KindDeployment, func(bc *v1beta1.BackupConfiguration) { + bc.Namespace = f.BackupNamespace() + bc.Spec.Repository.Namespace = f.Namespace() + }) + Expect(err).To(HaveOccurred()) + }) + + It("should allow RestoreSession from the same namespace", func() { + deployment := f.Deployment(framework.SourceDeployment, framework.SourcePVC, framework.SourceVolume) + + repo, err := f.SetupMinioRepository() + Expect(err).NotTo(HaveOccurred()) + + _, err = f.CreateRestoreSessionForWorkload(deployment.ObjectMeta, repo.Name, apis.KindDeployment, framework.SourceVolume, func(rs *v1beta1.RestoreSession) { + rs.Namespace = repo.Namespace + rs.Spec.Repository.Namespace = f.Namespace() + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should reject RestoreSession from different namespaces", func() { + deployment := f.Deployment(framework.SourceDeployment, framework.SourcePVC, framework.SourceVolume) + + repo, err := f.SetupMinioRepository() + Expect(err).NotTo(HaveOccurred()) + + _, err = f.CreateRestoreSessionForWorkload(deployment.ObjectMeta, repo.Name, apis.KindDeployment, framework.SourceVolume, func(rs *v1beta1.RestoreSession) { + rs.Namespace = f.RestoreNamespace() + rs.Spec.Repository.Namespace = f.Namespace() + }) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("Repository Created Later", func() { + It("invoker phase should update with Repository changes", func() { + deployment, err := f.DeployDeployment(framework.SourceDeployment, int32(1), framework.SourceVolume, func(dp *apps.Deployment) { + dp.Namespace = f.BackupNamespace() + }) + Expect(err).NotTo(HaveOccurred()) + + sampleData, err := f.GenerateSampleData(deployment.ObjectMeta, apis.KindDeployment) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Storage Secret") + cred := f.SecretForMinioBackend(true) + + if missing, _ := BeZero().Match(cred); missing { + Skip("Missing Minio credential") + } + _, err = f.CreateSecret(cred) + Expect(err).NotTo(HaveOccurred()) + + repo := f.NewMinioRepository(cred.Name) + + backupConfig, err := f.CreateBackupConfigForWorkload(deployment.ObjectMeta, repo, apis.KindDeployment, func(bc *v1beta1.BackupConfiguration) { + bc.Namespace = f.BackupNamespace() + bc.Spec.Repository.Namespace = f.Namespace() + }) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying the backup invoker phase is: NotReady") + inv := invoker.NewBackupConfigurationInvoker(f.StashClient, backupConfig) + f.EventuallyBackupInvokerPhase(inv).Should(BeEquivalentTo(v1beta1.BackupInvokerNotReady)) + + By("Creating Repository not allowing the BackupConfiguration") + repo.Spec.UsagePolicy = &v1alpha1.UsagePolicy{ + AllowedNamespaces: v1alpha1.AllowedNamespaces{ + From: &allowSameNamespace, + }, + } + _, err = f.CreateRepository(repo, &cred) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying the backup invoker phase is: Invalid") + f.EventuallyBackupInvokerPhase(inv).Should(BeEquivalentTo(v1beta1.BackupInvokerInvalid)) + + By("Updating Repository to allow all namespaces") + _, err = f.AllowNamespace(repo, allowAllNamespaces) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying the backup invoker phase is: Ready") + f.EventuallyBackupInvokerPhase(inv).Should(BeEquivalentTo(v1beta1.BackupInvokerReady)) + + backupSession, err := f.TakeInstantBackup(backupConfig.ObjectMeta, v1beta1.BackupInvokerRef{ + Name: backupConfig.Name, + Kind: v1beta1.ResourceKindBackupConfiguration, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying that BackupSession has succeeded") + completedBS, err := f.StashClient.StashV1beta1().BackupSessions(backupSession.Namespace).Get(context.TODO(), backupSession.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(completedBS.Status.Phase).Should(Equal(v1beta1.BackupSessionSucceeded)) + + By("Deleting the Repository") + err = f.DeleteRepository(repo) + Expect(err).NotTo(HaveOccurred()) + + restoredDeployment, err := f.DeployDeployment(framework.RestoredDeployment, int32(1), framework.RestoredVolume, func(dp *apps.Deployment) { + dp.Namespace = f.RestoreNamespace() + }) + Expect(err).NotTo(HaveOccurred()) + + By("Creating RestoreSession") + restoreSession, err := f.CreateRestoreSessionForWorkload(restoredDeployment.ObjectMeta, repo.Name, apis.KindDeployment, framework.RestoredVolume, func(restore *v1beta1.RestoreSession) { + restore.Namespace = f.RestoreNamespace() + restore.Spec.Repository.Namespace = f.Namespace() + }) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying that RestoreSession phase is Pending") + restoreInvoker := invoker.NewRestoreSessionInvoker(f.KubeClient, f.StashClient, restoreSession) + f.EventuallyRestoreInvokerPhase(restoreInvoker).Should(BeEquivalentTo(v1beta1.RestorePending)) + + By("Creating Repository not allowing the RestoreSession") + _, err = f.CreateRepository(repo, &cred) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying that RestoreSession phase is Invalid") + f.EventuallyRestoreInvokerPhase(restoreInvoker).Should(BeEquivalentTo(v1beta1.RestorePhaseInvalid)) + + By("Updating Repository to allow the RestoreSession") + _, err = f.AllowNamespace(repo, allowAllNamespaces) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying that RestoreSession phase is Succeeded") + f.EventuallyRestoreInvokerPhase(restoreInvoker).Should(BeEquivalentTo(v1beta1.RestoreSucceeded)) + + By("Verifying restored data is same as the original data") + restoredData := f.RestoredData(restoredDeployment.ObjectMeta, apis.KindDeployment) + Expect(restoredData).Should(BeSameAs(sampleData)) + }) + }) +}) diff --git a/vendor/modules.txt b/vendor/modules.txt index e3f331f4f..f252b155b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1582,8 +1582,8 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# stash.appscode.dev/apimachinery v0.17.1-0.20220203132324-9c5287433ad7 -## explicit; go 1.16 +# stash.appscode.dev/apimachinery v0.17.1-0.20220210134237-79d844fbde2c +## explicit; go 1.17 stash.appscode.dev/apimachinery/apis stash.appscode.dev/apimachinery/apis/repositories stash.appscode.dev/apimachinery/apis/repositories/install diff --git a/vendor/stash.appscode.dev/apimachinery/apis/constants.go b/vendor/stash.appscode.dev/apimachinery/apis/constants.go index 4fef72f30..c524f0f68 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/constants.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/constants.go @@ -165,6 +165,20 @@ const ( const ( PromJobStashBackup = "stash-backup" PromJobStashRestore = "stash-restore" + + // RepositoryMetricsPushed whether the Repository metrics for this backup session were pushed or not + RepositoryMetricsPushed = "RepositoryMetricsPushed" + // SuccessfullyPushedRepositoryMetrics indicates that the condition transitioned to this state because the repository metrics was successfully pushed to the pushgateway + SuccessfullyPushedRepositoryMetrics = "SuccessfullyPushedRepositoryMetrics" + // FailedToPushRepositoryMetrics indicates that the condition transitioned to this state because the Stash was unable to push the repository metrics to the pushgateway + FailedToPushRepositoryMetrics = "FailedToPushRepositoryMetrics" + + // MetricsPushed whether the metrics for this backup session were pushed or not + MetricsPushed = "MetricsPushed" + // SuccessfullyPushedMetrics indicates that the condition transitioned to this state because the metrics was successfully pushed to the pushgateway + SuccessfullyPushedMetrics = "SuccessfullyPushedMetrics" + // FailedToPushMetrics indicates that the condition transitioned to this state because the Stash was unable to push the metrics to the pushgateway + FailedToPushMetrics = "FailedToPushMetrics" ) // ==================== RBAC related constants ========================== @@ -215,6 +229,8 @@ const ( // This condition is particularly helpful when the restore addon require some additional operations to perform // before marking the RestoreSession Succeeded/Failed. RestoreCompleted = "RestoreCompleted" + // RestorerEnsured condition indicates whether the restore job / init-container was created or not. + RestorerEnsured = "RestorerEnsured" // GlobalPreBackupHookSucceeded indicates whether the global PreBackupHook was executed successfully or not GlobalPreBackupHookSucceeded = "GlobalPreBackupHookSucceeded" @@ -230,8 +246,6 @@ const ( RetentionPolicyApplied = "RetentionPolicyApplied" // RepositoryIntegrityVerified indicates whether the repository integrity check succeeded or not RepositoryIntegrityVerified = "RepositoryIntegrityVerified" - // RepositoryMetricsPushed whether the Repository metrics for this backup session were pushed or not - RepositoryMetricsPushed = "RepositoryMetricsPushed" ) // ================== Condition Types Related Constants =========================== @@ -316,10 +330,6 @@ const ( SuccessfullyVerifiedRepositoryIntegrity = "SuccessfullyVerifiedRepositoryIntegrity" // FailedToVerifyRepositoryIntegrity indicates that the condition transitioned to this state because the repository has failed the integrity check FailedToVerifyRepositoryIntegrity = "FailedToVerifyRepositoryIntegrity" - // SuccessfullyPushedRepositoryMetrics indicates that the condition transitioned to this state because the repository metrics was successfully pushed to the pushgateway - SuccessfullyPushedRepositoryMetrics = "SuccessfullyPushedRepositoryMetrics" - // FailedToPushRepositoryMetrics indicates that the condition transitioned to this state because the Stash was unable to push the repository metrics to the pushgateway - FailedToPushRepositoryMetrics = "FailedToPushRepositoryMetrics" ) // ==================== Action related constants ============ diff --git a/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go index d50877ecc..f9e3f5ec3 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go @@ -18277,6 +18277,22 @@ func schema_kmodulesxyz_offshoot_api_api_v1_ObjectMeta(ref common.ReferenceCallb Description: "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "labels": { + SchemaProps: spec.SchemaProps{ + Description: "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "annotations": { SchemaProps: spec.SchemaProps{ Description: "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go index 977d5895e..93854961c 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go @@ -18282,6 +18282,22 @@ func schema_kmodulesxyz_offshoot_api_api_v1_ObjectMeta(ref common.ReferenceCallb Description: "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "labels": { + SchemaProps: spec.SchemaProps{ + Description: "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "annotations": { SchemaProps: spec.SchemaProps{ Description: "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/backup_batch_types.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/backup_batch_types.go index df233c469..4b1ff5164 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/backup_batch_types.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/backup_batch_types.go @@ -39,6 +39,7 @@ const ( // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Schedule",type="string",JSONPath=".spec.schedule" // +kubebuilder:printcolumn:name="Paused",type="boolean",JSONPath=".spec.paused" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" type BackupBatch struct { metav1.TypeMeta `json:",inline,omitempty"` diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go index fe96443d5..4c5172652 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go @@ -18323,6 +18323,22 @@ func schema_kmodulesxyz_offshoot_api_api_v1_ObjectMeta(ref common.ReferenceCallb Description: "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "labels": { + SchemaProps: spec.SchemaProps{ + Description: "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "annotations": { SchemaProps: spec.SchemaProps{ Description: "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", diff --git a/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go index e160f6e0f..a6bf8d2f0 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go @@ -18277,6 +18277,22 @@ func schema_kmodulesxyz_offshoot_api_api_v1_ObjectMeta(ref common.ReferenceCallb Description: "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "labels": { + SchemaProps: spec.SchemaProps{ + Description: "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "annotations": { SchemaProps: spec.SchemaProps{ Description: "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations", diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml index 4ca43232a..86d4e974e 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupbatches.yaml @@ -25,6 +25,9 @@ spec: - jsonPath: .spec.paused name: Paused type: boolean + - jsonPath: .status.phase + name: Phase + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore.go b/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore.go index 8e88bfd25..25bd777f9 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/conditions/restore.go @@ -89,8 +89,8 @@ func SetRestoreJobCreatedConditionToFalse(inv invoker.RestoreInvoker, tref *api_ }) } -func SetInitContainerInjectedConditionToTrue(inv invoker.RestoreInvoker, tref api_v1beta1.TargetRef) error { - return inv.SetCondition(&tref, kmapi.Condition{ +func SetInitContainerInjectedConditionToTrue(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef) error { + return inv.SetCondition(tref, kmapi.Condition{ Type: apis.StashInitContainerInjected, Status: core.ConditionTrue, Reason: apis.InitContainerInjectionSucceeded, @@ -98,8 +98,8 @@ func SetInitContainerInjectedConditionToTrue(inv invoker.RestoreInvoker, tref ap }) } -func SetInitContainerInjectedConditionToFalse(inv invoker.RestoreInvoker, tref api_v1beta1.TargetRef, err error) error { - return inv.SetCondition(&tref, kmapi.Condition{ +func SetInitContainerInjectedConditionToFalse(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef, err error) error { + return inv.SetCondition(tref, kmapi.Condition{ Type: apis.StashInitContainerInjected, Status: core.ConditionFalse, Reason: apis.InitContainerInjectionFailed, @@ -107,8 +107,8 @@ func SetInitContainerInjectedConditionToFalse(inv invoker.RestoreInvoker, tref a }) } -func SetRestoreCompletedConditionToTrue(inv invoker.RestoreInvoker, tref api_v1beta1.TargetRef, msg string) error { - return inv.SetCondition(&tref, kmapi.Condition{ +func SetRestoreCompletedConditionToTrue(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef, msg string) error { + return inv.SetCondition(tref, kmapi.Condition{ Type: apis.RestoreCompleted, Status: core.ConditionTrue, Reason: "PostRestoreTasksExecuted", @@ -116,11 +116,47 @@ func SetRestoreCompletedConditionToTrue(inv invoker.RestoreInvoker, tref api_v1b }) } -func SetRestoreCompletedConditionToFalse(inv invoker.RestoreInvoker, tref api_v1beta1.TargetRef, msg string) error { - return inv.SetCondition(&tref, kmapi.Condition{ +func SetRestoreCompletedConditionToFalse(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef, msg string) error { + return inv.SetCondition(tref, kmapi.Condition{ Type: apis.RestoreCompleted, Status: core.ConditionFalse, Reason: "PostRestoreTasksNotExecuted", Message: msg, }) } + +func SetRestorerEnsuredToTrue(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef, msg string) error { + return inv.SetCondition(tref, kmapi.Condition{ + Type: apis.RestorerEnsured, + Status: core.ConditionTrue, + Reason: "SuccessfullyEnsuredRestorerEntity", + Message: msg, + }) +} + +func SetRestorerEnsuredToFalse(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef, msg string) error { + return inv.SetCondition(tref, kmapi.Condition{ + Type: apis.RestorerEnsured, + Status: core.ConditionFalse, + Reason: "FailedToEnsureRestorerEntity", + Message: msg, + }) +} + +func SetMetricsPushedConditionToFalse(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef, err error) error { + return inv.SetCondition(tref, kmapi.Condition{ + Type: apis.MetricsPushed, + Status: core.ConditionFalse, + Reason: apis.FailedToPushMetrics, + Message: fmt.Sprintf("Failed to push metrics. Reason: %v", err.Error()), + }) +} + +func SetMetricsPushedConditionToTrue(inv invoker.RestoreInvoker, tref *api_v1beta1.TargetRef) error { + return inv.SetCondition(tref, kmapi.Condition{ + Type: apis.MetricsPushed, + Status: core.ConditionTrue, + Reason: apis.SuccessfullyPushedMetrics, + Message: "Successfully pushed metrics.", + }) +} diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restore_invoker.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restore_invoker.go index ffa51018a..19ae936f6 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restore_invoker.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restore_invoker.go @@ -21,9 +21,11 @@ import ( "fmt" "strings" + "stash.appscode.dev/apimachinery/apis" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" cs "stash.appscode.dev/apimachinery/client/clientset/versioned" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" kmapi "kmodules.xyz/client-go/api/v1" @@ -61,7 +63,7 @@ type RestoreTargetHandler interface { type RestoreStatusHandler interface { GetStatus() RestoreInvokerStatus - UpdateStatus(status RestoreInvokerStatus) error + UpdateStatus(newStatus RestoreInvokerStatus) error } type RestoreInvokerStatus struct { @@ -148,30 +150,6 @@ func isRestoreMemberConditionTrue(status []v1beta1.RestoreMemberStatus, target v return false } -func upsertRestoreMemberStatus(cur []v1beta1.RestoreMemberStatus, new v1beta1.RestoreMemberStatus) []v1beta1.RestoreMemberStatus { - // if the member status already exist, then update it - for i := range cur { - if TargetMatched(cur[i].Ref, new.Ref) { - if new.Phase != "" { - cur[i].Phase = new.Phase - } - if len(new.Conditions) > 0 { - cur[i].Conditions = upsertConditions(cur[i].Conditions, new.Conditions) - } - if new.TotalHosts != nil { - cur[i].TotalHosts = new.TotalHosts - } - if len(new.Stats) > 0 { - cur[i].Stats = upsertRestoreHostStatus(cur[i].Stats, new.Stats) - } - return cur - } - } - // the member status does not exist. so, add new entry. - cur = append(cur, new) - return cur -} - func upsertConditions(cur []kmapi.Condition, new []kmapi.Condition) []kmapi.Condition { for i := range new { cur = kmapi.SetCondition(cur, new[i]) @@ -257,3 +235,71 @@ func TargetOfGroupKind(targetRef v1beta1.TargetRef, group, kind string) bool { } return false } + +func upsertRestoreTargetStatus(cur, new v1beta1.RestoreMemberStatus) v1beta1.RestoreMemberStatus { + if len(new.Conditions) > 0 { + cur.Conditions = upsertConditions(cur.Conditions, new.Conditions) + } + + if new.TotalHosts != nil { + cur.TotalHosts = new.TotalHosts + } + + if len(new.Stats) > 0 { + cur.Stats = upsertRestoreHostStatus(cur.Stats, new.Stats) + } + + cur.Phase = calculateRestoreTargetPhase(cur) + return cur +} + +func calculateRestoreTargetPhase(status v1beta1.RestoreMemberStatus) v1beta1.RestoreTargetPhase { + if kmapi.IsConditionFalse(status.Conditions, apis.RestorerEnsured) || + kmapi.IsConditionFalse(status.Conditions, apis.MetricsPushed) { + return v1beta1.TargetRestoreFailed + } + + allConditionTrue := true + for _, c := range status.Conditions { + if c.Status != core.ConditionTrue { + allConditionTrue = false + } + } + if !allConditionTrue || status.TotalHosts == nil { + return v1beta1.TargetRestorePending + } + + failedHostCount := int32(0) + unknownHostCount := int32(0) + successfulHostCount := int32(0) + for _, hostStats := range status.Stats { + switch hostStats.Phase { + case v1beta1.HostRestoreFailed: + failedHostCount++ + case v1beta1.HostRestoreUnknown: + unknownHostCount++ + case v1beta1.HostRestoreSucceeded: + successfulHostCount++ + } + } + + completedHosts := successfulHostCount + failedHostCount + unknownHostCount + if completedHosts < *status.TotalHosts { + return v1beta1.TargetRestoreRunning + } + + if failedHostCount > 0 { + return v1beta1.TargetRestoreFailed + } + if unknownHostCount > 0 { + return v1beta1.TargetRestorePhaseUnknown + } + + return v1beta1.TargetRestoreSucceeded +} + +func IsRestoreCompleted(phase v1beta1.RestorePhase) bool { + return phase == v1beta1.RestoreSucceeded || + phase == v1beta1.RestoreFailed || + phase == v1beta1.RestorePhaseUnknown +} diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go index 49b698509..73ef573dc 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restorebatch.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "stash.appscode.dev/apimachinery/apis" "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" cs "stash.appscode.dev/apimachinery/client/clientset/versioned" @@ -126,19 +127,14 @@ func (inv *RestoreBatchInvoker) GetCondition(target *v1beta1.TargetRef, conditio } func (inv *RestoreBatchInvoker) SetCondition(target *v1beta1.TargetRef, newCondition kmapi.Condition) error { - updatedRestoreBatch, err := v1beta1_util.UpdateRestoreBatchStatus(context.TODO(), inv.stashClient.StashV1beta1(), inv.restoreBatch.ObjectMeta, func(in *v1beta1.RestoreBatchStatus) (types.UID, *v1beta1.RestoreBatchStatus) { - if target != nil { - in.Members = setRestoreMemberCondition(in.Members, *target, newCondition) - } else { - in.Conditions = kmapi.SetCondition(in.Conditions, newCondition) - } - return inv.restoreBatch.UID, in - }, metav1.UpdateOptions{}) - if err != nil { - return err + status := inv.GetStatus() + + if target != nil { + status.TargetStatus = setRestoreMemberCondition(status.TargetStatus, *target, newCondition) + } else { + status.Conditions = kmapi.SetCondition(status.Conditions, newCondition) } - inv.restoreBatch = updatedRestoreBatch - return nil + return inv.UpdateStatus(status) } func (inv *RestoreBatchInvoker) IsConditionTrue(target *v1beta1.TargetRef, conditionType string) (bool, error) { @@ -225,37 +221,6 @@ func (inv *RestoreBatchInvoker) GetObjectJSON() (string, error) { return string(jsonObj), nil } -func (inv *RestoreBatchInvoker) UpdateStatus(status RestoreInvokerStatus) error { - updatedRestoreBatch, err := v1beta1_util.UpdateRestoreBatchStatus( - context.TODO(), - inv.stashClient.StashV1beta1(), - inv.restoreBatch.ObjectMeta, - func(in *v1beta1.RestoreBatchStatus) (types.UID, *v1beta1.RestoreBatchStatus) { - if status.Phase != "" { - in.Phase = status.Phase - } - if status.SessionDuration != "" { - in.SessionDuration = status.SessionDuration - } - if len(status.Conditions) > 0 { - in.Conditions = upsertConditions(in.Conditions, status.Conditions) - } - if len(status.TargetStatus) > 0 { - for i := range status.TargetStatus { - in.Members = upsertRestoreMemberStatus(in.Members, status.TargetStatus[i]) - } - } - return inv.restoreBatch.ObjectMeta.UID, in - }, - metav1.UpdateOptions{}, - ) - if err != nil { - return err - } - inv.restoreBatch = updatedRestoreBatch - return nil -} - func (inv *RestoreBatchInvoker) CreateEvent(eventType, source, reason, message string) error { objRef, err := inv.GetObjectRef() if err != nil { @@ -326,3 +291,88 @@ func (inv *RestoreBatchInvoker) EnsureKubeDBIntegration(appClient appcatalog_cs. func (inv *RestoreBatchInvoker) GetStatus() RestoreInvokerStatus { return getInvokerStatusFromRestoreBatch(inv.restoreBatch) } + +func (inv *RestoreBatchInvoker) UpdateStatus(status RestoreInvokerStatus) error { + startTime := inv.GetObjectMeta().CreationTimestamp + totalTargets := len(inv.GetTargetInfo()) + updatedRestoreBatch, err := v1beta1_util.UpdateRestoreBatchStatus( + context.TODO(), + inv.stashClient.StashV1beta1(), + inv.restoreBatch.ObjectMeta, + func(in *v1beta1.RestoreBatchStatus) (types.UID, *v1beta1.RestoreBatchStatus) { + if len(status.Conditions) > 0 { + in.Conditions = upsertConditions(in.Conditions, status.Conditions) + } + if len(status.TargetStatus) > 0 { + for i := range status.TargetStatus { + in.Members = upsertRestoreMemberStatus(in.Members, status.TargetStatus[i]) + } + } + + in.Phase = calculateRestoreBatchPhase(in, totalTargets) + if IsRestoreCompleted(in.Phase) && in.SessionDuration == "" { + duration := time.Since(startTime.Time) + in.SessionDuration = duration.Round(time.Second).String() + } + return inv.restoreBatch.ObjectMeta.UID, in + }, + metav1.UpdateOptions{}, + ) + if err != nil { + return err + } + inv.restoreBatch = updatedRestoreBatch + return nil +} + +func upsertRestoreMemberStatus(cur []v1beta1.RestoreMemberStatus, new v1beta1.RestoreMemberStatus) []v1beta1.RestoreMemberStatus { + // if the member status already exist, then update it + for i := range cur { + if TargetMatched(cur[i].Ref, new.Ref) { + cur[i] = upsertRestoreTargetStatus(cur[i], new) + return cur + } + } + // the member status does not exist. so, add new entry. + cur = append(cur, new) + return cur +} + +func calculateRestoreBatchPhase(status *v1beta1.RestoreBatchStatus, totalTargets int) v1beta1.RestorePhase { + if len(status.Conditions) == 0 || len(status.Members) == 0 || + kmapi.IsConditionFalse(status.Conditions, apis.RepositoryFound) || + kmapi.IsConditionFalse(status.Conditions, apis.BackendSecretFound) { + return v1beta1.RestorePending + } + + if kmapi.IsConditionFalse(status.Conditions, apis.ValidationPassed) { + return v1beta1.RestorePhaseInvalid + } + + failedTargetCount := 0 + unknownTargetCount := 0 + successfulTargetCount := 0 + + for _, m := range status.Members { + switch m.Phase { + case v1beta1.TargetRestoreFailed: + failedTargetCount++ + case v1beta1.TargetRestorePhaseUnknown: + unknownTargetCount++ + case v1beta1.TargetRestoreSucceeded: + successfulTargetCount++ + } + } + completedTargets := successfulTargetCount + failedTargetCount + unknownTargetCount + if completedTargets < len(status.Members) || completedTargets < totalTargets { + return v1beta1.RestoreRunning + } + if failedTargetCount > 0 { + return v1beta1.RestoreFailed + } + if unknownTargetCount > 0 { + return v1beta1.RestorePhaseUnknown + } + + return v1beta1.RestoreSucceeded +} diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go index 9fe1845e1..891af5f75 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/invoker/restoresession.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "stash.appscode.dev/apimachinery/apis" "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" "stash.appscode.dev/apimachinery/apis/stash/v1beta1" cs "stash.appscode.dev/apimachinery/client/clientset/versioned" @@ -119,15 +120,10 @@ func (inv *RestoreSessionInvoker) GetCondition(target *v1beta1.TargetRef, condit } func (inv *RestoreSessionInvoker) SetCondition(target *v1beta1.TargetRef, newCondition kmapi.Condition) error { - updatedRestoreSession, err := v1beta1_util.UpdateRestoreSessionStatus(context.TODO(), inv.stashClient.StashV1beta1(), inv.restoreSession.ObjectMeta, func(in *v1beta1.RestoreSessionStatus) (types.UID, *v1beta1.RestoreSessionStatus) { - in.Conditions = kmapi.SetCondition(in.Conditions, newCondition) - return inv.restoreSession.UID, in - }, metav1.UpdateOptions{}) - if err != nil { - return err - } - inv.restoreSession = updatedRestoreSession - return nil + status := inv.GetStatus() + status.Conditions = kmapi.SetCondition(status.Conditions, newCondition) + status.TargetStatus[0].Conditions = status.Conditions + return inv.UpdateStatus(status) } func (inv *RestoreSessionInvoker) IsConditionTrue(target *v1beta1.TargetRef, conditionType string) (bool, error) { @@ -209,47 +205,6 @@ func (inv *RestoreSessionInvoker) GetObjectJSON() (string, error) { return string(jsonObj), nil } -func (inv *RestoreSessionInvoker) UpdateStatus(status RestoreInvokerStatus) error { - updatedRestoreSession, err := v1beta1_util.UpdateRestoreSessionStatus( - context.TODO(), - inv.stashClient.StashV1beta1(), - inv.restoreSession.ObjectMeta, - func(in *v1beta1.RestoreSessionStatus) (types.UID, *v1beta1.RestoreSessionStatus) { - if status.Phase != "" { - in.Phase = status.Phase - } - if status.SessionDuration != "" { - in.SessionDuration = status.SessionDuration - } - if len(status.Conditions) > 0 { - in.Conditions = upsertConditions(in.Conditions, status.Conditions) - } - if len(status.TargetStatus) > 0 { - targetStatus := status.TargetStatus[0] - if targetStatus.TotalHosts != nil { - in.TotalHosts = targetStatus.TotalHosts - } - if targetStatus.Conditions != nil { - in.Conditions = upsertConditions(in.Conditions, targetStatus.Conditions) - } - if targetStatus.Stats != nil { - in.Stats = upsertRestoreHostStatus(in.Stats, targetStatus.Stats) - } - if targetStatus.Phase != "" { - in.Phase = v1beta1.RestorePhase(targetStatus.Phase) - } - } - return inv.restoreSession.ObjectMeta.UID, in - }, - metav1.UpdateOptions{}, - ) - if err != nil { - return err - } - inv.restoreSession = updatedRestoreSession - return nil -} - func (inv *RestoreSessionInvoker) CreateEvent(eventType, source, reason, message string) error { objRef, err := inv.GetObjectRef() if err != nil { @@ -317,3 +272,68 @@ func (inv *RestoreSessionInvoker) EnsureKubeDBIntegration(appClient appcatalog_c func (inv *RestoreSessionInvoker) GetStatus() RestoreInvokerStatus { return getInvokerStatusFromRestoreSession(inv.restoreSession) } + +func (inv *RestoreSessionInvoker) UpdateStatus(status RestoreInvokerStatus) error { + startTime := inv.GetObjectMeta().CreationTimestamp + updatedRestoreSession, err := v1beta1_util.UpdateRestoreSessionStatus( + context.TODO(), + inv.stashClient.StashV1beta1(), + inv.restoreSession.ObjectMeta, + func(in *v1beta1.RestoreSessionStatus) (types.UID, *v1beta1.RestoreSessionStatus) { + curStatus := v1beta1.RestoreMemberStatus{ + Conditions: in.Conditions, + TotalHosts: in.TotalHosts, + Stats: in.Stats, + } + newStatus := v1beta1.RestoreMemberStatus{ + Conditions: status.TargetStatus[0].Conditions, + TotalHosts: status.TargetStatus[0].TotalHosts, + Stats: status.TargetStatus[0].Stats, + } + updatedStatus := upsertRestoreTargetStatus(curStatus, newStatus) + + in.Conditions = updatedStatus.Conditions + in.Stats = updatedStatus.Stats + in.TotalHosts = updatedStatus.TotalHosts + in.Phase = calculateRestoreSessionPhase(updatedStatus) + + if IsRestoreCompleted(in.Phase) && in.SessionDuration == "" { + duration := time.Since(startTime.Time) + in.SessionDuration = duration.Round(time.Second).String() + } + return inv.restoreSession.ObjectMeta.UID, in + }, + metav1.UpdateOptions{}, + ) + if err != nil { + return err + } + inv.restoreSession = updatedRestoreSession + return nil +} + +func calculateRestoreSessionPhase(status v1beta1.RestoreMemberStatus) v1beta1.RestorePhase { + if len(status.Conditions) == 0 || + kmapi.IsConditionFalse(status.Conditions, apis.RepositoryFound) || + kmapi.IsConditionFalse(status.Conditions, apis.BackendSecretFound) || + kmapi.IsConditionFalse(status.Conditions, apis.RestoreTargetFound) { + return v1beta1.RestorePending + } + + if kmapi.IsConditionFalse(status.Conditions, apis.ValidationPassed) { + return v1beta1.RestorePhaseInvalid + } + + switch status.Phase { + case v1beta1.TargetRestorePending: + return v1beta1.RestorePending + case v1beta1.TargetRestoreSucceeded: + return v1beta1.RestoreSucceeded + case v1beta1.TargetRestoreFailed: + return v1beta1.RestoreFailed + case v1beta1.TargetRestorePhaseUnknown: + return v1beta1.RestorePhaseUnknown + default: + return v1beta1.RestoreRunning + } +}