Add wait for target logic + add conditions for BackupConfiguration + …

…BackupBatch + RestoreSession (#1108) Fixes #988 Fixes #1092 Tasks: - [x] Add wait for target logic for backup - [x] Add E2E tests for backup - [x] Add wait for target logic for restore - [x] Add E2E test for restore - [x] Merge stashed/apimachinery#3 - [x] Revendor against stash/apimachinery master Signed-off-by: hossainemruz <[email protected]>
stashed · May 22, 2020 · f450e9c · f450e9c
1 parent 8f8ff87
commit f450e9c
Show file tree

Hide file tree

Showing 70 changed files with 3,959 additions and 448 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -71,7 +71,7 @@ jobs:
       id: kind
       uses: engineerd/[email protected]
       with:
-        version: v0.8.0
+        version: v0.7.0
         config: hack/kubernetes/kind.yaml
         image: kindest/node:${{ matrix.k8s }}
 

diff --git a/Makefile b/Makefile
@@ -71,7 +71,7 @@ TAG              := $(VERSION)_$(OS)_$(ARCH)
 TAG_PROD         := $(TAG)
 TAG_DBG          := $(VERSION)-dbg_$(OS)_$(ARCH)
 
-GO_VERSION       ?= 1.14.3
+GO_VERSION       ?= 1.14
 BUILD_IMAGE      ?= appscode/golang-dev:$(GO_VERSION)
 TEST_IMAGE       ?= appscode/golang-dev:$(GO_VERSION)-stash
 
@@ -363,13 +363,12 @@ endif
 .PHONY: install
 install:
 	@cd ../installer; \
-	helm install stash charts/stash \
+	helm install stash charts/stash --wait \
 		--namespace=kube-system \
 		--set operator.registry=$(REGISTRY) \
 		--set operator.tag=$(TAG) \
 		--set imagePullPolicy=Always \
 		$(IMAGE_PULL_SECRETS); \
-	kubectl wait --for=condition=Ready pods -n kube-system -l 'app.kubernetes.io/name=stash,app.kubernetes.io/instance=stash' --timeout=5m; \
 	kubectl wait --for=condition=Available apiservice -l 'app.kubernetes.io/name=stash,app.kubernetes.io/instance=stash' --timeout=5m
 
 .PHONY: uninstall
@@ -379,7 +378,7 @@ uninstall:
 
 .PHONY: purge
 purge: uninstall
-	kubectl delete crds -l app=stash
+	kubectl delete crds -l app.kubernetes.io/name=stash
 
 .PHONY: dev
 dev: gen fmt push

diff --git a/go.mod b/go.mod
@@ -29,15 +29,15 @@ require (
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/kube-aggregator v0.0.0-20191114103820-f023614fb9ea
 	k8s.io/kubernetes v1.16.3
-	kmodules.xyz/client-go v0.0.0-20200216080917-08714f78f885
+	kmodules.xyz/client-go v0.0.0-20200515063639-98d65435fd5d
 	kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6
 	kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419
 	kmodules.xyz/objectstore-api v0.0.0-20200214040336-fe8f39a4210d
 	kmodules.xyz/offshoot-api v0.0.0-20200216080509-45ee6418d1c1
 	kmodules.xyz/openshift v0.0.0-20191127145035-f6c48a90dbb7
 	kmodules.xyz/prober v0.0.0-20191216013129-7c91e2b0edb2
 	kmodules.xyz/webhook-runtime v0.0.0-20191127075323-d4bfdee6974d
-	stash.appscode.dev/apimachinery v0.9.0-rc.6
+	stash.appscode.dev/apimachinery v0.9.0-rc.6.0.20200520105618-e83b90a7160f
 )
 
 replace (

diff --git a/go.sum b/go.sum
@@ -689,10 +689,8 @@ k8s.io/utils v0.0.0-20190801114015-581e00157fb1 h1:+ySTxfHnfzZb9ys375PXNlLhkJPLK
 k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 kmodules.xyz/client-go v0.0.0-20191127054604-26981530831d/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
 kmodules.xyz/client-go v0.0.0-20191211192817-f1dcd02124ba/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
-kmodules.xyz/client-go v0.0.0-20200116162153-e083ae16abca h1:WErv3VRRePgp4oszIHgrFnxIaIHyfP9+930B99B8BBM=
-kmodules.xyz/client-go v0.0.0-20200116162153-e083ae16abca/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
-kmodules.xyz/client-go v0.0.0-20200216080917-08714f78f885 h1:1v803gTRPSfjMYwW3YXoxs36gQ9YVe/1qWixPfLZ6bM=
-kmodules.xyz/client-go v0.0.0-20200216080917-08714f78f885/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
+kmodules.xyz/client-go v0.0.0-20200515063639-98d65435fd5d h1:X4tiCZ1QtemXxFtO+uFq1DM9DgMTqOQ1rBFExZT7yKk=
+kmodules.xyz/client-go v0.0.0-20200515063639-98d65435fd5d/go.mod h1:z/AkrES60XR+jeKGg5zJyPd13NdG/Di0BptbG1gnS60=
 kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6 h1:hFv3DzanQJ/bjgahqosmthGLkVgMB2KuQIsltOA02t0=
 kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
 kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891 h1:2W/fqLbAurvupIZL3TEFrlb7DnOIhlOXpNgzgcFFSmA=
@@ -725,6 +723,6 @@ sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca h1:6dsH6AYQ
 sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-stash.appscode.dev/apimachinery v0.9.0-rc.6 h1:rYp77PiYd05HwAtqbhVJFvmeGRZzcbOOMr6Pmh3zKKw=
-stash.appscode.dev/apimachinery v0.9.0-rc.6/go.mod h1:jLzf4kBtDJ1VeMuHm6PGPCR6is2cCfMO62FMrOwcig4=
+stash.appscode.dev/apimachinery v0.9.0-rc.6.0.20200520105618-e83b90a7160f h1:lpVl0wvZYW36B2MqaunCk8bMeV8Exn2kn30YZS2Quqo=
+stash.appscode.dev/apimachinery v0.9.0-rc.6.0.20200520105618-e83b90a7160f/go.mod h1:yq8KRoJKUUaTj/v3TRwBLEeEFNYuH4hF+2kaXXccNNM=
 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=
diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go
@@ -329,7 +329,7 @@ func (c *Controller) runResticBackup(restic *api.Restic, repository *api.Reposit
 			}
 		}
 		if err == nil {
-			_, err2 := stash_util.UpdateRepositoryStatus(c.stashClient.StashV1alpha1(), repository, func(in *api.RepositoryStatus) *api.RepositoryStatus {
+			_, err2 := stash_util.UpdateRepositoryStatus(c.stashClient.StashV1alpha1(), repository.ObjectMeta, func(in *api.RepositoryStatus) *api.RepositoryStatus {
 				in.BackupCount++
 				in.LastBackupTime = &startTime
 				if in.FirstBackupTime == nil {

diff --git a/pkg/controller/backup.go b/pkg/controller/backup.go
@@ -122,12 +122,11 @@ func (c *StashController) applyBackupConfigurationLogic(w *wapi.Workload, caller
 				targetInfo.Target.Ref.Name == w.Name {
 				err = c.ensureBackupSidecar(w, invoker, targetInfo, caller)
 				if err != nil {
-					return false, c.handleSidecarInjectionFailure(w, err)
+					return false, c.handleSidecarInjectionFailure(w, invoker, targetInfo.Target.Ref, err)
 				}
-				break
+				return true, c.handleSidecarInjectionSuccess(w, invoker, targetInfo.Target.Ref)
 			}
 		}
-		return true, c.handleSidecarInjectionSuccess(w)
 
 	} else if oldbc != nil && newbc == nil {
 		// there was BackupConfiguration before but it does not exist now.
@@ -168,13 +167,12 @@ func (c *StashController) applyBackupBatchLogic(w *wapi.Workload, caller string)
 						targetInfo.Target.Ref.Name == w.Name {
 						err = c.ensureBackupSidecar(w, invoker, targetInfo, caller)
 						if err != nil {
-							return false, c.handleSidecarInjectionFailure(w, err)
+							return false, c.handleSidecarInjectionFailure(w, invoker, targetInfo.Target.Ref, err)
 						}
-						break
+						// write sidecar injection failure/success event
+						return true, c.handleSidecarInjectionSuccess(w, invoker, targetInfo.Target.Ref)
 					}
 				}
-				// write sidecar injection failure/success event
-				return true, c.handleSidecarInjectionSuccess(w)
 			}
 		}
 	} else if oldbb != nil && newbb == nil {

diff --git a/pkg/controller/backup_configuration.go b/pkg/controller/backup_configuration.go
@@ -19,6 +19,7 @@ package controller
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"stash.appscode.dev/apimachinery/apis"
 	api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
@@ -74,21 +75,21 @@ func (c *StashController) runBackupConfigurationProcessor(key string) error {
 	if err != nil {
 		return err
 	}
-	err = c.applyBackupInvokerReconciliationLogic(invoker)
+	err = c.applyBackupInvokerReconciliationLogic(invoker, key)
 	if err != nil {
 		return err
 	}
 
 	// We have successfully completed respective stuffs for the current state of this resource.
 	// Hence, let's set observed generation as same as the current generation.
-	_, err = v1beta1_util.UpdateBackupConfigurationStatus(c.stashClient.StashV1beta1(), backupConfiguration, func(in *api_v1beta1.BackupConfigurationStatus) *api_v1beta1.BackupConfigurationStatus {
+	_, err = v1beta1_util.UpdateBackupConfigurationStatus(c.stashClient.StashV1beta1(), backupConfiguration.ObjectMeta, func(in *api_v1beta1.BackupConfigurationStatus) *api_v1beta1.BackupConfigurationStatus {
 		in.ObservedGeneration = backupConfiguration.Generation
 		return in
 	})
 	return err
 }
 
-func (c *StashController) applyBackupInvokerReconciliationLogic(invoker apis.Invoker) error {
+func (c *StashController) applyBackupInvokerReconciliationLogic(invoker apis.Invoker, key string) error {
 	// check if BackupBatch is being deleted. if it is being deleted then delete respective resources.
 	if invoker.ObjectMeta.DeletionTimestamp != nil {
 		if core_util.HasFinalizer(invoker.ObjectMeta, api_v1beta1.StashKey) {
@@ -112,18 +113,91 @@ func (c *StashController) applyBackupInvokerReconciliationLogic(invoker apis.Inv
 		if err != nil {
 			return err
 		}
+		someTargetMissing := false
 		for _, targetInfo := range invoker.TargetsInfo {
-			if targetInfo.Target != nil && invoker.Driver != api_v1beta1.VolumeSnapshotter &&
-				util.BackupModel(targetInfo.Target.Ref.Kind) == apis.ModelSidecar {
-				if err := c.EnsureV1beta1Sidecar(targetInfo.Target.Ref, invoker.ObjectMeta.Namespace); err != nil {
-					return c.handleWorkloadControllerTriggerFailure(invoker.ObjectRef, err)
+			if targetInfo.Target != nil {
+				tref := targetInfo.Target.Ref
+				wc := util.WorkloadClients{
+					KubeClient:       c.kubeClient,
+					OcClient:         c.ocClient,
+					StashClient:      c.stashClient,
+					CRDClient:        c.crdClient,
+					AppCatalogClient: c.appCatalogClient,
+				}
+				targetExist, err := wc.IsTargetExist(tref, invoker.ObjectMeta.Namespace)
+				if err != nil {
+					glog.Errorf("Failed to check whether %s %s %s/%s exist or not. Reason: %v.",
+						tref.APIVersion,
+						tref.Kind,
+						invoker.ObjectMeta.Namespace,
+						tref.Name,
+						err.Error(),
+					)
+					// Set the "BackupTargetFound" condition to "Unknown"
+					cerr := c.setBackupTargetFoundConditionToUnknown(invoker, tref, err)
+					return errors.NewAggregate([]error{err, cerr})
+				}
+				if !targetExist {
+					// Target does not exist. Log the information.
+					someTargetMissing = true
+					glog.Infof("Backup target %s %s %s/%s does not exist.",
+						tref.APIVersion,
+						tref.Kind,
+						invoker.ObjectMeta.Namespace,
+						tref.Name)
+					err = c.setBackupTargetFoundConditionToFalse(invoker, tref)
+					if err != nil {
+						return err
+					}
+					// Process next target.
+					continue
+				}
+				// Backup target exist. So, set "BackupTargetFound" condition to "True"
+				err = c.setBackupTargetFoundConditionToTrue(invoker, tref)
+				if err != nil {
+					return err
+				}
+				// For sidecar model, ensure the stash sidecar
+				if (invoker.Driver == "" || invoker.Driver == api_v1beta1.ResticSnapshotter) && util.BackupModel(tref.Kind) == apis.ModelSidecar {
+					err := c.EnsureV1beta1Sidecar(tref, invoker.ObjectMeta.Namespace)
+					if err != nil {
+						return c.handleWorkloadControllerTriggerFailure(invoker.ObjectRef, err)
+					}
 				}
 			}
+
+		}
+		// If some backup targets are missing, then retry after some time.
+		if someTargetMissing {
+			glog.Infof("Some targets are missing for backup invoker: %s %s/%s.\nRequeueing after 5 seconds......",
+				invoker.TypeMeta.Kind,
+				invoker.ObjectMeta.Namespace,
+				invoker.ObjectMeta.Name,
+			)
+			switch invoker.TypeMeta.Kind {
+			case api_v1beta1.ResourceKindBackupConfiguration:
+				c.bcQueue.GetQueue().AddAfter(key, 5*time.Second)
+			case api_v1beta1.ResourceKindBackupBatch:
+				c.backupBatchQueue.GetQueue().AddAfter(key, 5*time.Second)
+			default:
+				return fmt.Errorf("unable to requeue. Reason: Backup invoker %s  %s is not supported",
+					invoker.TypeMeta.APIVersion,
+					invoker.TypeMeta.Kind,
+				)
+			}
+			return nil
 		}
 		// create a CronJob that will create BackupSession on each schedule
 		err = c.EnsureBackupTriggeringCronJob(invoker)
 		if err != nil {
-			return c.handleCronJobCreationFailure(invoker.ObjectRef, err)
+			// Failed to ensure the backup triggering CronJob. So, set "CronJobCreated" condition to "False"
+			cerr := c.setCronJobCreatedConditionToFalse(invoker, err)
+			return c.handleCronJobCreationFailure(invoker.ObjectRef, errors.NewAggregate([]error{err, cerr}))
+		}
+		// Successfully ensured the backup triggering CronJob. So, set "CronJobCreated" condition to "True"
+		cerr := c.setCronJobCreatedConditionToTrue(invoker)
+		if cerr != nil {
+			return cerr
 		}
 	}
 	return nil

diff --git a/pkg/controller/backup_session.go b/pkg/controller/backup_session.go
@@ -390,7 +390,7 @@ func (c *StashController) ensureVolumeSnapshotterJob(invoker apis.Invoker, targe
 func (c *StashController) setBackupSessionFailed(invoker apis.Invoker, backupSession *api_v1beta1.BackupSession, backupErr error) error {
 
 	// set BackupSession phase to "Failed"
-	updatedBackupSession, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
+	updatedBackupSession, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession.ObjectMeta, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
 		in.Phase = api_v1beta1.BackupSessionFailed
 		in.Targets = backupSession.Status.Targets
 		return in
@@ -432,7 +432,7 @@ func (c *StashController) setTargetPhaseRunning(target *api_v1beta1.BackupTarget
 		return nil, err
 	}
 	// set target phase to "Running"
-	backupSession, err = stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
+	backupSession, err = stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession.ObjectMeta, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
 		if target != nil {
 			in.Targets = upsertTargetStatsEntry(backupSession.Status.Targets, api_v1beta1.Target{
 				TotalHosts: totalHosts,
@@ -450,7 +450,7 @@ func (c *StashController) setTargetPhaseRunning(target *api_v1beta1.BackupTarget
 
 func (c *StashController) setBackupSessionRunning(backupSession *api_v1beta1.BackupSession) (*api_v1beta1.BackupSession, error) {
 	// set BackupSession phase to "Running"
-	backupSession, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
+	backupSession, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession.ObjectMeta, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
 		in.Phase = api_v1beta1.BackupSessionRunning
 		return in
 	})
@@ -476,7 +476,7 @@ func (c *StashController) setBackupSessionSucceeded(invoker apis.Invoker, backup
 	sessionDuration := time.Since(backupSession.CreationTimestamp.Time)
 
 	// set BackupSession phase "Succeeded"
-	updatedBackupSession, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
+	updatedBackupSession, err := stash_util.UpdateBackupSessionStatus(c.stashClient.StashV1beta1(), backupSession.ObjectMeta, func(in *api_v1beta1.BackupSessionStatus) *api_v1beta1.BackupSessionStatus {
 		in.Phase = api_v1beta1.BackupSessionSucceeded
 		in.SessionDuration = sessionDuration.String()
 		in.Targets = backupSession.Status.Targets

diff --git a/pkg/controller/backupbatch.go b/pkg/controller/backupbatch.go
@@ -53,14 +53,14 @@ func (c *StashController) runBackupBatchProcessor(key string) error {
 	if err != nil {
 		return err
 	}
-	err = c.applyBackupInvokerReconciliationLogic(invoker)
+	err = c.applyBackupInvokerReconciliationLogic(invoker, key)
 	if err != nil {
 		return err
 	}
 
 	// We have successfully completed respective stuffs for the current state of this resource.
 	// Hence, let's set observed generation as same as the current generation.
-	_, err = v1beta1_util.UpdateBackupBatchStatus(c.stashClient.StashV1beta1(), backupBatch, func(in *api_v1beta1.BackupBatchStatus) *api_v1beta1.BackupBatchStatus {
+	_, err = v1beta1_util.UpdateBackupBatchStatus(c.stashClient.StashV1beta1(), backupBatch.ObjectMeta, func(in *api_v1beta1.BackupBatchStatus) *api_v1beta1.BackupBatchStatus {
 		in.ObservedGeneration = backupBatch.Generation
 		return in
 	})