fix(operator): Also consider StatefulSets/DaemonSets when checking Wo…

…rkload Deployment state (#406)

Closes #373

operator/config/rbac/role.yaml

@@ -14,14 +14,6 @@ rules:
   - statefulsets
   verbs:
   - get
-- apiGroups:
-  - apps
-  resources:
-  - deployments
-  - replicasets
-  - statefulsets
-  verbs:
-  - get
   - list
   - watch
 - apiGroups:

operator/controllers/common/errors.go

@@ -8,6 +8,7 @@ var ErrRetryCountExceeded = fmt.Errorf("retryCount for evaluation exceeded")
 var ErrNoValues = fmt.Errorf("no values")
 var ErrInvalidOperator = fmt.Errorf("invalid operator")
 var ErrCannotMarshalParams = fmt.Errorf("could not marshal parameters")
+var ErrUnsupportedWorkloadInstanceResourceReference = fmt.Errorf("unsupported Resource Reference")
 
 var ErrCannotRetrieveInstancesMsg = "could not retrieve instances: %w"
 var ErrCannotFetchAppMsg = "could not retrieve KeptnApp: %w"

operator/controllers/keptnworkloadinstance/controller.go

@@ -62,7 +62,7 @@ type KeptnWorkloadInstanceReconciler struct {
 //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks/finalizers,verbs=update
 //+kubebuilder:rbac:groups=core,resources=events,verbs=create;watch;patch
 //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
-//+kubebuilder:rbac:groups=apps,resources=replicasets;deployments;statefulsets,verbs=get;list;watch
+//+kubebuilder:rbac:groups=apps,resources=replicasets;deployments;statefulsets;daemonsets,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

operator/controllers/keptnworkloadinstance/controller_test.go

@@ -3,6 +3,7 @@ package keptnworkloadinstance
 import (
 	"context"
 	klcv1alpha1 "github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1"
+	"github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
 	"github.com/stretchr/testify/require"
 	testrequire "github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
@@ -13,45 +14,165 @@ import (
 	"testing"
 )
 
-func TestKeptnWorkloadInstanceReconciler_isReferencedWorkloadRunning(t *testing.T) {
+func TestKeptnWorkloadInstanceReconciler_reconcileDeployment_FailedReplicaSet(t *testing.T) {
 
 	rep := int32(1)
 	replicasetFail := makeReplicaSet("myrep", "default", &rep, 0)
-	statefulsetFail := makeStatefulSet("mystat", "default", &rep, 0)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(replicasetFail).Build()
+
+	err := klcv1alpha1.AddToScheme(fakeClient.Scheme())
+	testrequire.Nil(t, err)
+
+	workloadInstance := makeWorkloadInstanceWithRef(replicasetFail.ObjectMeta, "ReplicaSet")
+
+	err = fakeClient.Create(context.TODO(), workloadInstance)
+	require.Nil(t, err)
 
 	r := &KeptnWorkloadInstanceReconciler{
-		Client: fake.NewClientBuilder().WithObjects(replicasetFail, statefulsetFail).Build(),
+		Client: fakeClient,
 	}
-	isOwnerRunning, err := r.isReferencedWorkloadRunning(context.TODO(), klcv1alpha1.ResourceReference{UID: "myrep", Name: "myrep", Kind: "ReplicaSet"}, "default")
+
+	keptnState, err := r.reconcileDeployment(context.TODO(), workloadInstance)
 	testrequire.Nil(t, err)
-	if isOwnerRunning {
-		t.Errorf("Should fail!")
+	testrequire.Equal(t, common.StateProgressing, keptnState)
+}
+
+func makeWorkloadInstanceWithRef(objectMeta metav1.ObjectMeta, refKind string) *klcv1alpha1.KeptnWorkloadInstance {
+	workloadInstance := &klcv1alpha1.KeptnWorkloadInstance{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-wli",
+			Namespace: "default",
+		},
+		Spec: klcv1alpha1.KeptnWorkloadInstanceSpec{
+			KeptnWorkloadSpec: klcv1alpha1.KeptnWorkloadSpec{
+				ResourceReference: klcv1alpha1.ResourceReference{
+					UID:  objectMeta.UID,
+					Name: objectMeta.Name,
+					Kind: refKind,
+				},
+			},
+		},
 	}
+	return workloadInstance
+}
+
+func TestKeptnWorkloadInstanceReconciler_reconcileDeployment_FailedStatefulSet(t *testing.T) {
 
-	isOwnerRunning, err = r.isReferencedWorkloadRunning(context.TODO(), klcv1alpha1.ResourceReference{UID: "mystat", Name: "mystat", Kind: "StatefulSet"}, "default")
+	rep := int32(1)
+	statefulsetFail := makeStatefulSet("mystat", "default", &rep, 0)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(statefulsetFail).Build()
+
+	err := klcv1alpha1.AddToScheme(fakeClient.Scheme())
 	testrequire.Nil(t, err)
-	if isOwnerRunning {
-		t.Errorf("Should fail!")
+
+	workloadInstance := makeWorkloadInstanceWithRef(statefulsetFail.ObjectMeta, "StatefulSet")
+
+	err = fakeClient.Create(context.TODO(), workloadInstance)
+	require.Nil(t, err)
+
+	r := &KeptnWorkloadInstanceReconciler{
+		Client: fakeClient,
 	}
 
-	replicasetPass := makeReplicaSet("myrep", "default", &rep, 1)
-	statefulsetPass := makeStatefulSet("mystat", "default", &rep, 1)
+	keptnState, err := r.reconcileDeployment(context.TODO(), workloadInstance)
+	testrequire.Nil(t, err)
+	testrequire.Equal(t, common.StateProgressing, keptnState)
+}
+
+func TestKeptnWorkloadInstanceReconciler_reconcileDeployment_FailedDaemonSet(t *testing.T) {
 
-	r2 := &KeptnWorkloadInstanceReconciler{
-		Client: fake.NewClientBuilder().WithObjects(replicasetPass, statefulsetPass).Build(),
+	daemonSetFail := makeDaemonSet("mystat", "default", 1, 0)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(daemonSetFail).Build()
+
+	err := klcv1alpha1.AddToScheme(fakeClient.Scheme())
+	testrequire.Nil(t, err)
+
+	workloadInstance := makeWorkloadInstanceWithRef(daemonSetFail.ObjectMeta, "DaemonSet")
+
+	err = fakeClient.Create(context.TODO(), workloadInstance)
+	require.Nil(t, err)
+
+	r := &KeptnWorkloadInstanceReconciler{
+		Client: fakeClient,
+	}
+
+	keptnState, err := r.reconcileDeployment(context.TODO(), workloadInstance)
+	testrequire.Nil(t, err)
+	testrequire.Equal(t, common.StateProgressing, keptnState)
+}
+
+func TestKeptnWorkloadInstanceReconciler_reconcileDeployment_ReadyReplicaSet(t *testing.T) {
+
+	rep := int32(1)
+	replicaSet := makeReplicaSet("myrep", "default", &rep, 1)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(replicaSet).Build()
+
+	err := klcv1alpha1.AddToScheme(fakeClient.Scheme())
+	testrequire.Nil(t, err)
+
+	workloadInstance := makeWorkloadInstanceWithRef(replicaSet.ObjectMeta, "ReplicaSet")
+
+	err = fakeClient.Create(context.TODO(), workloadInstance)
+	require.Nil(t, err)
+
+	r := &KeptnWorkloadInstanceReconciler{
+		Client: fakeClient,
 	}
-	isOwnerRunning, err = r2.isReferencedWorkloadRunning(context.TODO(), klcv1alpha1.ResourceReference{UID: "myrep", Name: "myrep", Kind: "ReplicaSet"}, "default")
+
+	keptnState, err := r.reconcileDeployment(context.TODO(), workloadInstance)
+	testrequire.Nil(t, err)
+	testrequire.Equal(t, common.StateSucceeded, keptnState)
+}
+
+func TestKeptnWorkloadInstanceReconciler_reconcileDeployment_ReadyStatefulSet(t *testing.T) {
+
+	rep := int32(1)
+	statefulSet := makeStatefulSet("mystat", "default", &rep, 1)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(statefulSet).Build()
+
+	err := klcv1alpha1.AddToScheme(fakeClient.Scheme())
 	testrequire.Nil(t, err)
-	if !isOwnerRunning {
-		t.Errorf("Should find a replica owner!")
+
+	workloadInstance := makeWorkloadInstanceWithRef(statefulSet.ObjectMeta, "StatefulSet")
+
+	err = fakeClient.Create(context.TODO(), workloadInstance)
+	require.Nil(t, err)
+
+	r := &KeptnWorkloadInstanceReconciler{
+		Client: fakeClient,
 	}
 
-	isOwnerRunning, err = r2.isReferencedWorkloadRunning(context.TODO(), klcv1alpha1.ResourceReference{UID: "mystat", Name: "mystat", Kind: "StatefulSet"}, "default")
+	keptnState, err := r.reconcileDeployment(context.TODO(), workloadInstance)
 	testrequire.Nil(t, err)
-	if !isOwnerRunning {
-		t.Errorf("Should find a stateful set owner!")
+	testrequire.Equal(t, common.StateSucceeded, keptnState)
+}
+
+func TestKeptnWorkloadInstanceReconciler_reconcileDeployment_ReadyDaemonSet(t *testing.T) {
+
+	daemonSet := makeDaemonSet("mystat", "default", 1, 1)
+
+	fakeClient := fake.NewClientBuilder().WithObjects(daemonSet).Build()
+
+	err := klcv1alpha1.AddToScheme(fakeClient.Scheme())
+	testrequire.Nil(t, err)
+
+	workloadInstance := makeWorkloadInstanceWithRef(daemonSet.ObjectMeta, "DaemonSet")
+
+	err = fakeClient.Create(context.TODO(), workloadInstance)
+	require.Nil(t, err)
+
+	r := &KeptnWorkloadInstanceReconciler{
+		Client: fakeClient,
 	}
 
+	keptnState, err := r.reconcileDeployment(context.TODO(), workloadInstance)
+	testrequire.Nil(t, err)
+	testrequire.Equal(t, common.StateSucceeded, keptnState)
 }
 
 func TestKeptnWorkloadInstanceReconciler_IsPodRunning(t *testing.T) {
@@ -135,6 +256,26 @@ func makeStatefulSet(name string, namespace string, wanted *int32, available int
 
 }
 
+func makeDaemonSet(name string, namespace string, wanted int32, available int32) *appsv1.DaemonSet {
+
+	return &appsv1.DaemonSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "StatefulSet",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			UID:       types.UID(name),
+		},
+		Spec: appsv1.DaemonSetSpec{},
+		Status: appsv1.DaemonSetStatus{
+			DesiredNumberScheduled: wanted,
+			NumberReady:            available,
+		},
+	}
+
+}
+
 func Test_getLatestAppVersion(t *testing.T) {
 	type args struct {
 		apps *klcv1alpha1.KeptnAppVersionList

operator/controllers/keptnworkloadinstance/reconcile_deploymentstate.go

@@ -2,6 +2,8 @@ package keptnworkloadinstance
 
 import (
 	"context"
+	controllercommon "github.com/keptn/lifecycle-toolkit/operator/controllers/common"
+
 	klcv1alpha1 "github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1"
 	"github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
 	appsv1 "k8s.io/api/apps/v1"
@@ -11,60 +13,54 @@ import (
 )
 
 func (r *KeptnWorkloadInstanceReconciler) reconcileDeployment(ctx context.Context, workloadInstance *klcv1alpha1.KeptnWorkloadInstance) (common.KeptnState, error) {
-	if workloadInstance.Spec.ResourceReference.Kind == "Pod" {
-		isPodRunning, err := r.isPodRunning(ctx, workloadInstance.Spec.ResourceReference, workloadInstance.Namespace)
-		if err != nil {
-			return common.StateUnknown, err
-		}
-		if isPodRunning {
-			workloadInstance.Status.DeploymentStatus = common.StateSucceeded
-		} else {
-			workloadInstance.Status.DeploymentStatus = common.StateProgressing
-		}
+	var isRunning bool
+	var err error
+
+	switch workloadInstance.Spec.ResourceReference.Kind {
+	case "Pod":
+		isRunning, err = r.isPodRunning(ctx, workloadInstance.Spec.ResourceReference, workloadInstance.Namespace)
+	case "ReplicaSet":
+		isRunning, err = r.isReplicaSetRunning(ctx, workloadInstance.Spec.ResourceReference, workloadInstance.Namespace)
+	case "StatefulSet":
+		isRunning, err = r.isStatefulSetRunning(ctx, workloadInstance.Spec.ResourceReference, workloadInstance.Namespace)
+	case "DaemonSet":
+		isRunning, err = r.isDaemonSetRunning(ctx, workloadInstance.Spec.ResourceReference, workloadInstance.Namespace)
+	default:
+		isRunning, err = false, controllercommon.ErrUnsupportedWorkloadInstanceResourceReference
+	}
+
+	if err != nil {
+		return common.StateUnknown, err
+	}
+	if isRunning {
+		workloadInstance.Status.DeploymentStatus = common.StateSucceeded
 	} else {
-		isReplicaRunning, err := r.isReferencedWorkloadRunning(ctx, workloadInstance.Spec.ResourceReference, workloadInstance.Namespace)
-		if err != nil {
-			return common.StateUnknown, err
-		}
-		if isReplicaRunning {
-			workloadInstance.Status.DeploymentStatus = common.StateSucceeded
-		} else {
-			workloadInstance.Status.DeploymentStatus = common.StateProgressing
-		}
+		workloadInstance.Status.DeploymentStatus = common.StateProgressing
 	}
 
-	err := r.Client.Status().Update(ctx, workloadInstance)
+	err = r.Client.Status().Update(ctx, workloadInstance)
 	if err != nil {
 		return common.StateUnknown, err
 	}
 	return workloadInstance.Status.DeploymentStatus, nil
 }
 
-func (r *KeptnWorkloadInstanceReconciler) isReferencedWorkloadRunning(ctx context.Context, resource klcv1alpha1.ResourceReference, namespace string) (bool, error) {
-
-	var replicas *int32
-	var desiredReplicas int32
-	switch resource.Kind {
-	case "ReplicaSet":
-		rep := appsv1.ReplicaSet{}
-		err := r.Client.Get(ctx, types.NamespacedName{Name: resource.Name, Namespace: namespace}, &rep)
-		if err != nil {
-			return false, err
-		}
-		replicas = rep.Spec.Replicas
-		desiredReplicas = rep.Status.AvailableReplicas
-	case "StatefulSet":
-		sts := appsv1.StatefulSet{}
-		err := r.Client.Get(ctx, types.NamespacedName{Name: resource.Name, Namespace: namespace}, &sts)
-		if err != nil {
-			return false, err
-		}
-		replicas = sts.Spec.Replicas
-		desiredReplicas = sts.Status.AvailableReplicas
+func (r *KeptnWorkloadInstanceReconciler) isReplicaSetRunning(ctx context.Context, resource klcv1alpha1.ResourceReference, namespace string) (bool, error) {
+	rep := appsv1.ReplicaSet{}
+	err := r.Client.Get(ctx, types.NamespacedName{Name: resource.Name, Namespace: namespace}, &rep)
+	if err != nil {
+		return false, err
 	}
+	return *rep.Spec.Replicas == rep.Status.AvailableReplicas, nil
+}
 
-	return *replicas == desiredReplicas, nil
-
+func (r *KeptnWorkloadInstanceReconciler) isDaemonSetRunning(ctx context.Context, resource klcv1alpha1.ResourceReference, namespace string) (bool, error) {
+	daemonSet := &appsv1.DaemonSet{}
+	err := r.Client.Get(ctx, types.NamespacedName{Name: resource.Name, Namespace: namespace}, daemonSet)
+	if err != nil {
+		return false, err
+	}
+	return daemonSet.Status.DesiredNumberScheduled == daemonSet.Status.NumberReady, nil
 }
 
 func (r *KeptnWorkloadInstanceReconciler) isPodRunning(ctx context.Context, resource klcv1alpha1.ResourceReference, namespace string) (bool, error) {
@@ -82,3 +78,12 @@ func (r *KeptnWorkloadInstanceReconciler) isPodRunning(ctx context.Context, reso
 	}
 	return false, nil
 }
+
+func (r *KeptnWorkloadInstanceReconciler) isStatefulSetRunning(ctx context.Context, resource klcv1alpha1.ResourceReference, namespace string) (bool, error) {
+	sts := appsv1.StatefulSet{}
+	err := r.Client.Get(ctx, types.NamespacedName{Name: resource.Name, Namespace: namespace}, &sts)
+	if err != nil {
+		return false, err
+	}
+	return *sts.Spec.Replicas == sts.Status.AvailableReplicas, nil
+}