From 2340c1cdeff2a582e495632ca586bfb322e3ea14 Mon Sep 17 00:00:00 2001 From: Melissa Lee <43827333+halim-lee@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:29:02 -0500 Subject: [PATCH] Update frequency of reconcile and parameterize the reconcile settings (#661) --- api/v1/runtimecomponent_types.go | 33 +++++ api/v1/zz_generated.deepcopy.go | 10 ++ api/v1beta2/runtimecomponent_types.go | 20 +++ .../rc.app.stacks_runtimecomponents.yaml | 9 ++ ...ntime-component.clusterserviceversion.yaml | 2 +- common/config.go | 35 +++++ common/types.go | 8 ++ .../rc.app.stacks_runtimecomponents.yaml | 9 ++ .../controller/runtimecomponent_controller.go | 8 ++ .../deploy/kubectl/runtime-component-crd.yaml | 9 ++ .../daily/base/runtime-component-crd.yaml | 9 ++ utils/reconciler.go | 132 ++++++++++++++---- utils/status.go | 22 +-- 13 files changed, 267 insertions(+), 39 deletions(-) diff --git a/api/v1/runtimecomponent_types.go b/api/v1/runtimecomponent_types.go index 601ce10da..44caf1940 100644 --- a/api/v1/runtimecomponent_types.go +++ b/api/v1/runtimecomponent_types.go @@ -431,6 +431,9 @@ type RuntimeComponentStatus struct { // The generation identifier of this RuntimeComponent instance completely reconciled by the Operator. ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // The reconciliation interval in seconds. + ReconcileInterval *int32 `json:"reconcileInterval,omitempty"` } // Defines possible status conditions. @@ -440,6 +443,9 @@ type StatusCondition struct { Message string `json:"message,omitempty"` Status corev1.ConditionStatus `json:"status,omitempty"` Type StatusConditionType `json:"type,omitempty"` + + // The count of the number of reconciles the condition status type has not changed. + UnchangedConditionCount *int32 `json:"unchangedConditionCount,omitempty"` } // Defines the type of status condition. @@ -1190,6 +1196,7 @@ func (s *RuntimeComponentStatus) SetCondition(c common.StatusCondition) { condition.SetMessage(c.GetMessage()) condition.SetStatus(c.GetStatus()) condition.SetType(c.GetType()) + condition.SetUnchangedConditionCount(c.GetUnchangedConditionCount()) if !found { s.Conditions = append(s.Conditions, *condition) } @@ -1222,6 +1229,14 @@ func (s *RuntimeComponentStatus) SetReferences(refs common.StatusReferences) { s.References = refs } +func (s *RuntimeComponentStatus) GetReconcileInterval() *int32 { + return s.ReconcileInterval +} + +func (s *RuntimeComponentStatus) SetReconcileInterval(interval *int32) { + s.ReconcileInterval = interval +} + func (s *RuntimeComponentStatus) SetReference(name string, value string) { if s.References == nil { s.References = make(common.StatusReferences) @@ -1229,6 +1244,24 @@ func (s *RuntimeComponentStatus) SetReference(name string, value string) { s.References[name] = value } +func (sc *StatusCondition) GetUnchangedConditionCount() *int32 { + return sc.UnchangedConditionCount +} + +func (sc *StatusCondition) SetUnchangedConditionCount(count *int32) { + sc.UnchangedConditionCount = count +} + +func (s *RuntimeComponentStatus) UnsetUnchangedConditionCount(conditionType common.StatusConditionType) { + // Reset unchanged count for other status conditions + var emptyCount *int32 + for i := range s.Conditions { + if s.Conditions[i].GetType() != conditionType && s.Conditions[i].GetUnchangedConditionCount() != nil { + s.Conditions[i].SetUnchangedConditionCount(emptyCount) + } + } +} + func convertToCommonStatusConditionType(c StatusConditionType) common.StatusConditionType { switch c { case StatusConditionTypeReconciled: diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 6891ebb9e..41e44c313 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -705,6 +705,11 @@ func (in *RuntimeComponentStatus) DeepCopyInto(out *RuntimeComponentStatus) { (*out)[key] = val } } + if in.ReconcileInterval != nil { + in, out := &in.ReconcileInterval, &out.ReconcileInterval + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeComponentStatus. @@ -877,6 +882,11 @@ func (in *StatusCondition) DeepCopyInto(out *StatusCondition) { in, out := &in.LastTransitionTime, &out.LastTransitionTime *out = (*in).DeepCopy() } + if in.UnchangedConditionCount != nil { + in, out := &in.UnchangedConditionCount, &out.UnchangedConditionCount + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatusCondition. diff --git a/api/v1beta2/runtimecomponent_types.go b/api/v1beta2/runtimecomponent_types.go index f18249215..7374115c6 100644 --- a/api/v1beta2/runtimecomponent_types.go +++ b/api/v1beta2/runtimecomponent_types.go @@ -381,6 +381,26 @@ func (in *RuntimeComponentStatus) SetReference(s string, s2 string) { return } +func (s *RuntimeComponentStatus) GetReconcileInterval() *int32 { + return nil +} + +func (s *RuntimeComponentStatus) SetReconcileInterval(interval *int32) { + return +} + +func (s *StatusCondition) GetUnchangedConditionCount() *int32 { + return nil +} + +func (s *StatusCondition) SetUnchangedConditionCount(count *int32) { + return +} + +func (s *RuntimeComponentStatus) UnsetUnchangedConditionCount(conditionType common.StatusConditionType) { + return +} + // Defines possible status conditions. type StatusCondition struct { LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` diff --git a/bundle/manifests/rc.app.stacks_runtimecomponents.yaml b/bundle/manifests/rc.app.stacks_runtimecomponents.yaml index 27aff8754..0baa1cbfa 100644 --- a/bundle/manifests/rc.app.stacks_runtimecomponents.yaml +++ b/bundle/manifests/rc.app.stacks_runtimecomponents.yaml @@ -7829,6 +7829,11 @@ spec: type: description: Defines the type of status condition. type: string + unchangedConditionCount: + description: The count of the number of reconciles the condition + status type has not changed. + format: int32 + type: integer type: object type: array x-kubernetes-list-type: atomic @@ -7855,6 +7860,10 @@ spec: completely reconciled by the Operator. format: int64 type: integer + reconcileInterval: + description: The reconciliation interval in seconds. + format: int32 + type: integer references: additionalProperties: type: string diff --git a/bundle/manifests/runtime-component.clusterserviceversion.yaml b/bundle/manifests/runtime-component.clusterserviceversion.yaml index 56953a8c8..8fe32c3b6 100644 --- a/bundle/manifests/runtime-component.clusterserviceversion.yaml +++ b/bundle/manifests/runtime-component.clusterserviceversion.yaml @@ -71,7 +71,7 @@ metadata: categories: Application Runtime certified: "true" containerImage: icr.io/appcafe/runtime-component-operator:daily - createdAt: "2024-09-20T18:58:02Z" + createdAt: "2024-11-15T15:49:28Z" description: Deploys any runtime component with dynamic and auto-tuning configuration features.operators.openshift.io/disconnected: "true" features.operators.openshift.io/fips-compliant: "true" diff --git a/common/config.go b/common/config.go index 89ce80208..cf357fc4c 100644 --- a/common/config.go +++ b/common/config.go @@ -1,6 +1,9 @@ package common import ( + "errors" + "strconv" + corev1 "k8s.io/api/core/v1" ) @@ -17,6 +20,13 @@ const ( // OpConfigCMCADuration default duration for cert-manager issued service certificate OpConfigCMCertDuration = "certManagerCertDuration" + + // OpConfigReconcileIntervalSeconds default reconciliation interval in seconds + OpConfigReconcileIntervalSeconds = "reconcileIntervalSeconds" + + // OpConfigReconcileIntervalPercentage default reconciliation interval increase, represented as a percentage (100 equaling to 100%) + // When the reconciliation interval needs to increase, it will increase by the given percentage + OpConfigReconcileIntervalPercentage = "reconcileIntervalIncreasePercentage" ) // Config stores operator configuration @@ -32,6 +42,29 @@ func (oc OpConfig) LoadFromConfigMap(cm *corev1.ConfigMap) { oc[k] = v } } +func (oc OpConfig) CheckValidValue(key string, OperatorName string) error { + value := oc[key] + + intValue, err := strconv.Atoi(value) + if err != nil { + oc.SetConfigMapDefaultValue(key) + return errors.New(key + " in ConfigMap: " + OperatorName + " has an invalid syntax, error: " + err.Error()) + } else if key == OpConfigReconcileIntervalSeconds && intValue <= 0 { + oc.SetConfigMapDefaultValue(key) + return errors.New(key + " in ConfigMap: " + OperatorName + " is set to " + value + ". It must be greater than 0.") + } else if key == OpConfigReconcileIntervalPercentage && intValue < 0 { + oc.SetConfigMapDefaultValue(key) + return errors.New(key + " in ConfigMap: " + OperatorName + " is set to " + value + ". It must be greater than or equal to 0.") + } + + return nil +} + +// SetConfigMapDefaultValue sets default value for specified key +func (oc OpConfig) SetConfigMapDefaultValue(key string) { + cm := DefaultOpConfig() + oc[key] = cm[key] +} // DefaultOpConfig returns default configuration func DefaultOpConfig() OpConfig { @@ -39,5 +72,7 @@ func DefaultOpConfig() OpConfig { cfg[OpConfigDefaultHostname] = "" cfg[OpConfigCMCADuration] = "8766h" cfg[OpConfigCMCertDuration] = "2160h" + cfg[OpConfigReconcileIntervalSeconds] = "15" + cfg[OpConfigReconcileIntervalPercentage] = "100" return cfg } diff --git a/common/types.go b/common/types.go index c92e82f43..8710eca01 100644 --- a/common/types.go +++ b/common/types.go @@ -41,6 +41,9 @@ type StatusCondition interface { GetType() StatusConditionType SetType(StatusConditionType) + GetUnchangedConditionCount() *int32 + SetUnchangedConditionCount(*int32) + SetConditionFields(string, string, corev1.ConditionStatus) StatusCondition } @@ -83,6 +86,11 @@ type BaseComponentStatus interface { GetReferences() StatusReferences SetReferences(StatusReferences) SetReference(string, string) + + GetReconcileInterval() *int32 + SetReconcileInterval(*int32) + + UnsetUnchangedConditionCount(conditionType StatusConditionType) } const ( diff --git a/config/crd/bases/rc.app.stacks_runtimecomponents.yaml b/config/crd/bases/rc.app.stacks_runtimecomponents.yaml index 0fe228826..f4735ad15 100644 --- a/config/crd/bases/rc.app.stacks_runtimecomponents.yaml +++ b/config/crd/bases/rc.app.stacks_runtimecomponents.yaml @@ -7825,6 +7825,11 @@ spec: type: description: Defines the type of status condition. type: string + unchangedConditionCount: + description: The count of the number of reconciles the condition + status type has not changed. + format: int32 + type: integer type: object type: array x-kubernetes-list-type: atomic @@ -7851,6 +7856,10 @@ spec: completely reconciled by the Operator. format: int64 type: integer + reconcileInterval: + description: The reconciliation interval in seconds. + format: int32 + type: integer references: additionalProperties: type: string diff --git a/internal/controller/runtimecomponent_controller.go b/internal/controller/runtimecomponent_controller.go index 13aacb90a..281f18a1c 100644 --- a/internal/controller/runtimecomponent_controller.go +++ b/internal/controller/runtimecomponent_controller.go @@ -123,6 +123,14 @@ func (r *RuntimeComponentReconciler) Reconcile(ctx context.Context, req ctrl.Req return reconcile.Result{}, err } + if err = common.Config.CheckValidValue(common.OpConfigReconcileIntervalSeconds, OperatorName); err != nil { + return r.ManageError(err, common.StatusConditionTypeReconciled, instance) + } + + if err = common.Config.CheckValidValue(common.OpConfigReconcileIntervalPercentage, OperatorName); err != nil { + return r.ManageError(err, common.StatusConditionTypeReconciled, instance) + } + isKnativeSupported, err := r.IsGroupVersionSupported(servingv1.SchemeGroupVersion.String(), "Service") if err != nil { r.ManageError(err, common.StatusConditionTypeReconciled, instance) diff --git a/internal/deploy/kubectl/runtime-component-crd.yaml b/internal/deploy/kubectl/runtime-component-crd.yaml index 781d2fe4b..68f5afa3b 100644 --- a/internal/deploy/kubectl/runtime-component-crd.yaml +++ b/internal/deploy/kubectl/runtime-component-crd.yaml @@ -7828,6 +7828,11 @@ spec: type: description: Defines the type of status condition. type: string + unchangedConditionCount: + description: The count of the number of reconciles the condition + status type has not changed. + format: int32 + type: integer type: object type: array x-kubernetes-list-type: atomic @@ -7854,6 +7859,10 @@ spec: completely reconciled by the Operator. format: int64 type: integer + reconcileInterval: + description: The reconciliation interval in seconds. + format: int32 + type: integer references: additionalProperties: type: string diff --git a/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml b/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml index 781d2fe4b..68f5afa3b 100644 --- a/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml +++ b/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml @@ -7828,6 +7828,11 @@ spec: type: description: Defines the type of status condition. type: string + unchangedConditionCount: + description: The count of the number of reconciles the condition + status type has not changed. + format: int32 + type: integer type: object type: array x-kubernetes-list-type: atomic @@ -7854,6 +7859,10 @@ spec: completely reconciled by the Operator. format: int64 type: integer + reconcileInterval: + description: The reconciliation interval in seconds. + format: int32 + type: integer references: additionalProperties: type: string diff --git a/utils/reconciler.go b/utils/reconciler.go index cdbabb4f5..1d3fcecad 100644 --- a/utils/reconciler.go +++ b/utils/reconciler.go @@ -5,6 +5,8 @@ import ( "encoding/pem" "errors" "fmt" + "math" + "strconv" "time" networkingv1 "k8s.io/api/networking/v1" @@ -30,10 +32,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -const ( - ReconcileInterval = 15 -) - // ReconcilerBase base reconciler with some common behaviour type ReconcilerBase struct { apiReader client.Reader @@ -198,6 +196,52 @@ func addStatusWarnings(ba common.BaseComponent) { } +func getBaseReconcileInterval(newCondition common.StatusCondition, s common.BaseComponentStatus) time.Duration { + var newCount *int32 + newCondition.SetUnchangedConditionCount(newCount) + + baseIntervalInt, _ := strconv.Atoi(common.Config[common.OpConfigReconcileIntervalSeconds]) + baseInterval := int32(baseIntervalInt) + s.SetReconcileInterval(&baseInterval) + + return time.Duration(baseInterval) * time.Second +} + +func updateReconcileInterval(maxSeconds int, oldCondition common.StatusCondition, newCondition common.StatusCondition, s common.BaseComponentStatus) time.Duration { + oldReconcileInterval := float64(*s.GetReconcileInterval()) + + var newCount int32 + if count := oldCondition.GetUnchangedConditionCount(); count == nil { + newCount = 0 + oldReconcileInterval, _ = strconv.ParseFloat(common.Config[common.OpConfigReconcileIntervalSeconds], 64) + baseInterval := int32(oldReconcileInterval) + s.SetReconcileInterval(&baseInterval) + } else { + newCount = *count + 1 + } + newCondition.SetUnchangedConditionCount(&newCount) + + s.UnsetUnchangedConditionCount(newCondition.GetType()) + s.SetCondition(newCondition) + + // For every repeated 2 reconciliation errors, increase reconcile period + if newCount >= 2 && newCount%2 == 0 { + intervalIncreasePercentage, _ := strconv.ParseFloat(common.Config[common.OpConfigReconcileIntervalPercentage], 64) + exp := float64(newCount / 2) + increase := math.Pow(1+(intervalIncreasePercentage/100), exp) + baseInterval, _ := strconv.ParseFloat(common.Config[common.OpConfigReconcileIntervalSeconds], 64) + newInterval := int32(baseInterval * increase) + + // Only increase to the maximum interval + if newInterval < 0 || newInterval >= int32(maxSeconds) { + newInterval = int32(maxSeconds) + } + s.SetReconcileInterval(&newInterval) + return time.Duration(newInterval) * time.Second + } + return time.Duration(oldReconcileInterval) * time.Second +} + // ManageError ... func (r *ReconcilerBase) ManageError(issue error, conditionType common.StatusConditionType, ba common.BaseComponent) (reconcile.Result, error) { s := ba.GetStatus() @@ -207,12 +251,11 @@ func (r *ReconcilerBase) ManageError(issue error, conditionType common.StatusCon r.GetRecorder().Event(obj, "Warning", "ProcessingError", issue.Error()) oldCondition := s.GetCondition(conditionType) - newCondition := s.NewCondition(conditionType) newCondition.SetReason(string(apierrors.ReasonForError(issue))) newCondition.SetMessage(issue.Error()) newCondition.SetStatus(corev1.ConditionFalse) - s.SetCondition(newCondition) + r.setCondition(ba, oldCondition, newCondition) addStatusWarnings(ba) @@ -234,6 +277,18 @@ func (r *ReconcilerBase) ManageError(issue error, conditionType common.StatusCon s.SetCondition(reconciledNewCondition) } + var retryInterval time.Duration + // If the application was reconciled and now it is not or encountered a different error + // Use the default reconcile interval + if oldCondition == nil || s.GetReconcileInterval() == nil || oldCondition.GetStatus() != newCondition.GetStatus() || oldCondition.GetMessage() != newCondition.GetMessage() { + retryInterval = getBaseReconcileInterval(newCondition, s) + } else { + // If the application fails to reconcile again and the error message has not changed + // Increase the retry interval upto maxSeconds + maxSeconds := 240 // Max 4 minutes + retryInterval = updateReconcileInterval(maxSeconds, oldCondition, newCondition, s) + } + err := r.UpdateStatus(obj) if err != nil { @@ -247,14 +302,6 @@ func (r *ReconcilerBase) ManageError(issue error, conditionType common.StatusCon }, nil } - var retryInterval time.Duration - // If the application was reconciled and now it is not - if oldCondition == nil || oldCondition.GetStatus() == corev1.ConditionTrue { - retryInterval = time.Second - } else { - retryInterval = 5 * time.Second - } - return reconcile.Result{ //RequeueAfter: time.Duration(math.Min(float64(retryInterval.Nanoseconds()*2), float64(time.Hour.Nanoseconds()*6))), RequeueAfter: retryInterval, @@ -265,17 +312,51 @@ func (r *ReconcilerBase) ManageError(issue error, conditionType common.StatusCon // ManageSuccess ... func (r *ReconcilerBase) ManageSuccess(conditionType common.StatusConditionType, ba common.BaseComponent) (reconcile.Result, error) { s := ba.GetStatus() + oldRecCondition := s.GetCondition(conditionType) - statusCondition := s.NewCondition(conditionType) - statusCondition.SetReason("") - statusCondition.SetMessage("") - statusCondition.SetStatus(corev1.ConditionTrue) - s.SetCondition(statusCondition) + newRecCondition := s.NewCondition(conditionType) + newRecCondition.SetReason("") + newRecCondition.SetMessage("") + newRecCondition.SetStatus(corev1.ConditionTrue) + s.SetCondition(newRecCondition) addStatusWarnings(ba) - //Check application status (reconciliation & resource status & endpoint status) - readyStatus := r.CheckApplicationStatus(ba) + // Check application status (reconciliation & resource condition & endpoint condition) + // CheckApplicationStatus returns overall Application condition if ready + // If not ready, it returns Resources condition + oldCondition, newCondition := r.CheckApplicationStatus(ba) + + var retryInterval time.Duration + + // If the resources are not ready + if newCondition.GetStatus() != corev1.ConditionTrue { + // If the resources were ready and now they are not, or encountered a different error + // Use the default reconcile interval + if oldCondition == nil || oldCondition.GetStatus() != newCondition.GetStatus() || oldCondition.GetMessage() != newCondition.GetMessage() { + retryInterval = getBaseReconcileInterval(newCondition, s) + } else { + // If the resources stay unready and the error message has not changed + // Increase the retry interval upto maxSeconds + maxSeconds := 240 // Max 4 minutes + retryInterval = updateReconcileInterval(maxSeconds, oldCondition, newCondition, s) + } + } else { // If the application and resources are ready + if oldRecCondition == nil || oldRecCondition.GetStatus() != newRecCondition.GetStatus() || oldRecCondition.GetMessage() != newRecCondition.GetMessage() { + // If the application or the resources were not reconciled before + // Use the default reconcile interval + retryInterval = getBaseReconcileInterval(newRecCondition, s) + } else if oldCondition == nil || oldCondition.GetStatus() != newCondition.GetStatus() || oldCondition.GetMessage() != newCondition.GetMessage() { + // Or if the application (resources) were not ready before + // Use the default reconcile interval + retryInterval = getBaseReconcileInterval(newCondition, s) + } else { + // If the application and resources stay ready and there are no changes + // Increase the retry interval upto maxSeconds + maxSeconds := 120 // Max 2 minutes + retryInterval = updateReconcileInterval(maxSeconds, oldCondition, newCondition, s) + } + } err := r.UpdateStatus(ba.(client.Object)) if err != nil { @@ -286,15 +367,6 @@ func (r *ReconcilerBase) ManageSuccess(conditionType common.StatusConditionType, }, nil } - var retryInterval time.Duration - - // If resources are not ready - if readyStatus != corev1.ConditionTrue { - retryInterval = time.Second - } else { - retryInterval = ReconcileInterval * time.Second - } - return reconcile.Result{RequeueAfter: retryInterval}, nil } diff --git a/utils/status.go b/utils/status.go index 5747ade0a..23873c37d 100644 --- a/utils/status.go +++ b/utils/status.go @@ -29,13 +29,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func (r *ReconcilerBase) CheckApplicationStatus(ba common.BaseComponent) corev1.ConditionStatus { +func (r *ReconcilerBase) CheckApplicationStatus(ba common.BaseComponent) (common.StatusCondition, common.StatusCondition) { s := ba.GetStatus() status, msg, reason := corev1.ConditionFalse, "", "" // Check application and resources status scReconciled := s.GetCondition(common.StatusConditionTypeReconciled) + oldResourcesCondition := s.GetCondition(common.StatusConditionTypeResourcesReady) + var newResourcesCondition common.StatusCondition // If not reconciled, resources and endpoints will not be ready and reconciled status will show the errors if scReconciled == nil || scReconciled.GetStatus() != corev1.ConditionTrue { @@ -46,8 +48,8 @@ func (r *ReconcilerBase) CheckApplicationStatus(ba common.BaseComponent) corev1. r.CheckResourcesStatus(ba) r.ReportExternalEndpointStatus(ba) - scReady := s.GetCondition(common.StatusConditionTypeResourcesReady) - if scReady == nil || scReady.GetStatus() != corev1.ConditionTrue { + newResourcesCondition = s.GetCondition(common.StatusConditionTypeResourcesReady) + if newResourcesCondition == nil || newResourcesCondition.GetStatus() != corev1.ConditionTrue { msg = "Resources are not ready." reason = "ResourcesNotReady" } else { @@ -58,12 +60,16 @@ func (r *ReconcilerBase) CheckApplicationStatus(ba common.BaseComponent) corev1. // Check Application Ready status condition is created/updated conditionType := common.StatusConditionTypeReady - oldCondition := s.GetCondition(conditionType) - newCondition := s.NewCondition(conditionType) - newCondition.SetConditionFields(msg, reason, status) - r.setCondition(ba, oldCondition, newCondition) + oldAppCondition := s.GetCondition(conditionType) + newAppCondition := s.NewCondition(conditionType) + newAppCondition.SetConditionFields(msg, reason, status) + r.setCondition(ba, oldAppCondition, newAppCondition) - return status + if status == corev1.ConditionFalse { + return oldResourcesCondition, newResourcesCondition + } else { + return oldAppCondition, newAppCondition + } } func (r *ReconcilerBase) CheckResourcesStatus(ba common.BaseComponent) {