From d886297f211af39c5681fbd22cce6d4a05f897f6 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Fri, 22 Nov 2024 12:31:43 +0100 Subject: [PATCH] Create basic Reconcile methods for WatcherAPI This patch is basic recond methods to the watcherapi controller, as well as some initial input validation. The change adds the required CR spec and status, and creates the initial structure for initialization and deletion. In addition it some some input validation by accessing the osp secret and the database that should be created by the watcher controller. Finally, it is also adding some initial structure test for functional envtest testing in WatcherAPI. Related: OSPRH-11483 --- .../watcher.openstack.org_watcherapis.yaml | 81 ++++++- api/v1beta1/watcherapi_types.go | 17 +- api/v1beta1/zz_generated.deepcopy.go | 10 +- .../watcher.openstack.org_watcherapis.yaml | 81 ++++++- controllers/watcherapi_controller.go | 214 +++++++++++++++++- tests/functional/base_test.go | 26 +++ .../functional/watcherapi_controller_test.go | 106 +++++++++ 7 files changed, 513 insertions(+), 22 deletions(-) create mode 100644 tests/functional/watcherapi_controller_test.go diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index 9330018..93b499c 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -35,13 +35,88 @@ spec: spec: description: WatcherAPISpec defines the desired state of WatcherAPI properties: - foo: - description: Foo is an example field of WatcherAPI. Edit watcherapi_types.go - to remove/update + databaseAccount: + default: watcher + description: DatabaseAccount - MariaDBAccount CR name used for watcher + DB, defaults to watcher type: string + databaseInstance: + description: MariaDB instance name Required to use the mariadb-operator + instance to create the DB and user + type: string + passwordSelectors: + default: + service: WatcherPassword + description: PasswordSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + service: + default: WatcherPassword + description: Service - Selector to get the watcher service user + password from the Secret + type: string + type: object + secret: + default: osp-secret + description: Secret containing all passwords / keys needed + type: string + required: + - databaseInstance type: object status: description: WatcherAPIStatus defines the observed state of WatcherAPI + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: Severity provides a classification of Reason code, + so the current situation is immediately understandable and + could act accordingly. It is meant for situations where Status=False + and it should be indicated if it is just informational, warning + (next reconciliation might fix it) or an error (e.g. DB create + issue and no actions to automatically resolve the issue can/should + be done). For conditions where Status=Unknown or Status=True + the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes + injected by the opentack-operator in the top-level CR (e.g. the + ContainerImage) + format: int64 + type: integer type: object type: object served: true diff --git a/api/v1beta1/watcherapi_types.go b/api/v1beta1/watcherapi_types.go index 8c63bbf..4b1ce29 100644 --- a/api/v1beta1/watcherapi_types.go +++ b/api/v1beta1/watcherapi_types.go @@ -17,25 +17,28 @@ limitations under the License. package v1beta1 import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // WatcherAPISpec defines the desired state of WatcherAPI type WatcherAPISpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - // Foo is an example field of WatcherAPI. Edit watcherapi_types.go to remove/update - Foo string `json:"foo,omitempty"` + WatcherTemplate `json:",inline"` } // WatcherAPIStatus defines the observed state of WatcherAPI type WatcherAPIStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ObservedGeneration - the most recent generation observed for this + // service. If the observed generation is less than the spec generation, + // then the controller has not processed the latest changes injected by + // the opentack-operator in the top-level CR (e.g. the ContainerImage) + ObservedGeneration int64 `json:"observedGeneration,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 07f5bf9..c8ea3b8 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -74,7 +74,7 @@ func (in *WatcherAPI) DeepCopyInto(out *WatcherAPI) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPI. @@ -130,6 +130,7 @@ func (in *WatcherAPIList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WatcherAPISpec) DeepCopyInto(out *WatcherAPISpec) { *out = *in + out.WatcherTemplate = in.WatcherTemplate } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPISpec. @@ -145,6 +146,13 @@ func (in *WatcherAPISpec) DeepCopy() *WatcherAPISpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WatcherAPIStatus) DeepCopyInto(out *WatcherAPIStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPIStatus. diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index 9330018..93b499c 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -35,13 +35,88 @@ spec: spec: description: WatcherAPISpec defines the desired state of WatcherAPI properties: - foo: - description: Foo is an example field of WatcherAPI. Edit watcherapi_types.go - to remove/update + databaseAccount: + default: watcher + description: DatabaseAccount - MariaDBAccount CR name used for watcher + DB, defaults to watcher type: string + databaseInstance: + description: MariaDB instance name Required to use the mariadb-operator + instance to create the DB and user + type: string + passwordSelectors: + default: + service: WatcherPassword + description: PasswordSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + service: + default: WatcherPassword + description: Service - Selector to get the watcher service user + password from the Secret + type: string + type: object + secret: + default: osp-secret + description: Secret containing all passwords / keys needed + type: string + required: + - databaseInstance type: object status: description: WatcherAPIStatus defines the observed state of WatcherAPI + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: Severity provides a classification of Reason code, + so the current situation is immediately understandable and + could act accordingly. It is meant for situations where Status=False + and it should be indicated if it is just informational, warning + (next reconciliation might fix it) or an error (e.g. DB create + issue and no actions to automatically resolve the issue can/should + be done). For conditions where Status=Unknown or Status=True + the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes + injected by the opentack-operator in the top-level CR (e.g. the + ContainerImage) + format: int64 + type: integer type: object type: object served: true diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index 9e27934..2eba14e 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -18,11 +18,25 @@ package controllers import ( "context" + "fmt" + "time" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/go-logr/logr" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" + "github.com/openstack-k8s-operators/watcher-operator/pkg/watcher" + + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" ) // WatcherAPIReconciler reconciles a WatcherAPI object @@ -30,27 +44,211 @@ type WatcherAPIReconciler struct { ReconcilerBase } +// GetLogger returns a logger object with a prefix of "controller.name" and +// additional controller context fields +func (r *WatcherAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("WatcherAPI") +} + //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis/status,verbs=get;update;patch //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the WatcherAPI object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.3/pkg/reconcile -func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - _ = req - // TODO(user): your logic here +func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + + Log := r.GetLogger(ctx) + instance := &watcherv1beta1.WatcherAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciling WatcherAPI instance '%s'", instance.Name)) + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + isNewInstance := instance.Status.Conditions == nil + // Save a copy of the conditions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can + // persist any changes. + defer func() { + condition.RestoreLastTransitionTimes( + &instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + err = r.initStatus(instance) + if err != nil { + return ctrl.Result{}, nil + } + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer()) || isNewInstance { + return ctrl.Result{}, nil + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + configVars := make(map[string]env.Setter) + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + Log.Info(fmt.Sprintf("[API] Get secret 1 '%s'", instance.Spec.Secret)) + ctrlResult, err := r.getSecret(ctx, instance, helper, instance.Spec.Secret, []string{instance.Spec.PasswordSelectors.Service}, &configVars) + if err != nil { + return ctrlResult, err + } + + err = r.generateServiceConfigs(ctx, instance, helper, &configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // all our input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // We reached the end of the Reconcile, update the Ready condition based on + // the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } + + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +// NOTE - jgilaber this function is WIP, currently implements a fraction of its +// functionality and will be expanded of further iteration to actually generate +// the service configs +func (r *WatcherAPIReconciler) generateServiceConfigs( + ctx context.Context, instance *watcherv1beta1.WatcherAPI, + helper *helper.Helper, envVars *map[string]env.Setter, +) error { + Log := r.GetLogger(ctx) + Log.Info("generateServiceConfigs - reconciling") + + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, helper, watcher.DatabaseCRName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return err + } + + ospSecret, _, err := secret.GetSecret(ctx, helper, instance.Spec.Secret, instance.Namespace) + if err != nil { + return err + } + + // replace by actual usage in future iterations + _ = db + _ = ospSecret + _ = envVars + + return nil +} + +func (r *WatcherAPIReconciler) getSecret( + ctx context.Context, + instance *watcherv1beta1.WatcherAPI, + helper *helper.Helper, + secretName string, + expectedFields []string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + secretNamespacedName := types.NamespacedName{Name: secretName, Namespace: instance.Namespace} + hash, result, err := secret.VerifySecret(ctx, secretNamespacedName, expectedFields, helper.GetClient(), time.Second*10) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + Log.Info(fmt.Sprintf("OpenStack secret %s not found", secretName)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return result, nil + } + // Add a prefix to the var name to avoid accidental collision with other + // non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secretName] = env.SetValue(hash) + + return ctrl.Result{}, nil +} + +func (r *WatcherAPIReconciler) reconcileDelete(ctx context.Context, instance *watcherv1beta1.WatcherAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconcile Service '%s' delete started", instance.Name)) + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) return ctrl.Result{}, nil } +func (r *WatcherAPIReconciler) initStatus(instance *watcherv1beta1.WatcherAPI) error { + + cl := condition.CreateList( + // Mark ReadyCondition as Unknown from the beginning, because the + // Reconcile function is in progress. If this condition is not marked + // as True and is still in the "Unknown" state, we `Mirror(` the actual + // failure/in-progress operation + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + + instance.Status.Conditions.Init(&cl) + + // Update the lastObserved generation before evaluating conditions + instance.Status.ObservedGeneration = instance.Generation + + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *WatcherAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index f3d1b6e..5bdd24a 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -58,3 +58,29 @@ func WatcherConditionGetter(name types.NamespacedName) condition.Conditions { instance := GetWatcher(name) return instance.Status.Conditions } + +func CreateWatcherAPI(name types.NamespacedName, spec map[string]interface{}) client.Object { + raw := map[string]interface{}{ + "apiVersion": "watcher.openstack.org/v1beta1", + "kind": "WatcherAPI", + "metadata": map[string]interface{}{ + "name": name.Name, + "namespace": name.Namespace, + }, + "spec": spec, + } + return th.CreateUnstructured(raw) +} + +func GetWatcherAPI(name types.NamespacedName) *watcherv1.WatcherAPI { + instance := &watcherv1.WatcherAPI{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, instance)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + return instance +} + +func WatcherAPIConditionGetter(name types.NamespacedName) condition.Conditions { + instance := GetWatcherAPI(name) + return instance.Status.Conditions +} diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go new file mode 100644 index 0000000..9ae9c16 --- /dev/null +++ b/tests/functional/watcherapi_controller_test.go @@ -0,0 +1,106 @@ +package functional + +import ( + . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports + . "github.com/onsi/gomega" //revive:disable:dot-imports + + //revive:disable-next-line:dot-imports + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" +) + +var _ = Describe("WatcherAPI controller with minimal spec values", func() { + When("A Watcher instance is created from minimal spec", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, MinimalWatcherSpec)) + }) + + It("should have the Spec fields defaulted", func() { + WatcherAPI := GetWatcherAPI(watcherTest.Instance) + Expect(WatcherAPI.Spec.DatabaseInstance).Should(Equal("openstack")) + Expect(WatcherAPI.Spec.DatabaseAccount).Should(Equal("watcher")) + Expect(WatcherAPI.Spec.Secret).Should(Equal("osp-secret")) + Expect(WatcherAPI.Spec.PasswordSelectors).Should(Equal(watcherv1beta1.PasswordSelector{Service: "WatcherPassword"})) + }) + + It("should have the Status fields initialized", func() { + WatcherAPI := GetWatcherAPI(watcherTest.Instance) + Expect(WatcherAPI.Status.ObservedGeneration).To(Equal(int64(0))) + }) + + It("should have a finalizer", func() { + // the reconciler loop adds the finalizer so we have to wait for + // it to run + Eventually(func() []string { + return GetWatcherAPI(watcherTest.Instance).Finalizers + }, timeout, interval).Should(ContainElement("openstack.org/watcherapi")) + }) + + }) +}) + +var _ = Describe("WatcherAPI controller", func() { + When("A WatcherAPI instance is created", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherSpec())) + }) + + It("should have the Spec fields defaulted", func() { + WatcherAPI := GetWatcherAPI(watcherTest.Instance) + Expect(WatcherAPI.Spec.DatabaseInstance).Should(Equal("openstack")) + Expect(WatcherAPI.Spec.DatabaseAccount).Should(Equal("watcher")) + Expect(WatcherAPI.Spec.Secret).Should(Equal("test-osp-secret")) + }) + + It("should have the Status fields initialized", func() { + WatcherAPI := GetWatcherAPI(watcherTest.Instance) + Expect(WatcherAPI.Status.ObservedGeneration).To(Equal(int64(0))) + }) + + It("should have unknown Conditions initialized", func() { + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("should have input not ready", func() { + th.ExpectCondition( + watcherTest.Instance, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("should have a finalizer", func() { + // the reconciler loop adds the finalizer so we have to wait for + // it to run + Eventually(func() []string { + return GetWatcherAPI(watcherTest.Instance).Finalizers + }, timeout, interval).Should(ContainElement("openstack.org/watcherapi")) + }) + + }) + + When("Watcher DB is created", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherSpec())) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + watcherTest.Instance.Namespace, + GetWatcherAPI(watcherTest.Instance).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + }) + }) + +})