From cc555f46ca0cd38e9487bddf5fbec8f5f2d15d53 Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:54:54 +0100 Subject: [PATCH 1/7] fix(makefile): missing webhook field Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7dbc01b1..9a38f499 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,8 @@ dev-setup: {'op': 'replace', 'path': '/webhooks/5/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/pods\",'caBundle':\"$${CA_BUNDLE}\"}},\ {'op': 'replace', 'path': '/webhooks/6/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/persistentvolumeclaims\",'caBundle':\"$${CA_BUNDLE}\"}},\ {'op': 'replace', 'path': '/webhooks/7/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/services\",'caBundle':\"$${CA_BUNDLE}\"}},\ - {'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\ + {'op': 'replace', 'path': '/webhooks/8/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}},\ + {'op': 'replace', 'path': '/webhooks/9/clientConfig', 'value':{'url':\"$${WEBHOOK_URL}/tenants\",'caBundle':\"$${CA_BUNDLE}\"}}\ ]" && \ kubectl patch crd tenants.capsule.clastix.io \ --type='json' -p="[\ From 8b3f22ca3909a1d16129a05d93bc3ce1bc77248d Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:56:49 +0100 Subject: [PATCH 2/7] feat(api): additional metadata for pods Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- api/v1beta2/tenant_types.go | 2 ++ api/v1beta2/zz_generated.deepcopy.go | 5 +++++ pkg/api/pod_options.go | 11 +++++++++++ pkg/api/zz_generated.deepcopy.go | 20 ++++++++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 pkg/api/pod_options.go diff --git a/api/v1beta2/tenant_types.go b/api/v1beta2/tenant_types.go index 93e01b75..3b43ce78 100644 --- a/api/v1beta2/tenant_types.go +++ b/api/v1beta2/tenant_types.go @@ -17,6 +17,8 @@ type TenantSpec struct { NamespaceOptions *NamespaceOptions `json:"namespaceOptions,omitempty"` // Specifies options for the Service, such as additional metadata or block of certain type of Services. Optional. ServiceOptions *api.ServiceOptions `json:"serviceOptions,omitempty"` + // Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata. + PodOptions *api.PodOptions `json:"podOptions,omitempty"` // Specifies the allowed StorageClasses assigned to the Tenant. // Capsule assures that all PersistentVolumeClaim resources created in the Tenant can use only one of the allowed StorageClasses. // A default value can be specified, and all the PersistentVolumeClaim resources created will inherit the declared class. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 7245688f..119085f4 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -716,6 +716,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = new(api.ServiceOptions) (*in).DeepCopyInto(*out) } + if in.PodOptions != nil { + in, out := &in.PodOptions, &out.PodOptions + *out = new(api.PodOptions) + (*in).DeepCopyInto(*out) + } if in.StorageClasses != nil { in, out := &in.StorageClasses, &out.StorageClasses *out = new(api.DefaultAllowedListSpec) diff --git a/pkg/api/pod_options.go b/pkg/api/pod_options.go new file mode 100644 index 00000000..4c122e02 --- /dev/null +++ b/pkg/api/pod_options.go @@ -0,0 +1,11 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package api + +// +kubebuilder:object:generate=true + +type PodOptions struct { + // Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional. + AdditionalMetadata *AdditionalMetadataSpec `json:"additionalMetadata,omitempty"` +} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 70c8265b..d09c76c3 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -213,6 +213,26 @@ func (in *NetworkPolicySpec) DeepCopy() *NetworkPolicySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodOptions) DeepCopyInto(out *PodOptions) { + *out = *in + if in.AdditionalMetadata != nil { + in, out := &in.AdditionalMetadata, &out.AdditionalMetadata + *out = new(AdditionalMetadataSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodOptions. +func (in *PodOptions) DeepCopy() *PodOptions { + if in == nil { + return nil + } + out := new(PodOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceQuotaSpec) DeepCopyInto(out *ResourceQuotaSpec) { *out = *in From f5d31d07faf23fa8bad641428e7303c7ff484730 Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:57:21 +0100 Subject: [PATCH 3/7] feat(kustomize): additional metadata for pods Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- .../crd/bases/capsule.clastix.io_tenants.yaml | 18 ++++++++++++++++++ config/install.yaml | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/config/crd/bases/capsule.clastix.io_tenants.yaml b/config/crd/bases/capsule.clastix.io_tenants.yaml index 75c54723..a1d0f616 100644 --- a/config/crd/bases/capsule.clastix.io_tenants.yaml +++ b/config/crd/bases/capsule.clastix.io_tenants.yaml @@ -2859,6 +2859,24 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pods deployed in the Tenant + namespaces, such as additional metadata. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule + operator places on any Pod resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. diff --git a/config/install.yaml b/config/install.yaml index d753fe84..ad90fb18 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -2437,6 +2437,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. type: boolean From 3ff21bc404fc6f1a542cc2241e053942fb8e643f Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:57:56 +0100 Subject: [PATCH 4/7] feat(helm): additional metadata for pods Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- charts/capsule/crds/tenant-crd.yaml | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/charts/capsule/crds/tenant-crd.yaml b/charts/capsule/crds/tenant-crd.yaml index 314016fb..8ae0d6ae 100644 --- a/charts/capsule/crds/tenant-crd.yaml +++ b/charts/capsule/crds/tenant-crd.yaml @@ -632,6 +632,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. type: boolean @@ -1737,6 +1753,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object priorityClasses: description: Specifies the allowed priorityClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant @@ -2869,6 +2901,22 @@ spec: - name type: object type: array + podOptions: + description: Specifies options for the Pod, such as additional metadata. Optional. + properties: + additionalMetadata: + description: Specifies additional labels and annotations the Capsule operator places on any Service resource in the Tenant. Optional. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object preventDeletion: description: Prevent accidental deletion of the Tenant. When enabled, the deletion request will be declined. From 92e2bb3668418083d8072c0e7da1858673cf5ead Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:58:21 +0100 Subject: [PATCH 5/7] docs: additional metadata for pods Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- docs/content/general/crds-apis.md | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/content/general/crds-apis.md b/docs/content/general/crds-apis.md index 696a41eb..29a5d794 100644 --- a/docs/content/general/crds-apis.md +++ b/docs/content/general/crds-apis.md @@ -2963,6 +2963,13 @@ TenantSpec defines the desired state of Tenant. Specifies the label to control the placement of pods on a given pool of worker nodes. All namespaces created within the Tenant will have the node selector annotation. This annotation tells the Kubernetes scheduler to place pods on the nodes having the selector label. Optional.
false + + podOptions + object + + Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata.
+ + false preventDeletion boolean @@ -4397,6 +4404,65 @@ NetworkPolicyPort describes a port to allow traffic on +### Tenant.spec.podOptions + + + +Specifies options for the Pods deployed in the Tenant namespaces, such as additional metadata. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
additionalMetadataobject + Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional.
+ 		false
+ + +### Tenant.spec.podOptions.additionalMetadata + + + +Specifies additional labels and annotations the Capsule operator places on any Pod resource in the Tenant. Optional. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
annotationsmap[string]string +
+ 		false
labelsmap[string]string +
+ 		false
+ + ### Tenant.spec.priorityClasses From f7e8d12f37d437b4ca3b720deab4778351c4ef59 Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:58:37 +0100 Subject: [PATCH 6/7] feat: additional metadata for pods Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- controllers/pod/errors.go | 30 +++++++++ controllers/pod/metadata.go | 130 ++++++++++++++++++++++++++++++++++++ main.go | 6 ++ 3 files changed, 166 insertions(+) create mode 100644 controllers/pod/errors.go create mode 100644 controllers/pod/metadata.go diff --git a/controllers/pod/errors.go b/controllers/pod/errors.go new file mode 100644 index 00000000..ceafff4a --- /dev/null +++ b/controllers/pod/errors.go @@ -0,0 +1,30 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package pod + +import "fmt" + +type NonTenantObjectError struct { + objectName string +} + +func NewNonTenantObject(objectName string) error { + return &NonTenantObjectError{objectName: objectName} +} + +func (n NonTenantObjectError) Error() string { + return fmt.Sprintf("Skipping labels sync for %s as it doesn't belong to tenant", n.objectName) +} + +type NoPodMetadataError struct { + objectName string +} + +func NewNoPodMetadata(objectName string) error { + return &NoPodMetadataError{objectName: objectName} +} + +func (n NoPodMetadataError) Error() string { + return fmt.Sprintf("Skipping labels sync for %s because no AdditionalLabels or AdditionalAnnotations presents in Tenant spec", n.objectName) +} diff --git a/controllers/pod/metadata.go b/controllers/pod/metadata.go new file mode 100644 index 00000000..9c9d772f --- /dev/null +++ b/controllers/pod/metadata.go @@ -0,0 +1,130 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package pod + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/utils" +) + +type MetadataReconciler struct { + Client client.Client +} + +func (m *MetadataReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + var pod corev1.Pod + + logger := log.FromContext(ctx) + + tenant, err := m.getTenant(ctx, request.NamespacedName, m.Client) + if err != nil { + noTenantObjError := &NonTenantObjectError{} + noPodMetaError := &NoPodMetadataError{} + + if errors.As(err, &noTenantObjError) || errors.As(err, &noPodMetaError) { + return reconcile.Result{}, nil + } + + logger.Error(err, fmt.Sprintf("Cannot get tenant corev1.Pod %s/%s", request.Namespace, request.Name)) + + return reconcile.Result{}, err + } + + err = m.Client.Get(ctx, request.NamespacedName, &pod) + if err != nil { + if apierr.IsNotFound(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, err + } + + _, err = controllerutil.CreateOrUpdate(ctx, m.Client, &pod, func() (err error) { + pod.SetLabels(m.sync(pod.GetLabels(), tenant.Spec.PodOptions.AdditionalMetadata.Labels)) + pod.SetAnnotations(m.sync(pod.GetAnnotations(), tenant.Spec.PodOptions.AdditionalMetadata.Annotations)) + + return nil + }) + + return reconcile.Result{}, err +} + +func (m *MetadataReconciler) getTenant(ctx context.Context, namespacedName types.NamespacedName, client client.Client) (*capsulev1beta2.Tenant, error) { + ns := &corev1.Namespace{} + tenant := &capsulev1beta2.Tenant{} + + if err := client.Get(ctx, types.NamespacedName{Name: namespacedName.Namespace}, ns); err != nil { + return nil, err + } + + capsuleLabel, _ := utils.GetTypeLabel(&capsulev1beta2.Tenant{}) + if _, ok := ns.GetLabels()[capsuleLabel]; !ok { + return nil, NewNonTenantObject(namespacedName.Name) + } + + if err := client.Get(ctx, types.NamespacedName{Name: ns.Labels[capsuleLabel]}, tenant); err != nil { + return nil, err + } + + if tenant.Spec.PodOptions == nil || tenant.Spec.PodOptions.AdditionalMetadata == nil { + return nil, NewNoPodMetadata(namespacedName.Name) + } + + return tenant, nil +} + +func (m *MetadataReconciler) sync(available map[string]string, tenantSpec map[string]string) map[string]string { + if tenantSpec != nil { + if available == nil { + return tenantSpec + } + + for key, value := range tenantSpec { + if available[key] != value { + available[key] = value + } + } + } + + return available +} + +func (m *MetadataReconciler) forOptionPerInstanceName(ctx context.Context) builder.ForOption { + return builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + return m.isNamespaceInTenant(ctx, object.GetNamespace()) + })) +} + +func (m *MetadataReconciler) isNamespaceInTenant(ctx context.Context, namespace string) bool { + tl := &capsulev1beta2.TenantList{} + if err := m.Client.List(ctx, tl, client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(".status.namespaces", namespace), + }); err != nil { + return false + } + + return len(tl.Items) > 0 +} + +func (m *MetadataReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}, m.forOptionPerInstanceName(ctx)). + Complete(m) +} diff --git a/main.go b/main.go index d0f7190d..4ae07cea 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( capsulev1beta1 "github.com/projectcapsule/capsule/api/v1beta1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" configcontroller "github.com/projectcapsule/capsule/controllers/config" + podlabelscontroller "github.com/projectcapsule/capsule/controllers/pod" "github.com/projectcapsule/capsule/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/controllers/rbac" "github.com/projectcapsule/capsule/controllers/resources" @@ -291,6 +292,11 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") } + if err = (&podlabelscontroller.MetadataReconciler{Client: manager.GetClient()}).SetupWithManager(ctx, manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "PodLabels") + os.Exit(1) + } + if err = (&pv.Controller{}).SetupWithManager(manager); err != nil { setupLog.Error(err, "unable to create controller", "controller", "PersistentVolume") os.Exit(1) From c4fdcefd0178fbd69b9cc612c4a989cd98d77758 Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Thu, 23 Nov 2023 14:58:49 +0100 Subject: [PATCH 7/7] test: additional metadata for pods Signed-off-by: Dario Tranchitella Co-authored-by: Giuseppe Chiesa --- e2e/pod_metadata_test.go | 115 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 e2e/pod_metadata_test.go diff --git a/e2e/pod_metadata_test.go b/e2e/pod_metadata_test.go new file mode 100644 index 00000000..98722cb7 --- /dev/null +++ b/e2e/pod_metadata_test.go @@ -0,0 +1,115 @@ +//go:build e2e + +// Copyright 2020-2021 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("adding metadata to Pod objects", func() { + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-metadata", + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: capsulev1beta2.OwnerListSpec{ + { + Name: "gatsby", + Kind: "User", + }, + }, + PodOptions: &api.PodOptions{ + AdditionalMetadata: &api.AdditionalMetadataSpec{ + Labels: map[string]string{ + "k8s.io/custom-label": "foo", + "clastix.io/custom-label": "bar", + }, + Annotations: map[string]string{ + "k8s.io/custom-annotation": "bizz", + "clastix.io/custom-annotation": "buzz", + }, + }, + }, + }, + } + + JustBeforeEach(func() { + EventuallyCreation(func() error { + + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + }) + + JustAfterEach(func() { + Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + }) + + It("should apply them to Pod", func() { + ns := NewNamespace("") + NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed()) + fmt.Sprint("namespace created") + //TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + fmt.Sprint("tenant contains list namespace") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-metadata", + Namespace: ns.GetName(), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + ImagePullPolicy: "IfNotPresent", + }, + }, + RestartPolicy: "Always", + }, + } + + EventuallyCreation(func() (err error) { + _, err = ownerClient(tnt.Spec.Owners[0]).CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{}) + + return + }).Should(Succeed()) + + By("checking additional labels", func() { + Eventually(func() (ok bool) { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: pod.GetName(), Namespace: ns.GetName()}, pod)).Should(Succeed()) + for k, v := range tnt.Spec.PodOptions.AdditionalMetadata.Labels { + ok, _ = HaveKeyWithValue(k, v).Match(pod.GetLabels()) + if !ok { + return false + } + } + return true + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + + By("checking additional annotations", func() { + Eventually(func() (ok bool) { + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: pod.GetName(), Namespace: ns.GetName()}, pod)).Should(Succeed()) + for k, v := range tnt.Spec.PodOptions.AdditionalMetadata.Annotations { + ok, _ = HaveKeyWithValue(k, v).Match(pod.GetAnnotations()) + if !ok { + return false + } + } + return true + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + }) + }) + +})